From 48bdb30dc657d11829a86d97c69f215537501bb3 Mon Sep 17 00:00:00 2001 From: Eugene R Date: Sun, 31 Dec 2023 10:02:38 +0200 Subject: [PATCH] Add OpenAPI documentation for models (#105) * resolve linter warnings * Add OpenAPI documentation for models * Apply suggestions from code review Co-authored-by: Anton Korotkov <106995168+korotkov-aerospike@users.noreply.github.com> * update OpenAPI spec --------- Co-authored-by: Anton Korotkov <106995168+korotkov-aerospike@users.noreply.github.com> --- docs/swagger.json | 89 ++++++++++++++++++++- docs/swagger.yaml | 136 ++++++++++++++++++++++++++++++++- pkg/model/aerospike_cluster.go | 26 ++++--- pkg/model/backup_details.go | 8 +- pkg/model/backup_policy.go | 50 ++++++++---- pkg/model/backup_routine.go | 49 +++++++----- pkg/model/backup_state.go | 10 ++- pkg/model/config.go | 1 + pkg/model/http_server.go | 1 + pkg/model/restore_policy.go | 78 ++++++++++++++----- pkg/model/restore_request.go | 2 + pkg/model/secret_agent.go | 9 ++- pkg/model/storage.go | 22 ++++-- pkg/shared/backup.go | 2 +- pkg/shared/restore.go | 2 +- pkg/util/log_test.go | 1 + 16 files changed, 403 insertions(+), 83 deletions(-) diff --git a/docs/swagger.json b/docs/swagger.json index 7b03e55f..c616dfdd 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -929,149 +929,190 @@ }, "definitions": { "model.AerospikeCluster": { + "description": "AerospikeCluster represents the configuration for an Aerospike cluster for backup.", "type": "object", "properties": { "auth-mode": { + "description": "The authentication mode string (INTERNAL, EXTERNAL, EXTERNAL_INSECURE, PKI).", "type": "string" }, "host": { + "description": "The host that acts as the entry point to the cluster.", "type": "string" }, "password": { + "description": "The password for the cluster authentication.", "type": "string" }, "password-path": { + "description": "The file path with the password string, will take precedence over the password field.", "type": "string" }, "port": { + "description": "The port to connect to.", "type": "integer" }, "use-services-alternate": { + "description": "Whether should use \"services-alternate\" instead of \"services\" in info request during cluster tending.", "type": "boolean" }, "user": { + "description": "The username for the cluster authentication.", "type": "string" } } }, "model.BackupDetails": { + "description": "BackupDetails contains information about a backup.", "type": "object", "properties": { "key": { + "description": "The path to the backup files.", "type": "string" }, "last-modified": { + "description": "The backup time in the ISO 8601 format.", "type": "string" }, "size": { + "description": "The size of the backup in bytes.", "type": "integer" } } }, "model.BackupPolicy": { + "description": "BackupPolicy represents a scheduled backup policy.", "type": "object", "properties": { "bandwidth": { + "description": "Throttles backup write operations to the backup file(s) to not exceed the given\nbandwidth in MiB/s.", "type": "integer" }, "file-limit": { + "description": "File size limit (in MB) for --directory. If an .asb backup file crosses this size threshold,\na new backup file will be created.", "type": "integer" }, "filter-exp": { "type": "string" }, "max-records": { + "description": "An approximate limit for the number of records to process. Available in server 4.9 and above.", "type": "integer" }, "max-retries": { + "description": "Maximum number of retries before aborting the current transaction.", "type": "integer" }, "no-bins": { + "description": "Only backup record metadata (digest, TTL, generation count, key).", "type": "boolean" }, "no-indexes": { + "description": "Do not back up any secondary index definitions.", "type": "boolean" }, "no-records": { + "description": "Do not back up any record data (metadata or bin data).", "type": "boolean" }, "no-udfs": { + "description": "Do not back up any UDF modules.", "type": "boolean" }, "parallel": { + "description": "Maximum number of scan calls to run in parallel.", "type": "integer" }, "records-per-second": { + "description": "Limit total returned records per second (RPS). If RPS is zero (the default),\nthe records-per-second limit is not applied.", "type": "integer" }, "remove-artifacts": { + "description": "Clear directory or remove output file.", "type": "boolean" }, "remove-files": { + "description": "Whether to clear the output directory.", "type": "boolean" }, "retry-delay": { + "description": "RetryDelay defines the delay in milliseconds before retrying a failed operation.", "type": "integer" }, "socket-timeout": { + "description": "Socket timeout in milliseconds. If this value is 0, it is set to total-timeout.\nIf both are 0, there is no socket idle time limit.", "type": "integer" }, "total-timeout": { + "description": "Total socket timeout in milliseconds. Default is 0, that is, no timeout.", "type": "integer" } } }, "model.BackupRoutine": { + "description": "BackupRoutine represents a scheduled backup operation routine.", "type": "object", "properties": { "after-digest": { "type": "string" }, "backup-policy": { + "description": "The name of the corresponding bakup policy.", "type": "string" }, "bin-list": { + "description": "The list of backup bin names (optional, an empty list implies backing up all bins).", "type": "array", "items": { "type": "string" } }, "incr-interval": { + "description": "The interval for incremental backup in milliseconds (optional).", "type": "integer" }, "interval": { + "description": "The interval for full backup in milliseconds.", "type": "integer" }, "namespace": { + "description": "The name of the namespace to back up.", "type": "string" }, "node-list": { + "description": "The list of nodes in the Aerospike cluster to run the backup for.", "type": "array", "items": { "$ref": "#/definitions/model.Node" } }, "partition-list": { + "description": "Back up list of partition filters. Partition filters can be ranges, individual partitions,\nor records after a specific digest within a single partition.\nDefault number of partitions to back up: 0 to 4095: all partitions.", "type": "string" }, "secret-agent": { + "description": "The Secret Agent configuration for the routine (optional).", "type": "string" }, "set-list": { + "description": "The list of backup set names (optional, an empty list implies backing up all sets).", "type": "array", "items": { "type": "string" } }, "source-cluster": { + "description": "The name of the corresponding source cluster.", "type": "string" }, "storage": { + "description": "The name of the corresponding storage provider configuration.", "type": "string" } } }, "model.Config": { + "description": "Config represents the service configuration file.", "type": "object", "properties": { "aerospike-clusters": { @@ -1110,6 +1151,7 @@ } }, "model.HTTPServerConfig": { + "description": "HTTPServerConfig represents the service's HTTP server configuration.", "type": "object", "properties": { "address": { @@ -1124,6 +1166,7 @@ } }, "model.Node": { + "description": "Node represents the Aerospike node details.", "type": "object", "properties": { "ip": { @@ -1152,76 +1195,101 @@ } }, "model.RestoreNamespace": { + "description": "RestoreNamespace specifies an alternative namespace name for the restore operation.", "type": "object", "properties": { "destination": { + "description": "Destination namespace name.", "type": "string" }, "source": { + "description": "Original namespace name.", "type": "string" } } }, "model.RestorePolicy": { + "description": "RestorePolicy represents a policy for the restore operation.", "type": "object", "properties": { "bandwidth": { + "description": "Throttles read operations from the backup file(s) to not exceed the given I/O bandwidth\nin MiB/s and its database write operations to not exceed the given number of transactions\nper second.", "type": "integer" }, "batch-size": { + "description": "The max allowed number of records per an async batch write call.\nDefault is 128 with batch writes enabled, or 16 without batch writes.", "type": "integer" }, "bin-list": { + "description": "The bins to restore (optional, an empty list implies restoring all bins).", "type": "array", "items": { "type": "string" } }, "disable-batch-writes": { + "description": "Disables the use of batch writes when restoring records to the Aerospike cluster.\nBy default, the cluster is checked for batch write support.", "type": "boolean" }, "max-async-batches": { + "description": "The max number of outstanding async record batch write calls at a time.", "type": "integer" }, "namespace": { - "$ref": "#/definitions/model.RestoreNamespace" + "description": "Namespace details for the restore operation.\nBy default, the data is restored to the namespace from which it was taken.", + "allOf": [ + { + "$ref": "#/definitions/model.RestoreNamespace" + } + ] }, "no-generation": { + "description": "Records from backups take precedence. This option disables the generation check.\nWith this option, records from the backup always overwrite records that already exist in\nthe namespace, regardless of generation numbers.", "type": "boolean" }, "no-indexes": { + "description": "Do not restore any secondary index definitions.", "type": "boolean" }, "no-records": { + "description": "Do not restore any record data (metadata or bin data).\nBy default, record data, secondary index definitions, and UDF modules\nwill be restored.", "type": "boolean" }, "no-udfs": { + "description": "Do not restore any UDF modules.", "type": "boolean" }, "parallel": { + "description": "The number of client threads to spawn for writing to the cluster.", "type": "integer" }, "replace": { + "description": "Replace records. This controls how records from the backup overwrite existing records in\nthe namespace. By default, restoring a record from a backup only replaces the bins\ncontained in the backup; all other bins of an existing record remain untouched.", "type": "boolean" }, "set-list": { + "description": "The sets to restore (optional, an empty list implies restoring all sets).", "type": "array", "items": { "type": "string" } }, "timeout": { + "description": "Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs.", "type": "integer" }, "tps": { + "description": "Throttles read operations from the backup file(s) to not exceed the given I/O bandwidth\nin MiB/s and its database write operations to not exceed the given number of transactions\nper second.", "type": "integer" }, "unique": { + "description": "Existing records take precedence. With this option, only records that do not exist in\nthe namespace are restored, regardless of generation numbers. If a record exists in\nthe namespace, the record from the backup is ignored.", "type": "boolean" } } }, "model.RestoreRequest": { + "description": "RestoreRequest represents a restore operation request.", "type": "object", "properties": { "destination": { @@ -1239,6 +1307,7 @@ } }, "model.RestoreTimestampRequest": { + "description": "RestoreTimestampRequest represents a restore by timestamp operation request.", "type": "object", "properties": { "destination": { @@ -1260,16 +1329,19 @@ } }, "model.SecretAgent": { + "description": "SecretAgent represents the configuration of an Aerospike Secret Agent for a backup/restore operation.", "type": "object", "properties": { "address": { + "description": "Address of the Secret Agent.", "type": "string" }, "port": { + "description": "Port the Secret Agent is running on.", "type": "string" }, "timeout": { - "description": "timeout in milliseconds", + "description": "Timeout in milliseconds.", "type": "integer" }, "tls-ca": { @@ -1281,29 +1353,40 @@ } }, "model.Storage": { + "description": "Storage represents the configuration for a backup storage details.", "type": "object", "properties": { "path": { + "description": "The root path for the backup repository.", "type": "string" }, "s3-endpoint-override": { + "description": "An alternative endpoint for the S3 SDK to communicate (AWS S3 optional).", "type": "string" }, "s3-log-level": { "type": "string" }, "s3-profile": { + "description": "The S3 profile name (AWS S3 optional).", "type": "string" }, "s3-region": { + "description": "The S3 region string (AWS S3 optional).", "type": "string" }, "type": { - "$ref": "#/definitions/model.StorageType" + "description": "The type of the storage provider (0 - Local, 1 - AWS S3).", + "allOf": [ + { + "$ref": "#/definitions/model.StorageType" + } + ] } } }, "model.StorageType": { + "description": "StorageType represents the type of the backup storage.", "type": "integer", "enum": [ 0, diff --git a/docs/swagger.yaml b/docs/swagger.yaml index ea6973ad..4c2ae323 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,99 +1,161 @@ definitions: model.AerospikeCluster: + description: AerospikeCluster represents the configuration for an Aerospike cluster + for backup. properties: auth-mode: + description: The authentication mode string (INTERNAL, EXTERNAL, EXTERNAL_INSECURE, + PKI). type: string host: + description: The host that acts as the entry point to the cluster. type: string password: + description: The password for the cluster authentication. type: string password-path: + description: The file path with the password string, will take precedence + over the password field. type: string port: + description: The port to connect to. type: integer use-services-alternate: + description: Whether should use "services-alternate" instead of "services" + in info request during cluster tending. type: boolean user: + description: The username for the cluster authentication. type: string type: object model.BackupDetails: + description: BackupDetails contains information about a backup. properties: key: + description: The path to the backup files. type: string last-modified: + description: The backup time in the ISO 8601 format. type: string size: + description: The size of the backup in bytes. type: integer type: object model.BackupPolicy: + description: BackupPolicy represents a scheduled backup policy. properties: bandwidth: + description: |- + Throttles backup write operations to the backup file(s) to not exceed the given + bandwidth in MiB/s. type: integer file-limit: + description: |- + File size limit (in MB) for --directory. If an .asb backup file crosses this size threshold, + a new backup file will be created. type: integer filter-exp: type: string max-records: + description: An approximate limit for the number of records to process. Available + in server 4.9 and above. type: integer max-retries: + description: Maximum number of retries before aborting the current transaction. type: integer no-bins: + description: Only backup record metadata (digest, TTL, generation count, key). type: boolean no-indexes: + description: Do not back up any secondary index definitions. type: boolean no-records: + description: Do not back up any record data (metadata or bin data). type: boolean no-udfs: + description: Do not back up any UDF modules. type: boolean parallel: + description: Maximum number of scan calls to run in parallel. type: integer records-per-second: + description: |- + Limit total returned records per second (RPS). If RPS is zero (the default), + the records-per-second limit is not applied. type: integer remove-artifacts: + description: Clear directory or remove output file. type: boolean remove-files: + description: Whether to clear the output directory. type: boolean retry-delay: + description: RetryDelay defines the delay in milliseconds before retrying + a failed operation. type: integer socket-timeout: + description: |- + Socket timeout in milliseconds. If this value is 0, it is set to total-timeout. + If both are 0, there is no socket idle time limit. type: integer total-timeout: + description: Total socket timeout in milliseconds. Default is 0, that is, + no timeout. type: integer type: object model.BackupRoutine: + description: BackupRoutine represents a scheduled backup operation routine. properties: after-digest: type: string backup-policy: + description: The name of the corresponding bakup policy. type: string bin-list: + description: The list of backup bin names (optional, an empty list implies + backing up all bins). items: type: string type: array incr-interval: + description: The interval for incremental backup in milliseconds (optional). type: integer interval: + description: The interval for full backup in milliseconds. type: integer namespace: + description: The name of the namespace to back up. type: string node-list: + description: The list of nodes in the Aerospike cluster to run the backup + for. items: $ref: '#/definitions/model.Node' type: array partition-list: + description: |- + Back up list of partition filters. Partition filters can be ranges, individual partitions, + or records after a specific digest within a single partition. + Default number of partitions to back up: 0 to 4095: all partitions. type: string secret-agent: + description: The Secret Agent configuration for the routine (optional). type: string set-list: + description: The list of backup set names (optional, an empty list implies + backing up all sets). items: type: string type: array source-cluster: + description: The name of the corresponding source cluster. type: string storage: + description: The name of the corresponding storage provider configuration. type: string type: object model.Config: + description: Config represents the service configuration file. properties: aerospike-clusters: additionalProperties: @@ -119,6 +181,7 @@ definitions: type: object type: object model.HTTPServerConfig: + description: HTTPServerConfig represents the service's HTTP server configuration. properties: address: type: string @@ -128,6 +191,7 @@ definitions: $ref: '#/definitions/model.RateLimiterConfig' type: object model.Node: + description: Node represents the Aerospike node details. properties: ip: type: string @@ -146,52 +210,103 @@ definitions: type: array type: object model.RestoreNamespace: + description: RestoreNamespace specifies an alternative namespace name for the + restore operation. properties: destination: + description: Destination namespace name. type: string source: + description: Original namespace name. type: string type: object model.RestorePolicy: + description: RestorePolicy represents a policy for the restore operation. properties: bandwidth: + description: |- + Throttles read operations from the backup file(s) to not exceed the given I/O bandwidth + in MiB/s and its database write operations to not exceed the given number of transactions + per second. type: integer batch-size: + description: |- + The max allowed number of records per an async batch write call. + Default is 128 with batch writes enabled, or 16 without batch writes. type: integer bin-list: + description: The bins to restore (optional, an empty list implies restoring + all bins). items: type: string type: array disable-batch-writes: + description: |- + Disables the use of batch writes when restoring records to the Aerospike cluster. + By default, the cluster is checked for batch write support. type: boolean max-async-batches: + description: The max number of outstanding async record batch write calls + at a time. type: integer namespace: - $ref: '#/definitions/model.RestoreNamespace' + allOf: + - $ref: '#/definitions/model.RestoreNamespace' + description: |- + Namespace details for the restore operation. + By default, the data is restored to the namespace from which it was taken. no-generation: + description: |- + Records from backups take precedence. This option disables the generation check. + With this option, records from the backup always overwrite records that already exist in + the namespace, regardless of generation numbers. type: boolean no-indexes: + description: Do not restore any secondary index definitions. type: boolean no-records: + description: |- + Do not restore any record data (metadata or bin data). + By default, record data, secondary index definitions, and UDF modules + will be restored. type: boolean no-udfs: + description: Do not restore any UDF modules. type: boolean parallel: + description: The number of client threads to spawn for writing to the cluster. type: integer replace: + description: |- + Replace records. This controls how records from the backup overwrite existing records in + the namespace. By default, restoring a record from a backup only replaces the bins + contained in the backup; all other bins of an existing record remain untouched. type: boolean set-list: + description: The sets to restore (optional, an empty list implies restoring + all sets). items: type: string type: array timeout: + description: Timeout (ms) for Aerospike commands to write records, create + indexes and create UDFs. type: integer tps: + description: |- + Throttles read operations from the backup file(s) to not exceed the given I/O bandwidth + in MiB/s and its database write operations to not exceed the given number of transactions + per second. type: integer unique: + description: |- + Existing records take precedence. With this option, only records that do not exist in + the namespace are restored, regardless of generation numbers. If a record exists in + the namespace, the record from the backup is ignored. type: boolean type: object model.RestoreRequest: + description: RestoreRequest represents a restore operation request. properties: destination: $ref: '#/definitions/model.AerospikeCluster' @@ -203,6 +318,8 @@ definitions: $ref: '#/definitions/model.Storage' type: object model.RestoreTimestampRequest: + description: RestoreTimestampRequest represents a restore by timestamp operation + request. properties: destination: $ref: '#/definitions/model.AerospikeCluster' @@ -217,13 +334,17 @@ definitions: type: integer type: object model.SecretAgent: + description: SecretAgent represents the configuration of an Aerospike Secret Agent + for a backup/restore operation. properties: address: + description: Address of the Secret Agent. type: string port: + description: Port the Secret Agent is running on. type: string timeout: - description: timeout in milliseconds + description: Timeout in milliseconds. type: integer tls-ca: type: string @@ -231,21 +352,30 @@ definitions: type: boolean type: object model.Storage: + description: Storage represents the configuration for a backup storage details. properties: path: + description: The root path for the backup repository. type: string s3-endpoint-override: + description: An alternative endpoint for the S3 SDK to communicate (AWS S3 + optional). type: string s3-log-level: type: string s3-profile: + description: The S3 profile name (AWS S3 optional). type: string s3-region: + description: The S3 region string (AWS S3 optional). type: string type: - $ref: '#/definitions/model.StorageType' + allOf: + - $ref: '#/definitions/model.StorageType' + description: The type of the storage provider (0 - Local, 1 - AWS S3). type: object model.StorageType: + description: StorageType represents the type of the backup storage. enum: - 0 - 1 diff --git a/pkg/model/aerospike_cluster.go b/pkg/model/aerospike_cluster.go index 7958a90e..fee08c8d 100644 --- a/pkg/model/aerospike_cluster.go +++ b/pkg/model/aerospike_cluster.go @@ -8,16 +8,24 @@ import ( ) // AerospikeCluster represents the configuration for an Aerospike cluster for backup. +// @Description AerospikeCluster represents the configuration for an Aerospike cluster for backup. type AerospikeCluster struct { - pwdOnce sync.Once - pwd *string - Host *string `yaml:"host,omitempty" json:"host,omitempty"` - Port *int32 `yaml:"port,omitempty" json:"port,omitempty"` - UseServicesAlternate *bool `yaml:"use-services-alternate,omitempty" json:"use-services-alternate,omitempty"` - User *string `yaml:"user,omitempty" json:"user,omitempty"` - Password *string `yaml:"password,omitempty" json:"password,omitempty"` - PasswordPath *string `yaml:"password-path,omitempty" json:"password-path,omitempty"` - AuthMode *string `yaml:"auth-mode,omitempty" json:"auth-mode,omitempty"` + pwdOnce sync.Once + pwd *string + // The host that acts as the entry point to the cluster. + Host *string `yaml:"host,omitempty" json:"host,omitempty"` + // The port to connect to. + Port *int32 `yaml:"port,omitempty" json:"port,omitempty"` + // Whether should use "services-alternate" instead of "services" in info request during cluster tending. + UseServicesAlternate *bool `yaml:"use-services-alternate,omitempty" json:"use-services-alternate,omitempty"` + // The username for the cluster authentication. + User *string `yaml:"user,omitempty" json:"user,omitempty"` + // The password for the cluster authentication. + Password *string `yaml:"password,omitempty" json:"password,omitempty"` + // The file path with the password string, will take precedence over the password field. + PasswordPath *string `yaml:"password-path,omitempty" json:"password-path,omitempty"` + // The authentication mode string (INTERNAL, EXTERNAL, EXTERNAL_INSECURE, PKI). + AuthMode *string `yaml:"auth-mode,omitempty" json:"auth-mode,omitempty"` } // GetPassword tries to read and set the password once from PasswordPath, if it exists. diff --git a/pkg/model/backup_details.go b/pkg/model/backup_details.go index 1c946c31..8a312b6a 100644 --- a/pkg/model/backup_details.go +++ b/pkg/model/backup_details.go @@ -6,10 +6,14 @@ import ( ) // BackupDetails contains information about a backup. +// @Description BackupDetails contains information about a backup. type BackupDetails struct { - Key *string `yaml:"key,omitempty" json:"key,omitempty"` + // The path to the backup files. + Key *string `yaml:"key,omitempty" json:"key,omitempty"` + // The backup time in the ISO 8601 format. LastModified *time.Time `yaml:"last-modified,omitempty" json:"last-modified,omitempty"` - Size *int64 `yaml:"size,omitempty" json:"size,omitempty"` + // The size of the backup in bytes. + Size *int64 `yaml:"size,omitempty" json:"size,omitempty"` } // String satisfies the fmt.Stringer interface. diff --git a/pkg/model/backup_policy.go b/pkg/model/backup_policy.go index 51c7665c..aa74ab50 100644 --- a/pkg/model/backup_policy.go +++ b/pkg/model/backup_policy.go @@ -3,23 +3,43 @@ package model import "github.com/aerospike/backup/pkg/util" // BackupPolicy represents a scheduled backup policy. +// @Description BackupPolicy represents a scheduled backup policy. type BackupPolicy struct { - Parallel *int32 `yaml:"parallel,omitempty" json:"parallel,omitempty"` - SocketTimeout *uint32 `yaml:"socket-timeout,omitempty" json:"socket-timeout,omitempty"` - TotalTimeout *uint32 `yaml:"total-timeout,omitempty" json:"total-timeout,omitempty"` - MaxRetries *uint32 `yaml:"max-retries,omitempty" json:"max-retries,omitempty"` - RetryDelay *uint32 `yaml:"retry-delay,omitempty" json:"retry-delay,omitempty"` - RemoveFiles *bool `yaml:"remove-files,omitempty" json:"remove-files,omitempty"` - RemoveArtifacts *bool `yaml:"remove-artifacts,omitempty" json:"remove-artifacts,omitempty"` - NoBins *bool `yaml:"no-bins,omitempty" json:"no-bins,omitempty"` - NoRecords *bool `yaml:"no-records,omitempty" json:"no-records,omitempty"` - NoIndexes *bool `yaml:"no-indexes,omitempty" json:"no-indexes,omitempty"` - NoUdfs *bool `yaml:"no-udfs,omitempty" json:"no-udfs,omitempty"` - Bandwidth *uint64 `yaml:"bandwidth,omitempty" json:"bandwidth,omitempty"` - MaxRecords *uint64 `yaml:"max-records,omitempty" json:"max-records,omitempty"` + // Maximum number of scan calls to run in parallel. + Parallel *int32 `yaml:"parallel,omitempty" json:"parallel,omitempty"` + // Socket timeout in milliseconds. If this value is 0, it is set to total-timeout. + // If both are 0, there is no socket idle time limit. + SocketTimeout *uint32 `yaml:"socket-timeout,omitempty" json:"socket-timeout,omitempty"` + // Total socket timeout in milliseconds. Default is 0, that is, no timeout. + TotalTimeout *uint32 `yaml:"total-timeout,omitempty" json:"total-timeout,omitempty"` + // Maximum number of retries before aborting the current transaction. + MaxRetries *uint32 `yaml:"max-retries,omitempty" json:"max-retries,omitempty"` + // RetryDelay defines the delay in milliseconds before retrying a failed operation. + RetryDelay *uint32 `yaml:"retry-delay,omitempty" json:"retry-delay,omitempty"` + // Whether to clear the output directory. + RemoveFiles *bool `yaml:"remove-files,omitempty" json:"remove-files,omitempty"` + // Clear directory or remove output file. + RemoveArtifacts *bool `yaml:"remove-artifacts,omitempty" json:"remove-artifacts,omitempty"` + // Only backup record metadata (digest, TTL, generation count, key). + NoBins *bool `yaml:"no-bins,omitempty" json:"no-bins,omitempty"` + // Do not back up any record data (metadata or bin data). + NoRecords *bool `yaml:"no-records,omitempty" json:"no-records,omitempty"` + // Do not back up any secondary index definitions. + NoIndexes *bool `yaml:"no-indexes,omitempty" json:"no-indexes,omitempty"` + // Do not back up any UDF modules. + NoUdfs *bool `yaml:"no-udfs,omitempty" json:"no-udfs,omitempty"` + // Throttles backup write operations to the backup file(s) to not exceed the given + // bandwidth in MiB/s. + Bandwidth *uint64 `yaml:"bandwidth,omitempty" json:"bandwidth,omitempty"` + // An approximate limit for the number of records to process. Available in server 4.9 and above. + MaxRecords *uint64 `yaml:"max-records,omitempty" json:"max-records,omitempty"` + // Limit total returned records per second (RPS). If RPS is zero (the default), + // the records-per-second limit is not applied. RecordsPerSecond *uint32 `yaml:"records-per-second,omitempty" json:"records-per-second,omitempty"` - FileLimit *uint64 `yaml:"file-limit,omitempty" json:"file-limit,omitempty"` - FilterExp *string `yaml:"filter-exp,omitempty" json:"filter-exp,omitempty"` + // File size limit (in MB) for --directory. If an .asb backup file crosses this size threshold, + // a new backup file will be created. + FileLimit *uint64 `yaml:"file-limit,omitempty" json:"file-limit,omitempty"` + FilterExp *string `yaml:"filter-exp,omitempty" json:"filter-exp,omitempty"` } // CopySMDDisabled creates a new instance of the BackupPolicy struct with identical field values. diff --git a/pkg/model/backup_routine.go b/pkg/model/backup_routine.go index eebec050..19dbbdda 100644 --- a/pkg/model/backup_routine.go +++ b/pkg/model/backup_routine.go @@ -5,25 +5,39 @@ import ( ) const ( - MIN_FULL_BACKUP_INTERVAL_MILLIS int64 = 10000 - MIN_INCR_BACKUP_INTERVAL_MILLIS int64 = 1000 + minimumFullBackupIntervalMillis int64 = 10000 + minimumIncrBackupIntervalMillis int64 = 1000 ) // BackupRoutine represents a scheduled backup operation routine. +// @Description BackupRoutine represents a scheduled backup operation routine. type BackupRoutine struct { - BackupPolicy string `yaml:"backup-policy,omitempty" json:"backup-policy,omitempty"` - SourceCluster string `yaml:"source-cluster,omitempty" json:"source-cluster,omitempty"` - Storage string `yaml:"storage,omitempty" json:"storage,omitempty"` - SecretAgent *string `yaml:"secret-agent,omitempty" json:"secret-agent,omitempty"` + // The name of the corresponding bakup policy. + BackupPolicy string `yaml:"backup-policy,omitempty" json:"backup-policy,omitempty"` + // The name of the corresponding source cluster. + SourceCluster string `yaml:"source-cluster,omitempty" json:"source-cluster,omitempty"` + // The name of the corresponding storage provider configuration. + Storage string `yaml:"storage,omitempty" json:"storage,omitempty"` + // The Secret Agent configuration for the routine (optional). + SecretAgent *string `yaml:"secret-agent,omitempty" json:"secret-agent,omitempty"` - IntervalMillis *int64 `yaml:"interval,omitempty" json:"interval,omitempty"` + // The interval for full backup in milliseconds. + IntervalMillis *int64 `yaml:"interval,omitempty" json:"interval,omitempty"` + // The interval for incremental backup in milliseconds (optional). IncrIntervalMillis *int64 `yaml:"incr-interval,omitempty" json:"incr-interval,omitempty"` - Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` - SetList []string `yaml:"set-list,omitempty" json:"set-list,omitempty"` - BinList []string `yaml:"bin-list,omitempty" json:"bin-list,omitempty"` - NodeList []Node `yaml:"node-list,omitempty" json:"node-list,omitempty"` + // The name of the namespace to back up. + Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` + // The list of backup set names (optional, an empty list implies backing up all sets). + SetList []string `yaml:"set-list,omitempty" json:"set-list,omitempty"` + // The list of backup bin names (optional, an empty list implies backing up all bins). + BinList []string `yaml:"bin-list,omitempty" json:"bin-list,omitempty"` + // The list of nodes in the Aerospike cluster to run the backup for. + NodeList []Node `yaml:"node-list,omitempty" json:"node-list,omitempty"` + // Back up list of partition filters. Partition filters can be ranges, individual partitions, + // or records after a specific digest within a single partition. + // Default number of partitions to back up: 0 to 4095: all partitions. PartitionList *string `yaml:"partition-list,omitempty" json:"partition-list,omitempty"` AfterDigest *string `yaml:"after-digest,omitempty" json:"after-digest,omitempty"` } @@ -43,13 +57,13 @@ func (r *BackupRoutine) Validate() error { return routineValidationError("namespace") } if r.IntervalMillis == nil { - return routineValidationError("IntervalMillis") + return routineValidationError("interval") } - if *r.IntervalMillis < MIN_FULL_BACKUP_INTERVAL_MILLIS { - return fmt.Errorf("minimum backup interval is %d", MIN_FULL_BACKUP_INTERVAL_MILLIS) + if *r.IntervalMillis < minimumFullBackupIntervalMillis { + return fmt.Errorf("minimum full backup interval is %d", minimumFullBackupIntervalMillis) } - if r.IncrIntervalMillis != nil && *r.IncrIntervalMillis < MIN_INCR_BACKUP_INTERVAL_MILLIS { - return fmt.Errorf("minimum incremental backup interval is %d", MIN_INCR_BACKUP_INTERVAL_MILLIS) + if r.IncrIntervalMillis != nil && *r.IncrIntervalMillis < minimumIncrBackupIntervalMillis { + return fmt.Errorf("minimum incremental backup interval is %d", minimumIncrBackupIntervalMillis) } return nil } @@ -58,7 +72,8 @@ func routineValidationError(field string) error { return fmt.Errorf("%s specification for backup routine is required", field) } -// Node represents an Aerospike node details. +// Node represents the Aerospike node details. +// @Description Node represents the Aerospike node details. type Node struct { IP string `yaml:"ip" json:"ip"` Port int `yaml:"port" json:"port"` diff --git a/pkg/model/backup_state.go b/pkg/model/backup_state.go index ccaf4641..bd4add39 100644 --- a/pkg/model/backup_state.go +++ b/pkg/model/backup_state.go @@ -5,11 +5,15 @@ import ( "time" ) -// BackupState represents the state of a backup. +// BackupState represents the state of a backup routine. +// @Description BackupState represents the state of a backup routine. type BackupState struct { + // Last time the full backup was performed. LastFullRun time.Time `yaml:"last-run,omitempty" json:"last-run,omitempty"` + // Last time the incremental backup was performed. LastIncrRun time.Time `yaml:"last-incr-run,omitempty" json:"last-incr-run,omitempty"` - Performed int `yaml:"performed,omitempty" json:"performed,omitempty"` + // The number of successful full backups created for the routine. + Performed int `yaml:"performed,omitempty" json:"performed,omitempty"` } // String satisfies the fmt.Stringer interface. @@ -21,7 +25,7 @@ func (state BackupState) String() string { return string(backupState) } -// NewBackupState returns a BackupState with default values. +// NewBackupState returns a BackupState with the default values. func NewBackupState() *BackupState { return &BackupState{} } diff --git a/pkg/model/config.go b/pkg/model/config.go index a2c5f73a..40e3708a 100644 --- a/pkg/model/config.go +++ b/pkg/model/config.go @@ -6,6 +6,7 @@ import ( ) // Config represents the service configuration file. +// @Description Config represents the service configuration file. // //nolint:lll type Config struct { diff --git a/pkg/model/http_server.go b/pkg/model/http_server.go index 5722d8ae..5b92351b 100644 --- a/pkg/model/http_server.go +++ b/pkg/model/http_server.go @@ -1,6 +1,7 @@ package model // HTTPServerConfig represents the service's HTTP server configuration. +// @Description HTTPServerConfig represents the service's HTTP server configuration. type HTTPServerConfig struct { Address string `yaml:"address,omitempty" json:"address,omitempty"` Port int `yaml:"port,omitempty" json:"port,omitempty"` diff --git a/pkg/model/restore_policy.go b/pkg/model/restore_policy.go index b542a57a..031f7a5b 100644 --- a/pkg/model/restore_policy.go +++ b/pkg/model/restore_policy.go @@ -2,29 +2,67 @@ package model import "errors" -// RestorePolicy represents a policy for restore operation. +// RestorePolicy represents a policy for the restore operation. +// @Description RestorePolicy represents a policy for the restore operation. type RestorePolicy struct { - Parallel *uint32 `json:"parallel,omitempty"` - NoRecords *bool `json:"no-records,omitempty"` - NoIndexes *bool `json:"no-indexes,omitempty"` - NoUdfs *bool `json:"no-udfs,omitempty"` - Timeout *uint32 `json:"timeout,omitempty"` - DisableBatchWrites *bool `json:"disable-batch-writes,omitempty"` - MaxAsyncBatches *uint32 `json:"max-async-batches,omitempty"` - BatchSize *uint32 `json:"batch-size,omitempty"` - Namespace *RestoreNamespace `json:"namespace,omitempty"` - SetList []string `json:"set-list,omitempty"` - BinList []string `json:"bin-list,omitempty"` - Replace *bool `json:"replace,omitempty"` - Unique *bool `json:"unique,omitempty"` - NoGeneration *bool `json:"no-generation,omitempty"` - Bandwidth *uint64 `json:"bandwidth,omitempty"` - Tps *uint32 `json:"tps,omitempty"` + // The number of client threads to spawn for writing to the cluster. + Parallel *uint32 `json:"parallel,omitempty"` + // Do not restore any record data (metadata or bin data). + // By default, record data, secondary index definitions, and UDF modules + // will be restored. + NoRecords *bool `json:"no-records,omitempty"` + // Do not restore any secondary index definitions. + NoIndexes *bool `json:"no-indexes,omitempty"` + // Do not restore any UDF modules. + NoUdfs *bool `json:"no-udfs,omitempty"` + // Timeout (ms) for Aerospike commands to write records, create indexes and create UDFs. + Timeout *uint32 `json:"timeout,omitempty"` + // Disables the use of batch writes when restoring records to the Aerospike cluster. + // By default, the cluster is checked for batch write support. + DisableBatchWrites *bool `json:"disable-batch-writes,omitempty"` + // The max number of outstanding async record batch write calls at a time. + MaxAsyncBatches *uint32 `json:"max-async-batches,omitempty"` + // The max allowed number of records per an async batch write call. + // Default is 128 with batch writes enabled, or 16 without batch writes. + BatchSize *uint32 `json:"batch-size,omitempty"` + // Namespace details for the restore operation. + // By default, the data is restored to the namespace from which it was taken. + Namespace *RestoreNamespace `json:"namespace,omitempty"` + // The sets to restore (optional, an empty list implies restoring all sets). + SetList []string `json:"set-list,omitempty"` + // The bins to restore (optional, an empty list implies restoring all bins). + BinList []string `json:"bin-list,omitempty"` + // Replace records. This controls how records from the backup overwrite existing records in + // the namespace. By default, restoring a record from a backup only replaces the bins + // contained in the backup; all other bins of an existing record remain untouched. + Replace *bool `json:"replace,omitempty"` + // Existing records take precedence. With this option, only records that do not exist in + // the namespace are restored, regardless of generation numbers. If a record exists in + // the namespace, the record from the backup is ignored. + Unique *bool `json:"unique,omitempty"` + // Records from backups take precedence. This option disables the generation check. + // With this option, records from the backup always overwrite records that already exist in + // the namespace, regardless of generation numbers. + NoGeneration *bool `json:"no-generation,omitempty"` + // Throttles read operations from the backup file(s) to not exceed the given I/O bandwidth + // in MiB/s and its database write operations to not exceed the given number of transactions + // per second. + Bandwidth *uint64 `json:"bandwidth,omitempty"` + // Throttles read operations from the backup file(s) to not exceed the given I/O bandwidth + // in MiB/s and its database write operations to not exceed the given number of transactions + // per second. + Tps *uint32 `json:"tps,omitempty"` } +// RestoreNamespace specifies an alternative namespace name for the restore +// operation, where Source is the original namespace name and Destination is +// the namespace name to which the backup data is to be restored. +// +// @Description RestoreNamespace specifies an alternative namespace name for the restore +// @Description operation. type RestoreNamespace struct { - Source *string `json:"source,omitempty"` - Destination *string `json:"destination,omitempty"` + Source *string `json:"source,omitempty"` // Original namespace name. + Destination *string `json:"destination,omitempty"` // Destination namespace name. } // Validate validates the restore policy. @@ -40,7 +78,7 @@ func (p *RestorePolicy) Validate() error { return nil } -// Validate validates restore namespace. +// Validate validates the restore namespace. func (n *RestoreNamespace) Validate() error { if n.Source == nil { return errors.New("source namespace is not specified") diff --git a/pkg/model/restore_request.go b/pkg/model/restore_request.go index c6e82045..61fe29b8 100644 --- a/pkg/model/restore_request.go +++ b/pkg/model/restore_request.go @@ -6,6 +6,7 @@ import ( ) // RestoreRequest represents a restore operation request. +// @Description RestoreRequest represents a restore operation request. type RestoreRequest struct { DestinationCuster *AerospikeCluster `json:"destination,omitempty"` Policy *RestorePolicy `json:"policy,omitempty"` @@ -21,6 +22,7 @@ type RestoreRequestInternal struct { } // RestoreTimestampRequest represents a restore by timestamp operation request. +// @Description RestoreTimestampRequest represents a restore by timestamp operation request. type RestoreTimestampRequest struct { DestinationCuster *AerospikeCluster `json:"destination,omitempty"` Policy *RestorePolicy `json:"policy,omitempty"` diff --git a/pkg/model/secret_agent.go b/pkg/model/secret_agent.go index 69574ce5..1502b59c 100644 --- a/pkg/model/secret_agent.go +++ b/pkg/model/secret_agent.go @@ -4,10 +4,13 @@ package model // for a backup/restore operation. // Aerospike Secret Agent acts as a proxy layer between Aerospike server and one or more // external secrets management services, fetching secrets on behalf of the server. +// +// @Description SecretAgent represents the configuration of an Aerospike Secret Agent +// @Description for a backup/restore operation. type SecretAgent struct { - Address string `yaml:"address,omitempty" json:"address,omitempty"` - Port string `yaml:"port,omitempty" json:"port,omitempty"` - Timeout int32 `yaml:"timeout,omitempty" json:"timeout,omitempty"` // timeout in milliseconds + Address string `yaml:"address,omitempty" json:"address,omitempty"` // Address of the Secret Agent. + Port string `yaml:"port,omitempty" json:"port,omitempty"` // Port the Secret Agent is running on. + Timeout int32 `yaml:"timeout,omitempty" json:"timeout,omitempty"` // Timeout in milliseconds. TLSCAString string `yaml:"tls-ca,omitempty" json:"tls-ca,omitempty"` TLSEnabled bool `yaml:"tls-enabled,omitempty" json:"tls-enabled,omitempty"` } diff --git a/pkg/model/storage.go b/pkg/model/storage.go index a45610f7..7d79664f 100644 --- a/pkg/model/storage.go +++ b/pkg/model/storage.go @@ -3,20 +3,28 @@ package model import ( "errors" "fmt" + "github.com/aws/smithy-go/ptr" ) // Storage represents the configuration for a backup storage details. +// @Description Storage represents the configuration for a backup storage details. type Storage struct { - Type StorageType `yaml:"type,omitempty" json:"type,omitempty"` - Path *string `yaml:"path,omitempty" json:"path,omitempty"` - S3Region *string `yaml:"s3-region,omitempty" json:"s3-region,omitempty"` - S3Profile *string `yaml:"s3-profile,omitempty" json:"s3-profile,omitempty"` - S3EndpointOverride *string `yaml:"s3-endpoint-override,omitempty" json:"s3-endpoint-override,omitempty"` - S3LogLevel *string `yaml:"s3-log-level,omitempty" json:"s3-log-level,omitempty"` + // The type of the storage provider (0 - Local, 1 - AWS S3). + Type StorageType `yaml:"type,omitempty" json:"type,omitempty"` + // The root path for the backup repository. + Path *string `yaml:"path,omitempty" json:"path,omitempty"` + // The S3 region string (AWS S3 optional). + S3Region *string `yaml:"s3-region,omitempty" json:"s3-region,omitempty"` + // The S3 profile name (AWS S3 optional). + S3Profile *string `yaml:"s3-profile,omitempty" json:"s3-profile,omitempty"` + // An alternative endpoint for the S3 SDK to communicate (AWS S3 optional). + S3EndpointOverride *string `yaml:"s3-endpoint-override,omitempty" json:"s3-endpoint-override,omitempty"` + S3LogLevel *string `yaml:"s3-log-level,omitempty" json:"s3-log-level,omitempty"` } // StorageType represents the type of the backup storage. +// @Description StorageType represents the type of the backup storage. type StorageType int const ( @@ -43,6 +51,7 @@ func (s *Storage) Validate() error { return nil } +// validateType validates the storage provider type. func (s *Storage) validateType() error { switch s.Type { case Local, S3: @@ -52,6 +61,7 @@ func (s *Storage) validateType() error { } } +// SetDefaultProfile sets the "default" profile if not set. func (s *Storage) SetDefaultProfile() { if s.Type == S3 && s.S3Profile == nil { s.S3Profile = ptr.String("default") diff --git a/pkg/shared/backup.go b/pkg/shared/backup.go index 8bc281b8..fc882337 100644 --- a/pkg/shared/backup.go +++ b/pkg/shared/backup.go @@ -44,7 +44,7 @@ func NewBackup() *BackupShared { // BackupRun calls the backup_run function from the asbackup shared library. // -//nolint:funlen +//nolint:funlen,gocritic func (b *BackupShared) BackupRun(backupRoutine *model.BackupRoutine, backupPolicy *model.BackupPolicy, cluster *model.AerospikeCluster, storage *model.Storage, secretAgent *model.SecretAgent, opts BackupOptions) *BackupStat { diff --git a/pkg/shared/restore.go b/pkg/shared/restore.go index 6a8e052e..1058a242 100644 --- a/pkg/shared/restore.go +++ b/pkg/shared/restore.go @@ -41,7 +41,7 @@ func NewRestore() *RestoreShared { // RestoreRun calls the restore_run function from the asrestore shared library. // -//nolint:funlen +//nolint:funlen,gocritic func (r *RestoreShared) RestoreRun(restoreRequest *model.RestoreRequestInternal) bool { // lock to restrict parallel execution (shared library limitation) r.Lock() diff --git a/pkg/util/log_test.go b/pkg/util/log_test.go index 47a920ff..ca2ed729 100644 --- a/pkg/util/log_test.go +++ b/pkg/util/log_test.go @@ -20,6 +20,7 @@ func Test_libLogRegex(t *testing.T) { LogCaptured(strings.Join(tests, "\n")) } +//nolint:lll func Test_ignoreLinesRegex(t *testing.T) { testStrings := []string{ "time=2023-12-18T09:39:31.311Z level=ERROR source=/app/pkg/util/log.go:25 msg=\"[src/main/aerospike/as_pipe.c:210][read_file] Failed to open /proc/sys/net/core/rmem_max for reading\"",