APPS-1167 Backup Aerospike cluster configuration (#167)

* save cluster configuration * add more logs for errors * add more logs for errors * codereview comments * implement configuration read * add mock methds * add data folder in tests * add comments * format imports * use retrieve for configs * misprint * rename restore to retrieve * mod tidy * fix build * Add default value to the logger CaptureShared configuration property * fix read password * put config in correct place for removeFiles = true * add possible values to error message * add debug logs * add incremental data path * Revert "add debug logs" This reverts commit 465d0a1. * update dependencies * use new way to fetch configuration * add schema mock * singletion shemas * fix some warnings * fix some warnings * remove comment * use ASClientPolicy * fix retrive backups from s3 * init schemas only once * linter warnings * check error * fetch active nodes * don't use cache for clients * don't use cache for clients --------- Co-authored-by: yrizhkov <[email protected]>
aerospike · Apr 2, 2024 · 066d258 · 066d258
1 parent bc41b13
commit 066d258
Show file tree

Hide file tree

Showing 36 changed files with 690 additions and 193 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "modules/aerospike-tools-backup"]
 	path = modules/aerospike-tools-backup
 	url = https://github.com/aerospike/aerospike-tools-backup.git
+[submodule "modules/schema/schemas"]
+	path = modules/schema/schemas
+	url = https://github.com/aerospike/schemas
diff --git a/.golangci.yml b/.golangci.yml
@@ -1,6 +1,6 @@
 run:
   timeout: 2m
-  skip-dirs:
+  exclude-dirs:
     - examples
     - modules
     - docs

diff --git a/cmd/backup/main.go b/cmd/backup/main.go
@@ -123,15 +123,15 @@ func readConfiguration() (*model.Config, error) {
 
 func runHTTPServer(ctx context.Context, backends service.BackendsHolder,
 	config *model.Config, scheduler quartz.Scheduler) error {
-	server := server.NewHTTPServer(backends, config, scheduler)
+	httpServer := server.NewHTTPServer(backends, config, scheduler)
 	go func() {
-		server.Start()
+		httpServer.Start()
 	}()
 
 	<-ctx.Done()
 	time.Sleep(time.Millisecond * 100) // wait for other goroutines to exit
 	// shutdown the HTTP server gracefully
-	if err := server.Shutdown(); err != nil {
+	if err := httpServer.Shutdown(); err != nil {
 		slog.Error("HTTP server shutdown failed", "error", err)
 		return err
 	}

diff --git a/docs/docs.go b/docs/docs.go
@@ -1281,6 +1281,49 @@ const docTemplate = `{
                 }
             }
         },
+        "/v1/retrieve/configuration/{name}/{timestamp}": {
+            "get": {
+                "produces": [
+                    "application/zip"
+                ],
+                "tags": [
+                    "Restore"
+                ],
+                "summary": "Retrieve Aerospike cluster configuration backup",
+                "operationId": "retrieveConfiguration",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Backup routine name",
+                        "name": "name",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "type": "integer",
+                        "format": "int64",
+                        "description": "Backup timestamp",
+                        "name": "timestamp",
+                        "in": "path",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "type": "file"
+                        }
+                    },
+                    "400": {
+                        "description": "Bad Request",
+                        "schema": {
+                            "type": "string"
+                        }
+                    }
+                }
+            }
+        },
         "/version": {
             "get": {
                 "tags": [

diff --git a/docs/openapi.json b/docs/openapi.json
@@ -1404,6 +1404,54 @@
         "x-codegen-request-body-name" : "request"
       }
     },
+    "/v1/retrieve/configuration/{name}/{timestamp}" : {
+      "get" : {
+        "operationId" : "retrieveConfiguration",
+        "parameters" : [ {
+          "description" : "Backup routine name",
+          "in" : "path",
+          "name" : "name",
+          "required" : true,
+          "schema" : {
+            "type" : "string"
+          }
+        }, {
+          "description" : "Backup timestamp",
+          "in" : "path",
+          "name" : "timestamp",
+          "required" : true,
+          "schema" : {
+            "format" : "int64",
+            "type" : "integer"
+          }
+        } ],
+        "responses" : {
+          "200" : {
+            "content" : {
+              "application/zip" : {
+                "schema" : {
+                  "format" : "binary",
+                  "type" : "string"
+                }
+              }
+            },
+            "description" : "OK"
+          },
+          "400" : {
+            "content" : {
+              "application/zip" : {
+                "schema" : {
+                  "type" : "string"
+                }
+              }
+            },
+            "description" : "Bad Request"
+          }
+        },
+        "summary" : "Retrieve Aerospike cluster configuration backup",
+        "tags" : [ "Restore" ]
+      }
+    },
     "/version" : {
       "get" : {
         "operationId" : "version",

diff --git a/docs/openapi.yaml b/docs/openapi.yaml
@@ -976,6 +976,40 @@ paths:
       tags:
       - Restore
       x-codegen-request-body-name: request
+  /v1/retrieve/configuration/{name}/{timestamp}:
+    get:
+      operationId: retrieveConfiguration
+      parameters:
+      - description: Backup routine name
+        in: path
+        name: name
+        required: true
+        schema:
+          type: string
+      - description: Backup timestamp
+        in: path
+        name: timestamp
+        required: true
+        schema:
+          format: int64
+          type: integer
+      responses:
+        "200":
+          content:
+            application/zip:
+              schema:
+                format: binary
+                type: string
+          description: OK
+        "400":
+          content:
+            application/zip:
+              schema:
+                type: string
+          description: Bad Request
+      summary: Retrieve Aerospike cluster configuration backup
+      tags:
+      - Restore
   /version:
     get:
       operationId: version

diff --git a/go.mod b/go.mod
@@ -3,12 +3,14 @@ module github.com/aerospike/backup
 go 1.22
 
 require (
-	github.com/aerospike/aerospike-client-go/v7 v7.1.0
-	github.com/aws/aws-sdk-go-v2 v1.24.0
-	github.com/aws/aws-sdk-go-v2/config v1.26.1
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5
-	github.com/aws/smithy-go v1.19.0
-	github.com/prometheus/client_golang v1.18.0
+	github.com/aerospike/aerospike-client-go/v7 v7.2.0
+	github.com/aerospike/aerospike-management-lib v1.3.0
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/config v1.27.10
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/aws/smithy-go v1.20.2
+	github.com/go-logr/logr v1.4.1
+	github.com/prometheus/client_golang v1.19.0
 	github.com/reugn/go-quartz v0.11.2
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
@@ -21,48 +23,52 @@ require (
 
 require (
 	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.12 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deckarep/golang-set/v2 v2.3.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/spec v0.20.6 // indirect
 	github.com/go-openapi/swag v0.19.15 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/swaggo/files/v2 v2.0.0 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
-	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.14.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect
-	google.golang.org/grpc v1.61.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	golang.org/x/tools v0.19.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )