From 3f3fd6be25d5605bab3239085a24cd0f1654db3b Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 17 Sep 2024 10:10:32 +1000 Subject: [PATCH] Mongo dump/restore cleanup so they share code, we had 2 copies of the setup code --- api/ws/handlers/system.go | 7 ++- api/ws/wsHelpers/sync-mongo-dump.go | 60 +++----------------------- api/ws/wsHelpers/sync-mongo-restore.go | 53 +---------------------- api/ws/wsHelpers/sync-mongo.go | 60 ++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 107 deletions(-) diff --git a/api/ws/handlers/system.go b/api/ws/handlers/system.go index aef432af..2fadc8a8 100644 --- a/api/ws/handlers/system.go +++ b/api/ws/handlers/system.go @@ -40,7 +40,12 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) ( hctx.Svcs.Log.Infof("PIXLISE Backup Requested, will be written to bucket: %v", hctx.Svcs.Config.DataBackupBucket) // Run MongoDump, save to a local archive file - dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName)) + dump, err := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName)) + + if err != nil { + hctx.Svcs.Log.Errorf("Failed to create dump instance: %v", err) + return nil, err + } err = dump.Init() if err != nil { diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index a903ac93..00b15776 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -4,10 +4,7 @@ import ( "fmt" "os" "path" - "strings" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/options" "github.com/mongodb/mongo-tools/mongodump" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/core/fileaccess" @@ -18,56 +15,13 @@ import ( var dataBackupLocalPath = "./backup" var dataBackupS3Path = "DB" -func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) *mongodump.MongoDump { - var toolOptions *options.ToolOptions - - ssl := options.SSL{} - - isLocal := strings.Contains(mongoDetails.Host, "localhost") && len(mongoDetails.User) <= 0 && len(mongoDetails.Password) <= 0 - - if !isLocal { - ssl = options.SSL{ - UseSSL: true, - SSLCAFile: "./global-bundle.pem", - SSLPEMKeyFile: "./global-bundle.pem", - } - } - - auth := options.Auth{ - Username: mongoDetails.User, - Password: mongoDetails.Password, - } - - connection := &options.Connection{ - Host: mongoDetails.Host, - } - - // Trim excess - protocolPrefix := "mongodb://" - connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) - - connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "") - - logger.Infof("MongoDump connecting to: %v, user %v, db-to-dump: %v...", connectionURI, auth.Username, dbName) - - uri, err := options.NewURI(connectionURI) +func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) (*mongodump.MongoDump, error) { + toolOptions, err := makeMongoToolOptions(mongoDetails, logger, dbName) if err != nil { - logger.Errorf("%v", err) - return nil + return nil, err } - retryWrites := false - - toolOptions = &options.ToolOptions{ - RetryWrites: &retryWrites, - SSL: &ssl, - Connection: connection, - Auth: &auth, - Verbosity: &options.Verbosity{}, - URI: uri, - } - - toolOptions.Namespace = &options.Namespace{DB: dbName} + logger.Infof("MongoDump connecting to: %v, user %v, db-to-dump: %v...", toolOptions.URI.ConnectionString, toolOptions.Auth.Username, dbName) outputOptions := &mongodump.OutputOptions{ Out: dataBackupLocalPath, @@ -78,15 +32,11 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails } inputOptions := &mongodump.InputOptions{} - log.SetVerbosity(nil /*toolOptions.Verbosity*/) - lw := LogWriter{logger: logger} - log.SetWriter(lw) - return &mongodump.MongoDump{ ToolOptions: toolOptions, InputOptions: inputOptions, OutputOptions: outputOptions, - } + }, nil } func UploadArchive(svcs *services.APIServices) error { diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 6fdf969a..316d8cb9 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -5,8 +5,6 @@ import ( "path" "strings" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/options" "github.com/mongodb/mongo-tools/mongorestore" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/core/fileaccess" @@ -15,55 +13,12 @@ import ( ) func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) { - var toolOptions *options.ToolOptions - - ssl := options.SSL{} - - isLocal := strings.Contains(mongoDetails.Host, "localhost") && len(mongoDetails.User) <= 0 && len(mongoDetails.Password) <= 0 - - if !isLocal { - ssl = options.SSL{ - UseSSL: true, - SSLCAFile: "./global-bundle.pem", - SSLPEMKeyFile: "./global-bundle.pem", - } - } - - auth := options.Auth{ - Username: mongoDetails.User, - Password: mongoDetails.Password, - } - - connection := &options.Connection{ - Host: mongoDetails.Host, - } - - // Trim excess - protocolPrefix := "mongodb://" - connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) - - connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "") - - logger.Infof("MongoRestore connecting to: %v, user %v, restore-to-db: %v, restore-from-db: %v...", connectionURI, auth.Username, restoreToDBName, restoreFromDBName) - - uri, err := options.NewURI(connectionURI) + toolOptions, err := makeMongoToolOptions(mongoDetails, logger, restoreToDBName) if err != nil { - logger.Errorf("%v", err) return nil, err } - retryWrites := false - - toolOptions = &options.ToolOptions{ - RetryWrites: &retryWrites, - SSL: &ssl, - Connection: connection, - Auth: &auth, - Verbosity: &options.Verbosity{}, - URI: uri, - } - - toolOptions.Namespace = &options.Namespace{DB: restoreToDBName} + logger.Infof("MongoRestore connecting to: %v, user %v, restore-to-db: %v, restore-from-db: %v...", toolOptions.URI.ConnectionString, toolOptions.Auth.Username, restoreToDBName, restoreFromDBName) outputOptions := &mongorestore.OutputOptions{ NumParallelCollections: 1, @@ -85,10 +40,6 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta NSTo: []string{restoreToDBName}, } - log.SetVerbosity(nil /*toolOptions.Verbosity*/) - lw := LogWriter{logger: logger} - log.SetWriter(lw) - return mongorestore.New(mongorestore.Options{ ToolOptions: toolOptions, InputOptions: inputOptions, diff --git a/api/ws/wsHelpers/sync-mongo.go b/api/ws/wsHelpers/sync-mongo.go index 105ed470..af7ef88a 100644 --- a/api/ws/wsHelpers/sync-mongo.go +++ b/api/ws/wsHelpers/sync-mongo.go @@ -3,6 +3,12 @@ package wsHelpers import ( "fmt" "os" + "strings" + + "github.com/mongodb/mongo-tools/common/log" + "github.com/mongodb/mongo-tools/common/options" + "github.com/pixlise/core/v4/core/logger" + "github.com/pixlise/core/v4/core/mongoDBConnection" ) func ResetLocalMongoBackupDir() error { @@ -23,3 +29,57 @@ func ClearLocalMongoArchive() error { return nil } + +func makeMongoToolOptions(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbNamespace string) (*options.ToolOptions, error) { + var toolOptions *options.ToolOptions + + log.SetVerbosity(nil /*toolOptions.Verbosity*/) + lw := LogWriter{logger: logger} + log.SetWriter(lw) + + ssl := options.SSL{} + + isLocal := strings.Contains(mongoDetails.Host, "localhost") && len(mongoDetails.User) <= 0 && len(mongoDetails.Password) <= 0 + + if !isLocal { + ssl = options.SSL{ + UseSSL: true, + SSLCAFile: "./global-bundle.pem", + SSLPEMKeyFile: "./global-bundle.pem", + } + } + + auth := options.Auth{ + Username: mongoDetails.User, + Password: mongoDetails.Password, + } + + connection := &options.Connection{ + Host: mongoDetails.Host, + } + + // Trim excess + protocolPrefix := "mongodb://" + connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) + + connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "") + + uri, err := options.NewURI(connectionURI) + if err != nil { + return nil, err + } + + retryWrites := false + + toolOptions = &options.ToolOptions{ + RetryWrites: &retryWrites, + SSL: &ssl, + Connection: connection, + Auth: &auth, + Verbosity: &options.Verbosity{}, + URI: uri, + Namespace: &options.Namespace{DB: dbNamespace}, + } + + return toolOptions, nil +}