Mongo dump/restore cleanup so they share code, we had 2 copies of the…

… setup code
pixlise · Sep 17, 2024 · 3f3fd6b · 3f3fd6b
1 parent d057610
commit 3f3fd6b
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 107 deletions.
diff --git a/api/ws/handlers/system.go b/api/ws/handlers/system.go
@@ -40,7 +40,12 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) (
 	hctx.Svcs.Log.Infof("PIXLISE Backup Requested, will be written to bucket: %v", hctx.Svcs.Config.DataBackupBucket)
 
 	// Run MongoDump, save to a local archive file
-	dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName))
+	dump, err := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName))
+
+	if err != nil {
+		hctx.Svcs.Log.Errorf("Failed to create dump instance: %v", err)
+		return nil, err
+	}
 
 	err = dump.Init()
 	if err != nil {

diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go
@@ -4,10 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"strings"
 
-	"github.com/mongodb/mongo-tools/common/log"
-	"github.com/mongodb/mongo-tools/common/options"
 	"github.com/mongodb/mongo-tools/mongodump"
 	"github.com/pixlise/core/v4/api/services"
 	"github.com/pixlise/core/v4/core/fileaccess"
@@ -18,56 +15,13 @@ import (
 var dataBackupLocalPath = "./backup"
 var dataBackupS3Path = "DB"
 
-func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) *mongodump.MongoDump {
-	var toolOptions *options.ToolOptions
-
-	ssl := options.SSL{}
-
-	isLocal := strings.Contains(mongoDetails.Host, "localhost") && len(mongoDetails.User) <= 0 && len(mongoDetails.Password) <= 0
-
-	if !isLocal {
-		ssl = options.SSL{
-			UseSSL:        true,
-			SSLCAFile:     "./global-bundle.pem",
-			SSLPEMKeyFile: "./global-bundle.pem",
-		}
-	}
-
-	auth := options.Auth{
-		Username: mongoDetails.User,
-		Password: mongoDetails.Password,
-	}
-
-	connection := &options.Connection{
-		Host: mongoDetails.Host,
-	}
-
-	// Trim excess
-	protocolPrefix := "mongodb://"
-	connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)
-
-	connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "")
-
-	logger.Infof("MongoDump connecting to: %v, user %v, db-to-dump: %v...", connectionURI, auth.Username, dbName)
-
-	uri, err := options.NewURI(connectionURI)
+func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) (*mongodump.MongoDump, error) {
+	toolOptions, err := makeMongoToolOptions(mongoDetails, logger, dbName)
 	if err != nil {
-		logger.Errorf("%v", err)
-		return nil
+		return nil, err
 	}
 
-	retryWrites := false
-
-	toolOptions = &options.ToolOptions{
-		RetryWrites: &retryWrites,
-		SSL:         &ssl,
-		Connection:  connection,
-		Auth:        &auth,
-		Verbosity:   &options.Verbosity{},
-		URI:         uri,
-	}
-
-	toolOptions.Namespace = &options.Namespace{DB: dbName}
+	logger.Infof("MongoDump connecting to: %v, user %v, db-to-dump: %v...", toolOptions.URI.ConnectionString, toolOptions.Auth.Username, dbName)
 
 	outputOptions := &mongodump.OutputOptions{
 		Out:  dataBackupLocalPath,
@@ -78,15 +32,11 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails
 	}
 	inputOptions := &mongodump.InputOptions{}
 
-	log.SetVerbosity(nil /*toolOptions.Verbosity*/)
-	lw := LogWriter{logger: logger}
-	log.SetWriter(lw)
-
 	return &mongodump.MongoDump{
 		ToolOptions:   toolOptions,
 		InputOptions:  inputOptions,
 		OutputOptions: outputOptions,
-	}
+	}, nil
 }
 
 func UploadArchive(svcs *services.APIServices) error {

diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go
@@ -5,8 +5,6 @@ import (
 	"path"
 	"strings"
 
-	"github.com/mongodb/mongo-tools/common/log"
-	"github.com/mongodb/mongo-tools/common/options"
 	"github.com/mongodb/mongo-tools/mongorestore"
 	"github.com/pixlise/core/v4/api/services"
 	"github.com/pixlise/core/v4/core/fileaccess"
@@ -15,55 +13,12 @@ import (
 )
 
 func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) {
-	var toolOptions *options.ToolOptions
-
-	ssl := options.SSL{}
-
-	isLocal := strings.Contains(mongoDetails.Host, "localhost") && len(mongoDetails.User) <= 0 && len(mongoDetails.Password) <= 0
-
-	if !isLocal {
-		ssl = options.SSL{
-			UseSSL:        true,
-			SSLCAFile:     "./global-bundle.pem",
-			SSLPEMKeyFile: "./global-bundle.pem",
-		}
-	}
-
-	auth := options.Auth{
-		Username: mongoDetails.User,
-		Password: mongoDetails.Password,
-	}
-
-	connection := &options.Connection{
-		Host: mongoDetails.Host,
-	}
-
-	// Trim excess
-	protocolPrefix := "mongodb://"
-	connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)
-
-	connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "")
-
-	logger.Infof("MongoRestore connecting to: %v, user %v, restore-to-db: %v, restore-from-db: %v...", connectionURI, auth.Username, restoreToDBName, restoreFromDBName)
-
-	uri, err := options.NewURI(connectionURI)
+	toolOptions, err := makeMongoToolOptions(mongoDetails, logger, restoreToDBName)
 	if err != nil {
-		logger.Errorf("%v", err)
 		return nil, err
 	}
 
-	retryWrites := false
-
-	toolOptions = &options.ToolOptions{
-		RetryWrites: &retryWrites,
-		SSL:         &ssl,
-		Connection:  connection,
-		Auth:        &auth,
-		Verbosity:   &options.Verbosity{},
-		URI:         uri,
-	}
-
-	toolOptions.Namespace = &options.Namespace{DB: restoreToDBName}
+	logger.Infof("MongoRestore connecting to: %v, user %v, restore-to-db: %v, restore-from-db: %v...", toolOptions.URI.ConnectionString, toolOptions.Auth.Username, restoreToDBName, restoreFromDBName)
 
 	outputOptions := &mongorestore.OutputOptions{
 		NumParallelCollections: 1,
@@ -85,10 +40,6 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta
 		NSTo:      []string{restoreToDBName},
 	}
 
-	log.SetVerbosity(nil /*toolOptions.Verbosity*/)
-	lw := LogWriter{logger: logger}
-	log.SetWriter(lw)
-
 	return mongorestore.New(mongorestore.Options{
 		ToolOptions:     toolOptions,
 		InputOptions:    inputOptions,

diff --git a/api/ws/wsHelpers/sync-mongo.go b/api/ws/wsHelpers/sync-mongo.go
@@ -3,6 +3,12 @@ package wsHelpers
 import (
 	"fmt"
 	"os"
+	"strings"
+
+	"github.com/mongodb/mongo-tools/common/log"
+	"github.com/mongodb/mongo-tools/common/options"
+	"github.com/pixlise/core/v4/core/logger"
+	"github.com/pixlise/core/v4/core/mongoDBConnection"
 )
 
 func ResetLocalMongoBackupDir() error {
@@ -23,3 +29,57 @@ func ClearLocalMongoArchive() error {
 
 	return nil
 }
+
+func makeMongoToolOptions(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbNamespace string) (*options.ToolOptions, error) {
+	var toolOptions *options.ToolOptions
+
+	log.SetVerbosity(nil /*toolOptions.Verbosity*/)
+	lw := LogWriter{logger: logger}
+	log.SetWriter(lw)
+
+	ssl := options.SSL{}
+
+	isLocal := strings.Contains(mongoDetails.Host, "localhost") && len(mongoDetails.User) <= 0 && len(mongoDetails.Password) <= 0
+
+	if !isLocal {
+		ssl = options.SSL{
+			UseSSL:        true,
+			SSLCAFile:     "./global-bundle.pem",
+			SSLPEMKeyFile: "./global-bundle.pem",
+		}
+	}
+
+	auth := options.Auth{
+		Username: mongoDetails.User,
+		Password: mongoDetails.Password,
+	}
+
+	connection := &options.Connection{
+		Host: mongoDetails.Host,
+	}
+
+	// Trim excess
+	protocolPrefix := "mongodb://"
+	connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)
+
+	connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "")
+
+	uri, err := options.NewURI(connectionURI)
+	if err != nil {
+		return nil, err
+	}
+
+	retryWrites := false
+
+	toolOptions = &options.ToolOptions{
+		RetryWrites: &retryWrites,
+		SSL:         &ssl,
+		Connection:  connection,
+		Auth:        &auth,
+		Verbosity:   &options.Verbosity{},
+		URI:         uri,
+		Namespace:   &options.Namespace{DB: dbNamespace},
+	}
+
+	return toolOptions, nil
+}