backup: use multipartUpload to transfer large backup files (#5564)

Co-authored-by: Davies Liu <[email protected]>

pkg/meta/sql.go

@@ -290,8 +290,8 @@ func retriveUrlConnsOptions(murl string) (string, int, int, int, int) {
 
 	var vOpenConns int = 0
 	var vIdleConns int = runtime.GOMAXPROCS(-1) * 2
-	var vIdleTime  int = 300
-	var vLifeTime  int = 0
+	var vIdleTime int = 300
+	var vLifeTime int = 0
 
 	if optIndex != -1 {
 		baseurl := murl[:optIndex]
@@ -307,11 +307,11 @@ func retriveUrlConnsOptions(murl string) (string, int, int, int, int) {
 			}
 			if vals.Has("max_idle_time") {
 				vIdleTime, _ = strconv.Atoi(vals.Get("max_idle_time"))
-				vals.Del("max_idle_time");
+				vals.Del("max_idle_time")
 			}
 			if vals.Has("max_life_time") {
 				vLifeTime, _ = strconv.Atoi(vals.Get("max_life_time"))
-				vals.Del("max_life_time");
+				vals.Del("max_life_time")
 			}
 			optsurl = vals.Encode()
 		}

pkg/sync/sync.go

@@ -727,7 +727,14 @@ func doCopyMultiple(src, dst object.ObjectStorage, key string, size int64, uploa
 	return chksum, nil
 }
 
-func copyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error) {
+func InitForCopyData() {
+	concurrent = make(chan int, 10)
+	progress := utils.NewProgress(true)
+	copied = progress.AddCountSpinner("Copied objects")
+	copiedBytes = progress.AddByteSpinner("Copied bytes")
+}
+
+func CopyData(src, dst object.ObjectStorage, key string, size int64, calChksum bool) (uint32, error) {
 	start := time.Now()
 	var err error
 	var srcChksum uint32
@@ -825,7 +832,7 @@ func worker(tasks <-chan object.Object, src, dst object.ObjectStorage, config *C
 					logger.Errorf("copy link failed: %s", err)
 				}
 			} else {
-				srcChksum, err = copyData(src, dst, key, obj.Size(), config.CheckAll || config.CheckNew)
+				srcChksum, err = CopyData(src, dst, key, obj.Size(), config.CheckAll || config.CheckNew)
 			}
 
 			if err == nil && (config.CheckAll || config.CheckNew) {

pkg/vfs/backup.go

@@ -18,9 +18,13 @@ package vfs
 
 import (
 	"compress/gzip"
+	"errors"
 	"io"
 	"os"
+	"path/filepath"
 	"sort"
+	"strings"
+	"syscall"
 	"time"
 
 	"github.com/juicedata/juicefs/pkg/meta"
@@ -92,7 +96,17 @@ func Backup(m meta.Meta, blob object.ObjectStorage, interval time.Duration, skip
 
 func backup(m meta.Meta, blob object.ObjectStorage, now time.Time, fast, skipTrash bool) (string, error) {
 	name := "dump-" + now.UTC().Format("2006-01-02-150405") + ".json.gz"
-	fp, err := os.CreateTemp("", "juicefs-meta-*")
+	localDir := os.TempDir()
+	if !strings.HasSuffix(localDir, "/") {
+		localDir += "/"
+	}
+	fp, err := os.Create(filepath.Join(localDir, "meta", name))
+	if errors.Is(err, syscall.ENOENT) {
+		if err = os.MkdirAll(filepath.Join(localDir, "meta"), 0755); err != nil {
+			return "", err
+		}
+		fp, err = os.Create(filepath.Join(localDir, "meta", name))
+	}
 	if err != nil {
 		return "", err
 	}
@@ -108,11 +122,19 @@ func backup(m meta.Meta, blob object.ObjectStorage, now time.Time, fast, skipTra
 	if err != nil {
 		return "", err
 	}
-	if _, err = fp.Seek(0, io.SeekStart); err != nil {
+	size, err := fp.Seek(0, io.SeekCurrent)
+	if err != nil {
 		return "", err
 	}
+
 	fpath := "meta/" + name
-	return blob.String() + fpath, blob.Put(fpath, fp)
+	disk, err := object.CreateStorage("file", localDir, "", "", "")
+	if err != nil {
+		return "", err
+	}
+	osync.InitForCopyData()
+	_, err = osync.CopyData(disk, blob, fpath, size, true)
+	return blob.String() + fpath, err
 }
 
 func cleanupBackups(blob object.ObjectStorage, now time.Time) {