add overwrite strategy

ReadMe.md

@@ -54,4 +54,8 @@ s3:
   path: ""
   disable_ssl: false
   disable_progress_bar: false
+  # Define behavior for rewrite exists files with the same size. Must set to "skip", "etag" or "always"
+  # "skip" - the fastest but can make backup inconsistently
+  # "etag" - calculate etag for local files, set this if your network is very slow
+  overwrite_strategy: "always"
 ```

config.go

@@ -1,11 +1,11 @@
 package main
 
 import (
-	"os"
 	"fmt"
 	"io/ioutil"
+	"os"
 
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 )
 
 // Config - config file format
@@ -26,6 +26,7 @@ type S3Config struct {
 	Path               string `yaml:"path"`
 	DisableSSL         bool   `yaml:"disable_ssl"`
 	DisableProgressBar bool   `yaml:"disable_progress_bar"`
+	OverwriteStrategy  string `yaml:"overwrite_strategy"`
 }
 
 // ClickHouseConfig - clickhouse settings section
@@ -51,7 +52,18 @@ func LoadConfig(configLocation string) (*Config, error) {
 	if err != nil {
 		return nil, fmt.Errorf("can't parse with: %v", err)
 	}
-	return config, nil
+	return config, validateConfig(config)
+}
+
+func validateConfig (config *Config) error {
+	switch config.S3.OverwriteStrategy {
+		case
+		"skip",
+		"etag",
+		"always": break
+		default: return fmt.Errorf("unknown s3.overwrite_strategy it can be 'skip', 'etag', 'always'")
+	}
+	return nil
 }
 
 // PrintDefaultConfig - print default config to stdout
@@ -73,6 +85,7 @@ func defaultConfig() *Config {
 			Region:     "us-east-1",
 			DisableSSL: false,
 			ACL:        "private",
+			OverwriteStrategy: "always",
 		},
 	}
 }

s3.go

@@ -16,7 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	"gopkg.in/cheggaaa/pb.v1"
+	pb "gopkg.in/cheggaaa/pb.v1"
 )
 
 const (
@@ -137,11 +137,13 @@ func (s *S3) Download(s3Path string, localPath string) error {
 		bar.Increment()
 		if existsFile, ok := localFiles[s3File.key]; ok {
 			if existsFile.size == s3File.size {
-				// TODO: calculate Etag must be disabled via config
-				if s3File.etag == GetEtag(existsFile.fullpath) {
-					// log.Printf("Skip download file '%s' already exists", s3File.key)
-					// Skip download file
+				switch s.Config.OverwriteStrategy {
+				case "skip":
 					continue
+				case "etag":
+					if s3File.etag == GetEtag(existsFile.fullpath) {
+						continue
+					}
 				}
 			}
 		}
@@ -255,11 +257,15 @@ func (s *S3) newSyncFolderIterator(localPath, dstPath string) (*SyncFolderIterat
 			if existFile, ok := existsFiles[key]; ok {
 				delete(existsFiles, key)
 				if existFile.size == info.Size() {
-					// TODO: calculate Etag must be disabled via config
-					if existFile.etag == GetEtag(filePath) {
-						// log.Printf("File '%s' already uploaded and has the same size and etag. Skip", key)
-						skipFilesCount++
+					switch s.Config.OverwriteStrategy {
+					case "skip":
 						return nil
+					case "etag":
+						if existFile.etag == GetEtag(filePath) {
+							// log.Printf("File '%s' already uploaded and has the same size and etag. Skip", key)
+							skipFilesCount++
+							return nil
+						}
 					}
 				}
 			}