Add S3 to S3 copy functionality (#335)

Makefile

@@ -36,6 +36,11 @@ fixture:
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket/foo/
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket/bar/baz/
+	aws s3 --endpoint-url http://localhost:4572 mb s3://s3-source
+	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source
+	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/foo/
+	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/bar/baz/
+	aws s3 --endpoint-url http://localhost:4572 mb s3://s3-destination
 	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload
 	aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-upload/dest_only_file
 	aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload-file

README.md

@@ -29,11 +29,12 @@ func main() {
 
   // Sync from local to s3
   syncManager.Sync("local/path/to/dir", "s3://yourbucket/path/to/dir")
+
+  // Sync from s3 to s3
+  syncManager.Sync("s3://yourbucket/path/to/dir", "s3://anotherbucket/path/to/dir")
 }
 ```
 
-- Note: Sync from s3 to s3 is not implemented yet.
-
 ## Sets the custom logger
 
 You can set your custom logger.

s3sync.go

@@ -155,8 +155,32 @@ func isS3URL(url *url.URL) bool {
 	return url.Scheme == "s3"
 }
 
-func (m *Manager) syncS3ToS3(ctx context.Context, chJob chan func(), sourcePath, destPath *s3Path) error {
-	return errors.New("S3 to S3 sync feature is not implemented")
+func (m *Manager) syncS3ToS3(ctx context.Context, chJob chan func(), sourcePath *s3Path, destPath *s3Path) error {
+	wg := &sync.WaitGroup{}
+	errs := &multiErr{}
+	for source := range filterFilesForSync(
+		m.listS3Files(ctx, sourcePath), m.listS3Files(ctx, destPath), m.del,
+	) {
+		wg.Add(1)
+		source := source
+		chJob <- func() {
+			defer wg.Done()
+			if source.err != nil {
+				errs.Append(source.err)
+				return
+			}
+			switch source.op {
+			case opUpdate:
+				if err := m.copyS3ToS3(ctx, source.fileInfo, sourcePath, destPath); err != nil {
+					errs.Append(err)
+				}
+			}
+		}
+	}
+	wg.Wait()
+
+	return errs.ErrOrNil()
+
 }
 
 func (m *Manager) syncLocalToS3(ctx context.Context, chJob chan func(), sourcePath string, destPath *s3Path) error {
@@ -222,6 +246,27 @@ func (m *Manager) syncS3ToLocal(ctx context.Context, chJob chan func(), sourcePa
 	return errs.ErrOrNil()
 }
 
+func (m *Manager) copyS3ToS3(ctx context.Context, file *fileInfo, sourcePath *s3Path, destPath *s3Path) error {
+	println("Copying", file.name, "to", destPath.String())
+	if m.dryrun {
+		return nil
+	}
+
+	_, err := m.s3.CopyObject(&s3.CopyObjectInput{
+		Bucket:     aws.String(destPath.bucket),
+		CopySource: aws.String(sourcePath.bucket + "/" + file.name),
+		Key:        aws.String(file.name),
+		ACL:        m.acl,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	m.updateFileTransferStatistics(file.size)
+	return nil
+}
+
 func (m *Manager) download(file *fileInfo, sourcePath *s3Path, destPath string) error {
 	var targetFilename string
 	if !strings.HasSuffix(destPath, "/") && file.singleFile {

s3sync_test.go

@@ -34,9 +34,15 @@ func TestS3syncNotImplemented(t *testing.T) {
 	if err := m.Sync("foo", "bar"); err == nil {
 		t.Fatal("local to local sync is not supported")
 	}
+}
+
+func TestS3ToS3(t *testing.T) {
+	m := New(getSession())
 
-	if err := m.Sync("s3://foo", "s3://bar"); err == nil {
-		t.Fatal("s3 to s3 sync is not implemented yet")
+	err := m.Sync("s3://s3-source", "s3://s3-destination")
+
+	if err != nil {
+		t.Fatal(err.Error())
 	}
 }
 
@@ -134,6 +140,27 @@ func TestS3sync(t *testing.T) {
 		fileHasSize(t, filepath.Join(temp, "test.md"), dummyFileSize)
 	})
 
+	t.Run("S3ToS3Copy", func(t *testing.T) {
+		if err := New(getSession()).Sync("s3://s3-source", "s3://s3-destination"); err != nil {
+			t.Fatal("Sync should be successful", err)
+		}
+
+		objs := listObjectsSorted(t, "s3-destination")
+		if n := len(objs); n != 3 {
+			t.Fatalf("Number of the files should be 3 (result: %v)", objs)
+		}
+		for _, obj := range objs {
+			if obj.size != dummyFileSize {
+				t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size)
+			}
+		}
+		if objs[0].path != "README.md" ||
+			objs[1].path != "bar/baz/README.md" ||
+			objs[2].path != "foo/README.md" {
+			t.Error("Unexpected keys", objs)
+		}
+	})
+
 	t.Run("Upload", func(t *testing.T) {
 		temp, err := ioutil.TempDir("", "s3synctest")
 		defer os.RemoveAll(temp)