diff --git a/Makefile b/Makefile index d147af3..dee090b 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,11 @@ fixture: aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket/foo/ aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket/bar/baz/ + aws s3 --endpoint-url http://localhost:4572 mb s3://s3-source + aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source + aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/foo/ + aws s3 --endpoint-url http://localhost:4572 cp README.md s3://s3-source/bar/baz/ + aws s3 --endpoint-url http://localhost:4572 mb s3://s3-destination aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload aws s3 --endpoint-url http://localhost:4572 cp README.md s3://example-bucket-upload/dest_only_file aws s3 --endpoint-url http://localhost:4572 mb s3://example-bucket-upload-file diff --git a/README.md b/README.md index 00b7aed..4d655ae 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,12 @@ func main() { // Sync from local to s3 syncManager.Sync("local/path/to/dir", "s3://yourbucket/path/to/dir") + + // Sync from s3 to s3 + syncManager.Sync("s3://yourbucket/path/to/dir", "s3://anotherbucket/path/to/dir") } ``` -- Note: Sync from s3 to s3 is not implemented yet. - ## Sets the custom logger You can set your custom logger. diff --git a/s3sync.go b/s3sync.go index e4f7e31..7c8dbf3 100644 --- a/s3sync.go +++ b/s3sync.go @@ -155,8 +155,32 @@ func isS3URL(url *url.URL) bool { return url.Scheme == "s3" } -func (m *Manager) syncS3ToS3(ctx context.Context, chJob chan func(), sourcePath, destPath *s3Path) error { - return errors.New("S3 to S3 sync feature is not implemented") +func (m *Manager) syncS3ToS3(ctx context.Context, chJob chan func(), sourcePath *s3Path, destPath *s3Path) error { + wg := &sync.WaitGroup{} + errs := &multiErr{} + for source := range filterFilesForSync( + m.listS3Files(ctx, sourcePath), m.listS3Files(ctx, destPath), m.del, + ) { + wg.Add(1) + source := source + chJob <- func() { + defer wg.Done() + if source.err != nil { + errs.Append(source.err) + return + } + switch source.op { + case opUpdate: + if err := m.copyS3ToS3(ctx, source.fileInfo, sourcePath, destPath); err != nil { + errs.Append(err) + } + } + } + } + wg.Wait() + + return errs.ErrOrNil() + } func (m *Manager) syncLocalToS3(ctx context.Context, chJob chan func(), sourcePath string, destPath *s3Path) error { @@ -222,6 +246,27 @@ func (m *Manager) syncS3ToLocal(ctx context.Context, chJob chan func(), sourcePa return errs.ErrOrNil() } +func (m *Manager) copyS3ToS3(ctx context.Context, file *fileInfo, sourcePath *s3Path, destPath *s3Path) error { + println("Copying", file.name, "to", destPath.String()) + if m.dryrun { + return nil + } + + _, err := m.s3.CopyObject(&s3.CopyObjectInput{ + Bucket: aws.String(destPath.bucket), + CopySource: aws.String(sourcePath.bucket + "/" + file.name), + Key: aws.String(file.name), + ACL: m.acl, + }) + + if err != nil { + return err + } + + m.updateFileTransferStatistics(file.size) + return nil +} + func (m *Manager) download(file *fileInfo, sourcePath *s3Path, destPath string) error { var targetFilename string if !strings.HasSuffix(destPath, "/") && file.singleFile { diff --git a/s3sync_test.go b/s3sync_test.go index 74a814d..96d3d54 100644 --- a/s3sync_test.go +++ b/s3sync_test.go @@ -34,9 +34,15 @@ func TestS3syncNotImplemented(t *testing.T) { if err := m.Sync("foo", "bar"); err == nil { t.Fatal("local to local sync is not supported") } +} + +func TestS3ToS3(t *testing.T) { + m := New(getSession()) - if err := m.Sync("s3://foo", "s3://bar"); err == nil { - t.Fatal("s3 to s3 sync is not implemented yet") + err := m.Sync("s3://s3-source", "s3://s3-destination") + + if err != nil { + t.Fatal(err.Error()) } } @@ -134,6 +140,27 @@ func TestS3sync(t *testing.T) { fileHasSize(t, filepath.Join(temp, "test.md"), dummyFileSize) }) + t.Run("S3ToS3Copy", func(t *testing.T) { + if err := New(getSession()).Sync("s3://s3-source", "s3://s3-destination"); err != nil { + t.Fatal("Sync should be successful", err) + } + + objs := listObjectsSorted(t, "s3-destination") + if n := len(objs); n != 3 { + t.Fatalf("Number of the files should be 3 (result: %v)", objs) + } + for _, obj := range objs { + if obj.size != dummyFileSize { + t.Errorf("Object size should be %d, actual %d", dummyFileSize, obj.size) + } + } + if objs[0].path != "README.md" || + objs[1].path != "bar/baz/README.md" || + objs[2].path != "foo/README.md" { + t.Error("Unexpected keys", objs) + } + }) + t.Run("Upload", func(t *testing.T) { temp, err := ioutil.TempDir("", "s3synctest") defer os.RemoveAll(temp)