[s3] Add CopyFrom CopyTo ListAllObjects (#62)

evalphobia · May 5, 2020 · d088e97 · d088e97
1 parent f569fe7
commit d088e97
Show file tree

Hide file tree

Showing 6 changed files with 676 additions and 100 deletions.
diff --git a/README.md b/README.md
@@ -28,105 +28,97 @@ aws-sdk-go-wrapper
 [23]: https://img.shields.io/badge/License-Apache%202.0-blue.svg
 [24]: LICENSE.md
 
-(checked SDK version [aws-sdk-go](https://github.com/awslabs/aws-sdk-go/) :: [v1.29.14](https://github.com/awslabs/aws-sdk-go/tree/v1.29.14)
 
 Simple wrapper for aws-sdk-go
-At this time, it suports services below,
-
-- [`CloudTrail`](/cloudtrail)
-    - LookupEvents
-- [`CloudWatch`](/cloudwatch)
-    - GetMetricStatistics
-- [`CostExplorer`](/costexplorer)
-    - GetCostAndUsage
-- [`DynamoDB`](/dynamodb)
-    - BatchWriteItem
-    - CreateTable
-    - DeleteItem
-    - DeleteTable
-    - DescribeTable
-    - GetItem
-    - ListTables
-    - PutItem
-    - Query
-    - UpdateTable
-    - Scan
-- [`IAM`](/iam)
-    - GetGroup
-    - GetGroupPolicy
-    - GetPolicyVersion
-    - GetRolePolicy
-    - GetUserPolicy
-    - ListEntitiesForPolicy
-    - ListGroups
-    - ListGroupPolicies
-    - ListPolicies
-    - ListUsers
-    - ListUserPolicies
-    - ListRoles
-    - ListRolePolicies
-- [`Kinesis`](/kinesis)
-    - CreateStream
-    - DeleteStream
-    - DescribeStream
-    - GetRecords
-    - GetShardIterator
-    - PutRecord
-- [`KMS`](/kms)
-    - CreateAlias
-    - CreateKey
-    - Decrypt
-    - DescribeKey
-    - Encrypt
-    - ReEncrypt
-    - ScheduleKeyDeletion
-- [`Pinpoint`](/pinpoint)
-    - SendEmail
-- [`Rekognition`](/rekognition)
-    - CompareFaces
-    - CreateCollection
-    - DeleteCollection
-    - DeleteFaces
-    - DetectFaces
-    - DetectLabels
-    - DetectModerationLabels
-    - GetCelebrityInfo
-    - IndexFaces
-    - ListCollections
-    - ListFaces
-    - RecognizeCelebrities
-    - SearchFaces
-    - SearchFacesByImage
-- [`S3`](/s3)
-    - CreateBucket
-    - DeleteBucket
-    - DeleteObject
-    - GetObject
-    - HeadObject
-    - PutObject
-- [`SNS`](/sns)
-    - CreatePlatformEndpoint
-    - CreateTopic
-    - DeleteTopic
-    - GetEndpointAttributes
-    - GetPlatformApplicationAttributes
-    - Publish
-    - SetEndpointAttributes
-    - Subscribe
-- [`SQS`](/sqs)
-    - ChangeMessageVisibility
-    - CreateQueue
-    - DeleteMessage
-    - DeleteMessageBatch
-    - DeleteQueue
-    - GetQueueAttributes
-    - GetQueueUrl
-    - ListQueues
-    - PurgeQueue
-    - ReceiveMessage
-    - SendMessageBatch
-- [`X-Ray`](/xray)
-    - PutTraceSegments
+At this time, this library suports these AWS services below,
+
+| Service | API |
+| :--- | :-- |
+| [`CloudTrail`](/cloudtrail) | LookupEvents |
+| [`CloudWatch`](/cloudwatch) | GetMetricStatistics |
+| [`CostExplorer`](/costexplorer) | GetCostAndUsage |
+| [`DynamoDB`](/dynamodb) | BatchWriteItem |
+|  | CreateTable |
+|  | DeleteItem |
+|  | DeleteTable |
+|  | DescribeTable |
+|  | GetItem |
+|  | ListTables |
+|  | PutItem |
+|  | Query |
+|  | UpdateTable |
+|  | Scan |
+| [`IAM`](/iam) | GetGroup |
+|  | GetGroupPolicy |
+|  | GetPolicyVersion |
+|  | GetRolePolicy |
+|  | GetUserPolicy |
+|  | ListEntitiesForPolicy |
+|  | ListGroups |
+|  | ListGroupPolicies |
+|  | ListPolicies |
+|  | ListUsers |
+|  | ListUserPolicies |
+|  | ListRoles |
+|  | ListRolePolicies |
+| [`Kinesis`](/kinesis) | CreateStream |
+|  | DeleteStream |
+|  | DescribeStream |
+|  | GetRecords |
+|  | GetShardIterator |
+|  | PutRecord |
+| [`KMS`](/kms) | CreateAlias |
+|  | CreateKey |
+|  | Decrypt |
+|  | DescribeKey |
+|  | Encrypt |
+|  | ReEncrypt |
+|  | ReEncrypt |
+|  | ScheduleKeyDeletion |
+| [`Pinpoint`](/pinpoint) | SendEmail |
+| [`Rekognition`](/rekognition) | CompareFaces |
+|  | CreateCollection |
+|  | DeleteCollection |
+|  | DeleteFaces |
+|  | DetectFaces |
+|  | DetectLabels |
+|  | DetectModerationLabels |
+|  | GetCelebrityInfo |
+|  | IndexFaces |
+|  | ListCollections |
+|  | ListFaces |
+|  | RecognizeCelebrities |
+|  | SearchFaces |
+|  | SearchFacesByImage |
+| [`S3`](/s3) | CreateBucket |
+|  | CopyObject |
+|  | DeleteBucket |
+|  | DeleteObject |
+|  | GetObject |
+|  | HeadObject |
+|  | ListObjectsV2 |
+|  | PutObject |
+| [`SNS`](/sns) | CreatePlatformEndpoint |
+|  | CreateTopic |
+|  | DeleteTopic |
+|  | GetEndpointAttributes |
+|  | GetPlatformApplicationAttributes |
+|  | Publish |
+|  | SetEndpointAttributes |
+|  | Subscribe |
+| [`SQS`](/sqs) | ChangeMessageVisibility |
+|  | CreateQueue |
+|  | DeleteMessage |
+|  | DeleteMessageBatch |
+|  | DeleteQueue |
+|  | GetQueueAttributes |
+|  | GetQueueUrl |
+|  | ListQueues |
+|  | PurgeQueue |
+|  | ReceiveMessage |
+|  | SendMessageBatch |
+| [`X-Ray`](/xray) | PutTraceSegments |
+
 
 # Quick Usage
 

diff --git a/s3/bucket.go b/s3/bucket.go
@@ -180,6 +180,75 @@ func (b *Bucket) IsExists(path string) bool {
 	return err == nil
 }
 
+// ListAllObjects fetches a list of all of the objects in the bucket and prefix.
+func (b *Bucket) ListAllObjects(prefix string) ([]Object, error) {
+	var contents []Object
+	nextToken := ""
+	for {
+		resp, err := b.ListObjectsV2(ListObjectsRequest{
+			Prefix:            prefix,
+			ContinuationToken: nextToken,
+		})
+		if err != nil {
+			return contents, err
+		}
+		contents = append(contents, resp.Contents...)
+		if !resp.IsTruncated {
+			return contents, nil
+		}
+		nextToken = resp.NextContinuationToken
+	}
+}
+
+// ListObjectsV2 executes ListObjectsV2 operation.
+func (b *Bucket) ListObjectsV2(opt ...ListObjectsRequest) (ListObjectsResponse, error) {
+	var o ListObjectsRequest
+	if len(opt) != 0 {
+		o = opt[0]
+	}
+	o.Bucket = b.nameWithPrefix
+	resp, err := b.service.client.ListObjectsV2(o.ToInput())
+	return NewListObjectsResponse(resp), err
+}
+
+// CopyTo copies an object to destination bucket and path.
+func (b *Bucket) CopyTo(srcPath, destBucket, destPath string, opt ...CopyObjectRequest) (CopyObjectResponse, error) {
+	var o CopyObjectRequest
+	if len(opt) != 0 {
+		o = opt[0]
+	}
+
+	bucketName := destBucket
+	if o.UseSamePrefix {
+		bucketName = b.service.prefix + bucketName
+	}
+
+	o.SrcBucket = b.nameWithPrefix
+	o.SrcPath = srcPath
+	o.DestBucket = bucketName
+	o.DestPath = destPath
+	return b.service.CopyObject(o)
+}
+
+// CopyFrom copies an object from source buckwt and path.
+func (b *Bucket) CopyFrom(srcBucket, srcPath, destPath string, opt ...CopyObjectRequest) (CopyObjectResponse, error) {
+	var o CopyObjectRequest
+	if len(opt) != 0 {
+		o = opt[0]
+	}
+
+	bucketName := srcBucket
+	if o.UseSamePrefix {
+		bucketName = b.service.prefix + bucketName
+	}
+
+	o.SrcBucket = bucketName
+	o.SrcPath = srcPath
+	o.DestBucket = b.nameWithPrefix
+	o.DestPath = destPath
+	return b.service.CopyObject(o)
+}
+
 // DeleteObject deletes the object of target path.
 func (b *Bucket) DeleteObject(path string) error {
 	_, err := b.service.client.DeleteObject(&SDK.DeleteObjectInput{

diff --git a/s3/bucket_test.go b/s3/bucket_test.go
@@ -1,15 +1,17 @@
 package s3
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 const (
-	testS3Path        = "test_path"
-	testPutBucketName = "test-put-bucket"
-	testBaseURL       = "http://localhost:4567/" + testPutBucketName
+	testS3Path         = "test_path"
+	testPutBucketName  = "test-put-bucket"
+	testCopyBucketName = "test-copy-bucket"
+	testBaseURL        = "http://localhost:4567/" + testPutBucketName
 )
 
 func testPutObject(t *testing.T) {
@@ -207,6 +209,98 @@ func TestGetSecretURLWithExpire(t *testing.T) {
 	assert.Contains(data, "X-Amz-Expires=10")
 }
 
+func TestListAllObjects(t *testing.T) {
+	a := assert.New(t)
+	createBucket(testPutBucketName)
+	testPutObject(t)
+
+	const (
+		targetPrefix = "list-path/"
+		objectCount  = 5
+	)
+
+	f := openFile(t)
+	defer f.Close() // nolint:gosec
+	obj := NewPutObject(f)
+
+	svc := getTestClient(t)
+	b, err := svc.GetBucket(testPutBucketName)
+	a.NoError(err)
+
+	for i := 0; i < objectCount; i++ {
+		b.DeleteObject(fmt.Sprintf("%s/%d", targetPrefix, i))
+	}
+	list, err := b.ListAllObjects(targetPrefix)
+	a.NoError(err)
+	a.Len(list, 0)
+
+	for i := 0; i < objectCount; i++ {
+		b.AddObject(obj, fmt.Sprintf("%s/%d", targetPrefix, i))
+	}
+	err = b.PutAll()
+	a.NoError(err)
+
+	list, err = b.ListAllObjects(targetPrefix)
+	a.NoError(err)
+	a.Len(list, objectCount)
+}
+
+func TestCopyFrom(t *testing.T) {
+	a := assert.New(t)
+	createBucket(testPutBucketName)
+	createBucket(testCopyBucketName)
+	testPutObject(t)
+
+	f := openFile(t)
+	fs, _ := f.Stat()
+	defer f.Close() // nolint:gosec
+
+	svc := getTestClient(t)
+	b1, err := svc.GetBucket(testPutBucketName)
+	a.NoError(err)
+
+	b2, err := svc.GetBucket(testCopyBucketName)
+	a.NoError(err)
+
+	// copy data
+	resp, err := b2.CopyFrom(testPutBucketName, testS3Path, testS3Path)
+	a.NoError(err)
+	a.NotEmpty(resp.ETag)
+
+	// check copied content
+	data, err := b1.GetObjectByte(testS3Path)
+	a.NoError(err)
+	a.Equal(int(fs.Size()), len(data))
+}
+
+func TestCopyTo(t *testing.T) {
+	a := assert.New(t)
+	createBucket(testPutBucketName)
+	createBucket(testCopyBucketName)
+	testPutObject(t)
+
+	f := openFile(t)
+	fs, _ := f.Stat()
+	defer f.Close() // nolint:gosec
+
+	svc := getTestClient(t)
+	b1, err := svc.GetBucket(testPutBucketName)
+	a.NoError(err)
+
+	b2, err := svc.GetBucket(testCopyBucketName)
+	a.NoError(err)
+
+	// copy data
+	resp, err := b1.CopyTo(testS3Path, testCopyBucketName, testS3Path)
+	a.NoError(err)
+	a.NotEmpty(resp.ETag)
+
+	// check copied content
+	data, err := b2.GetObjectByte(testS3Path)
+	a.NoError(err)
+	a.Equal(int(fs.Size()), len(data))
+}
+
 func TestDeleteObject(t *testing.T) {
 	assert := assert.New(t)
 	createBucket(testPutBucketName)