diff --git a/README.md b/README.md index 918b42e..c3a74fe 100644 --- a/README.md +++ b/README.md @@ -28,105 +28,97 @@ aws-sdk-go-wrapper [23]: https://img.shields.io/badge/License-Apache%202.0-blue.svg [24]: LICENSE.md -(checked SDK version [aws-sdk-go](https://github.com/awslabs/aws-sdk-go/) :: [v1.29.14](https://github.com/awslabs/aws-sdk-go/tree/v1.29.14) Simple wrapper for aws-sdk-go -At this time, it suports services below, - -- [`CloudTrail`](/cloudtrail) - - LookupEvents -- [`CloudWatch`](/cloudwatch) - - GetMetricStatistics -- [`CostExplorer`](/costexplorer) - - GetCostAndUsage -- [`DynamoDB`](/dynamodb) - - BatchWriteItem - - CreateTable - - DeleteItem - - DeleteTable - - DescribeTable - - GetItem - - ListTables - - PutItem - - Query - - UpdateTable - - Scan -- [`IAM`](/iam) - - GetGroup - - GetGroupPolicy - - GetPolicyVersion - - GetRolePolicy - - GetUserPolicy - - ListEntitiesForPolicy - - ListGroups - - ListGroupPolicies - - ListPolicies - - ListUsers - - ListUserPolicies - - ListRoles - - ListRolePolicies -- [`Kinesis`](/kinesis) - - CreateStream - - DeleteStream - - DescribeStream - - GetRecords - - GetShardIterator - - PutRecord -- [`KMS`](/kms) - - CreateAlias - - CreateKey - - Decrypt - - DescribeKey - - Encrypt - - ReEncrypt - - ScheduleKeyDeletion -- [`Pinpoint`](/pinpoint) - - SendEmail -- [`Rekognition`](/rekognition) - - CompareFaces - - CreateCollection - - DeleteCollection - - DeleteFaces - - DetectFaces - - DetectLabels - - DetectModerationLabels - - GetCelebrityInfo - - IndexFaces - - ListCollections - - ListFaces - - RecognizeCelebrities - - SearchFaces - - SearchFacesByImage -- [`S3`](/s3) - - CreateBucket - - DeleteBucket - - DeleteObject - - GetObject - - HeadObject - - PutObject -- [`SNS`](/sns) - - CreatePlatformEndpoint - - CreateTopic - - DeleteTopic - - GetEndpointAttributes - - GetPlatformApplicationAttributes - - Publish - - SetEndpointAttributes - - Subscribe -- [`SQS`](/sqs) - - ChangeMessageVisibility - - CreateQueue - - DeleteMessage - - DeleteMessageBatch - - DeleteQueue - - GetQueueAttributes - - GetQueueUrl - - ListQueues - - PurgeQueue - - ReceiveMessage - - SendMessageBatch -- [`X-Ray`](/xray) - - PutTraceSegments +At this time, this library suports these AWS services below, + +| Service | API | +| :--- | :-- | +| [`CloudTrail`](/cloudtrail) | LookupEvents | +| [`CloudWatch`](/cloudwatch) | GetMetricStatistics | +| [`CostExplorer`](/costexplorer) | GetCostAndUsage | +| [`DynamoDB`](/dynamodb) | BatchWriteItem | +| | CreateTable | +| | DeleteItem | +| | DeleteTable | +| | DescribeTable | +| | GetItem | +| | ListTables | +| | PutItem | +| | Query | +| | UpdateTable | +| | Scan | +| [`IAM`](/iam) | GetGroup | +| | GetGroupPolicy | +| | GetPolicyVersion | +| | GetRolePolicy | +| | GetUserPolicy | +| | ListEntitiesForPolicy | +| | ListGroups | +| | ListGroupPolicies | +| | ListPolicies | +| | ListUsers | +| | ListUserPolicies | +| | ListRoles | +| | ListRolePolicies | +| [`Kinesis`](/kinesis) | CreateStream | +| | DeleteStream | +| | DescribeStream | +| | GetRecords | +| | GetShardIterator | +| | PutRecord | +| [`KMS`](/kms) | CreateAlias | +| | CreateKey | +| | Decrypt | +| | DescribeKey | +| | Encrypt | +| | ReEncrypt | +| | ReEncrypt | +| | ScheduleKeyDeletion | +| [`Pinpoint`](/pinpoint) | SendEmail | +| [`Rekognition`](/rekognition) | CompareFaces | +| | CreateCollection | +| | DeleteCollection | +| | DeleteFaces | +| | DetectFaces | +| | DetectLabels | +| | DetectModerationLabels | +| | GetCelebrityInfo | +| | IndexFaces | +| | ListCollections | +| | ListFaces | +| | RecognizeCelebrities | +| | SearchFaces | +| | SearchFacesByImage | +| [`S3`](/s3) | CreateBucket | +| | CopyObject | +| | DeleteBucket | +| | DeleteObject | +| | GetObject | +| | HeadObject | +| | ListObjectsV2 | +| | PutObject | +| [`SNS`](/sns) | CreatePlatformEndpoint | +| | CreateTopic | +| | DeleteTopic | +| | GetEndpointAttributes | +| | GetPlatformApplicationAttributes | +| | Publish | +| | SetEndpointAttributes | +| | Subscribe | +| [`SQS`](/sqs) | ChangeMessageVisibility | +| | CreateQueue | +| | DeleteMessage | +| | DeleteMessageBatch | +| | DeleteQueue | +| | GetQueueAttributes | +| | GetQueueUrl | +| | ListQueues | +| | PurgeQueue | +| | ReceiveMessage | +| | SendMessageBatch | +| [`X-Ray`](/xray) | PutTraceSegments | + # Quick Usage diff --git a/s3/bucket.go b/s3/bucket.go index 7bcb6ef..2cb0c50 100644 --- a/s3/bucket.go +++ b/s3/bucket.go @@ -180,6 +180,75 @@ func (b *Bucket) IsExists(path string) bool { return err == nil } +// ListAllObjects fetches a list of all of the objects in the bucket and prefix. +func (b *Bucket) ListAllObjects(prefix string) ([]Object, error) { + var contents []Object + nextToken := "" + for { + resp, err := b.ListObjectsV2(ListObjectsRequest{ + Prefix: prefix, + ContinuationToken: nextToken, + }) + if err != nil { + return contents, err + } + contents = append(contents, resp.Contents...) + if !resp.IsTruncated { + return contents, nil + } + nextToken = resp.NextContinuationToken + } +} + +// ListObjectsV2 executes ListObjectsV2 operation. +func (b *Bucket) ListObjectsV2(opt ...ListObjectsRequest) (ListObjectsResponse, error) { + var o ListObjectsRequest + if len(opt) != 0 { + o = opt[0] + } + o.Bucket = b.nameWithPrefix + resp, err := b.service.client.ListObjectsV2(o.ToInput()) + return NewListObjectsResponse(resp), err +} + +// CopyTo copies an object to destination bucket and path. +func (b *Bucket) CopyTo(srcPath, destBucket, destPath string, opt ...CopyObjectRequest) (CopyObjectResponse, error) { + var o CopyObjectRequest + if len(opt) != 0 { + o = opt[0] + } + + bucketName := destBucket + if o.UseSamePrefix { + bucketName = b.service.prefix + bucketName + } + + o.SrcBucket = b.nameWithPrefix + o.SrcPath = srcPath + o.DestBucket = bucketName + o.DestPath = destPath + return b.service.CopyObject(o) +} + +// CopyFrom copies an object from source buckwt and path. +func (b *Bucket) CopyFrom(srcBucket, srcPath, destPath string, opt ...CopyObjectRequest) (CopyObjectResponse, error) { + var o CopyObjectRequest + if len(opt) != 0 { + o = opt[0] + } + + bucketName := srcBucket + if o.UseSamePrefix { + bucketName = b.service.prefix + bucketName + } + + o.SrcBucket = bucketName + o.SrcPath = srcPath + o.DestBucket = b.nameWithPrefix + o.DestPath = destPath + return b.service.CopyObject(o) +} + // DeleteObject deletes the object of target path. func (b *Bucket) DeleteObject(path string) error { _, err := b.service.client.DeleteObject(&SDK.DeleteObjectInput{ diff --git a/s3/bucket_test.go b/s3/bucket_test.go index 133ed3e..9124101 100644 --- a/s3/bucket_test.go +++ b/s3/bucket_test.go @@ -1,15 +1,17 @@ package s3 import ( + "fmt" "testing" "github.com/stretchr/testify/assert" ) const ( - testS3Path = "test_path" - testPutBucketName = "test-put-bucket" - testBaseURL = "http://localhost:4567/" + testPutBucketName + testS3Path = "test_path" + testPutBucketName = "test-put-bucket" + testCopyBucketName = "test-copy-bucket" + testBaseURL = "http://localhost:4567/" + testPutBucketName ) func testPutObject(t *testing.T) { @@ -207,6 +209,98 @@ func TestGetSecretURLWithExpire(t *testing.T) { assert.Contains(data, "X-Amz-Expires=10") } +func TestListAllObjects(t *testing.T) { + a := assert.New(t) + createBucket(testPutBucketName) + testPutObject(t) + + const ( + targetPrefix = "list-path/" + objectCount = 5 + ) + + f := openFile(t) + defer f.Close() // nolint:gosec + obj := NewPutObject(f) + + svc := getTestClient(t) + b, err := svc.GetBucket(testPutBucketName) + a.NoError(err) + + for i := 0; i < objectCount; i++ { + b.DeleteObject(fmt.Sprintf("%s/%d", targetPrefix, i)) + } + list, err := b.ListAllObjects(targetPrefix) + a.NoError(err) + a.Len(list, 0) + + for i := 0; i < objectCount; i++ { + b.AddObject(obj, fmt.Sprintf("%s/%d", targetPrefix, i)) + } + err = b.PutAll() + a.NoError(err) + + list, err = b.ListAllObjects(targetPrefix) + a.NoError(err) + a.Len(list, objectCount) +} + +func TestCopyFrom(t *testing.T) { + a := assert.New(t) + createBucket(testPutBucketName) + createBucket(testCopyBucketName) + testPutObject(t) + + f := openFile(t) + fs, _ := f.Stat() + defer f.Close() // nolint:gosec + + svc := getTestClient(t) + b1, err := svc.GetBucket(testPutBucketName) + a.NoError(err) + + b2, err := svc.GetBucket(testCopyBucketName) + a.NoError(err) + + // copy data + resp, err := b2.CopyFrom(testPutBucketName, testS3Path, testS3Path) + a.NoError(err) + a.NotEmpty(resp.ETag) + + // check copied content + data, err := b1.GetObjectByte(testS3Path) + a.NoError(err) + a.Equal(int(fs.Size()), len(data)) +} + +func TestCopyTo(t *testing.T) { + a := assert.New(t) + createBucket(testPutBucketName) + createBucket(testCopyBucketName) + testPutObject(t) + + f := openFile(t) + fs, _ := f.Stat() + defer f.Close() // nolint:gosec + + svc := getTestClient(t) + b1, err := svc.GetBucket(testPutBucketName) + a.NoError(err) + + b2, err := svc.GetBucket(testCopyBucketName) + a.NoError(err) + + // copy data + resp, err := b1.CopyTo(testS3Path, testCopyBucketName, testS3Path) + a.NoError(err) + a.NotEmpty(resp.ETag) + + // check copied content + data, err := b2.GetObjectByte(testS3Path) + a.NoError(err) + a.Equal(int(fs.Size()), len(data)) +} + func TestDeleteObject(t *testing.T) { assert := assert.New(t) createBucket(testPutBucketName) diff --git a/s3/client.go b/s3/client.go index dbbf108..f9ce242 100644 --- a/s3/client.go +++ b/s3/client.go @@ -144,6 +144,16 @@ func (svc *S3) ForceDeleteBucket(name string) error { return nil } +// CopyObject executes `CopyObject` operation. +func (svc *S3) CopyObject(req CopyObjectRequest) (CopyObjectResponse, error) { + out, err := svc.copyObject(req.ToInput()) + return NewCopyObjectResponse(out), err +} + +func (svc *S3) copyObject(input *SDK.CopyObjectInput) (*SDK.CopyObjectOutput, error) { + return svc.client.CopyObject(input) +} + // Infof logging information. func (svc *S3) Infof(format string, v ...interface{}) { svc.logger.Infof(serviceName, format, v...) diff --git a/s3/request_type.go b/s3/request_type.go new file mode 100644 index 0000000..8f78a83 --- /dev/null +++ b/s3/request_type.go @@ -0,0 +1,228 @@ +package s3 + +import ( + "fmt" + "time" + + SDK "github.com/aws/aws-sdk-go/service/s3" + "github.com/evalphobia/aws-sdk-go-wrapper/private/pointers" +) + +// CopyObjectRequest has parameters for `CopyObject` operation. +type CopyObjectRequest struct { + SrcBucket string + SrcPath string + DestBucket string + DestPath string + // if true, add prefix to bucket name + UseSamePrefix bool + + // optional params + // ref: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html + ACL string + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + CopySourceIfMatch string + CopySourceIfModifiedSince time.Time + CopySourceIfNoneMatch string + CopySourceIfUnmodifiedSince time.Time + CopySourceSSECustomerAlgorithm string + CopySourceSSECustomerKey string + CopySourceSSECustomerKeyMD5 string + Expires time.Time + GrantFullControl string + GrantRead string + GrantReadACP string + GrantWriteACP string + Metadata map[string]string + MetadataDirective string + ObjectLockLegalHoldStatus string + ObjectLockMode string + ObjectLockRetainUntilDate time.Time + RequestPayer string + SSECustomerAlgorithm string + SSECustomerKey string + SSECustomerKeyMD5 string + SSEKMSEncryptionContext string + SSEKMSKeyID string + ServerSideEncryption string + StorageClass string + Tagging string + TaggingDirective string + WebsiteRedirectLocation string +} + +func (r CopyObjectRequest) ToInput() *SDK.CopyObjectInput { + in := &SDK.CopyObjectInput{} + + if r.SrcBucket != "" && r.SrcPath != "" { + in.SetCopySource(fmt.Sprintf("/%s/%s", r.SrcBucket, r.SrcPath)) + } + if r.DestBucket != "" { + in.SetBucket(r.DestBucket) + } + if r.DestPath != "" { + in.SetKey(r.DestPath) + } + + if r.ACL != "" { + in.SetACL(r.ACL) + } + if r.CacheControl != "" { + in.SetCacheControl(r.CacheControl) + } + if r.ContentDisposition != "" { + in.SetContentDisposition(r.ContentDisposition) + } + if r.ContentEncoding != "" { + in.SetContentEncoding(r.ContentEncoding) + } + if r.ContentLanguage != "" { + in.SetContentLanguage(r.ContentLanguage) + } + if r.ContentType != "" { + in.SetContentType(r.ContentType) + } + if r.CopySourceIfMatch != "" { + in.SetCopySourceIfMatch(r.CopySourceIfMatch) + } + if r.CopySourceIfNoneMatch != "" { + in.SetCopySourceIfNoneMatch(r.CopySourceIfNoneMatch) + } + if r.CopySourceSSECustomerAlgorithm != "" { + in.SetCopySourceSSECustomerAlgorithm(r.CopySourceSSECustomerAlgorithm) + } + if r.CopySourceSSECustomerKey != "" { + in.SetCopySourceSSECustomerKey(r.CopySourceSSECustomerKey) + } + if r.CopySourceSSECustomerKeyMD5 != "" { + in.SetCopySourceSSECustomerKeyMD5(r.CopySourceSSECustomerKeyMD5) + } + if r.GrantFullControl != "" { + in.SetGrantFullControl(r.GrantFullControl) + } + if r.GrantRead != "" { + in.SetGrantRead(r.GrantRead) + } + if r.GrantReadACP != "" { + in.SetGrantReadACP(r.GrantReadACP) + } + if r.GrantWriteACP != "" { + in.SetGrantWriteACP(r.GrantWriteACP) + } + if r.MetadataDirective != "" { + in.SetMetadataDirective(r.MetadataDirective) + } + if r.ObjectLockLegalHoldStatus != "" { + in.SetObjectLockLegalHoldStatus(r.ObjectLockLegalHoldStatus) + } + if r.ObjectLockMode != "" { + in.SetObjectLockMode(r.ObjectLockMode) + } + if r.RequestPayer != "" { + in.SetRequestPayer(r.RequestPayer) + } + if r.SSECustomerAlgorithm != "" { + in.SetSSECustomerAlgorithm(r.SSECustomerAlgorithm) + } + if r.SSECustomerKey != "" { + in.SetSSECustomerKey(r.SSECustomerKey) + } + if r.SSECustomerKeyMD5 != "" { + in.SetSSECustomerKeyMD5(r.SSECustomerKeyMD5) + } + if r.SSEKMSEncryptionContext != "" { + in.SetSSEKMSEncryptionContext(r.SSEKMSEncryptionContext) + } + if r.SSEKMSKeyID != "" { + in.SetSSEKMSKeyId(r.SSEKMSKeyID) + } + if r.ServerSideEncryption != "" { + in.SetServerSideEncryption(r.ServerSideEncryption) + } + if r.StorageClass != "" { + in.SetStorageClass(r.StorageClass) + } + if r.Tagging != "" { + in.SetTagging(r.Tagging) + } + if r.TaggingDirective != "" { + in.SetTaggingDirective(r.TaggingDirective) + } + if r.WebsiteRedirectLocation != "" { + in.SetWebsiteRedirectLocation(r.WebsiteRedirectLocation) + } + + if !r.CopySourceIfModifiedSince.IsZero() { + in.SetCopySourceIfModifiedSince(r.CopySourceIfModifiedSince) + } + if !r.CopySourceIfUnmodifiedSince.IsZero() { + in.SetCopySourceIfUnmodifiedSince(r.CopySourceIfUnmodifiedSince) + } + if !r.Expires.IsZero() { + in.SetExpires(r.Expires) + } + if !r.ObjectLockRetainUntilDate.IsZero() { + in.SetObjectLockRetainUntilDate(r.ObjectLockRetainUntilDate) + } + + if len(r.Metadata) != 0 { + m := make(map[string]*string, len(r.Metadata)) + for k, v := range r.Metadata { + m[k] = pointers.String(v) + } + in.SetMetadata(m) + } + return in +} + +// ListObjectsRequest has parameters for `ListObjectsV2` operation. +type ListObjectsRequest struct { + Bucket string + + // optional + ContinuationToken string + Delimiter string + EncodingType string + FetchOwner bool + MaxKeys int64 + Prefix string + RequestPayer string + StartAfter string +} + +func (r ListObjectsRequest) ToInput() *SDK.ListObjectsV2Input { + in := &SDK.ListObjectsV2Input{} + if r.Bucket != "" { + in.SetBucket(r.Bucket) + } + + if r.ContinuationToken != "" { + in.SetContinuationToken(r.ContinuationToken) + } + if r.Delimiter != "" { + in.SetDelimiter(r.Delimiter) + } + if r.EncodingType != "" { + in.SetEncodingType(r.EncodingType) + } + if r.FetchOwner { + in.SetFetchOwner(r.FetchOwner) + } + if r.MaxKeys != 0 { + in.SetMaxKeys(r.MaxKeys) + } + if r.Prefix != "" { + in.SetPrefix(r.Prefix) + } + if r.RequestPayer != "" { + in.SetRequestPayer(r.RequestPayer) + } + if r.StartAfter != "" { + in.SetStartAfter(r.StartAfter) + } + return in +} diff --git a/s3/response_type.go b/s3/response_type.go new file mode 100644 index 0000000..88c1e24 --- /dev/null +++ b/s3/response_type.go @@ -0,0 +1,183 @@ +package s3 + +import ( + "time" + + SDK "github.com/aws/aws-sdk-go/service/s3" +) + +// CopyObjectResponse contains data from CopyObject. +type CopyObjectResponse struct { + ETag string + LastModified time.Time + + CopySourceVersionID string + Expiration string + RequestCharged string + SSECustomerAlgorithm string + SSECustomerKeyMD5 string + SSEKMSEncryptionContext string + SSEKMSKeyID string + ServerSideEncryption string + VersionID string +} + +func NewCopyObjectResponse(out *SDK.CopyObjectOutput) CopyObjectResponse { + r := CopyObjectResponse{} + if out == nil { + return r + } + + if out.CopySourceVersionId != nil { + r.CopySourceVersionID = *out.CopySourceVersionId + } + if out.Expiration != nil { + r.Expiration = *out.Expiration + } + if out.RequestCharged != nil { + r.RequestCharged = *out.RequestCharged + } + if out.SSECustomerAlgorithm != nil { + r.SSECustomerAlgorithm = *out.SSECustomerAlgorithm + } + if out.SSECustomerKeyMD5 != nil { + r.SSECustomerKeyMD5 = *out.SSECustomerKeyMD5 + } + if out.SSEKMSEncryptionContext != nil { + r.SSEKMSEncryptionContext = *out.SSEKMSEncryptionContext + } + if out.SSEKMSKeyId != nil { + r.SSEKMSKeyID = *out.SSEKMSKeyId + } + if out.ServerSideEncryption != nil { + r.ServerSideEncryption = *out.ServerSideEncryption + } + if out.VersionId != nil { + r.VersionID = *out.VersionId + } + + if out.CopyObjectResult != nil { + d := out.CopyObjectResult + if d.ETag != nil { + r.ETag = *d.ETag + } + if d.LastModified != nil { + r.LastModified = *d.LastModified + } + } + return r +} + +// ListObjectsResponse contains data from ListObjectsV2. +type ListObjectsResponse struct { + CommonPrefixes []string + Contents []Object + ContinuationToken string + Delimiter string + EncodingType string + IsTruncated bool + KeyCount int64 + MaxKeys int64 + Name string + NextContinuationToken string + Prefix string + StartAfter string +} + +func NewListObjectsResponse(out *SDK.ListObjectsV2Output) ListObjectsResponse { + r := ListObjectsResponse{} + if out == nil { + return r + } + + if out.ContinuationToken != nil { + r.ContinuationToken = *out.ContinuationToken + } + if out.Delimiter != nil { + r.Delimiter = *out.Delimiter + } + if out.EncodingType != nil { + r.EncodingType = *out.EncodingType + } + if out.IsTruncated != nil { + r.IsTruncated = *out.IsTruncated + } + if out.KeyCount != nil { + r.KeyCount = *out.KeyCount + } + if out.MaxKeys != nil { + r.MaxKeys = *out.MaxKeys + } + if out.Name != nil { + r.Name = *out.Name + } + if out.NextContinuationToken != nil { + r.NextContinuationToken = *out.NextContinuationToken + } + if out.Prefix != nil { + r.Prefix = *out.Prefix + } + if out.StartAfter != nil { + r.StartAfter = *out.StartAfter + } + + if len(out.CommonPrefixes) != 0 { + list := make([]string, 0, len(out.CommonPrefixes)) + for _, v := range out.CommonPrefixes { + if v != nil && v.Prefix != nil { + list = append(list, *v.Prefix) + } + } + r.CommonPrefixes = list + } + + if len(out.Contents) != 0 { + list := make([]Object, len(out.Contents)) + for i, v := range out.Contents { + list[i] = NewObject(v) + } + r.Contents = list + } + + return r +} + +type Object struct { + ETag string + Key string + LastModified time.Time + Size int64 + StorageClass string + OwnerID string + OwnerDisplayName string +} + +func NewObject(d *SDK.Object) Object { + o := Object{} + if d == nil { + return o + } + + if d.ETag != nil { + o.ETag = *d.ETag + } + if d.Key != nil { + o.Key = *d.Key + } + if d.Size != nil { + o.Size = *d.Size + } + if d.LastModified != nil { + o.LastModified = *d.LastModified + } + if d.Owner != nil { + owner := d.Owner + if owner.ID != nil { + o.OwnerID = *owner.ID + } + if owner.DisplayName != nil { + o.OwnerDisplayName = *owner.DisplayName + } + } + return o +}