From 90cbac15b44bd44e2e931ffdba8692e518007160 Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Fri, 26 Jan 2024 22:59:49 +0900 Subject: [PATCH 1/8] Fix: cp local file to s3 --- app/external/s3_helper_test.go | 23 +++++++- cmd/subcmd/s3hub/cp.go | 29 +++++++--- cmd/subcmd/s3hub/cp_test.go | 100 +++++++++++++++++++++++++++++++++ go.mod | 4 ++ go.sum | 36 ++++++++++++ 5 files changed, 182 insertions(+), 10 deletions(-) diff --git a/app/external/s3_helper_test.go b/app/external/s3_helper_test.go index 6dab305..b86ec1b 100644 --- a/app/external/s3_helper_test.go +++ b/app/external/s3_helper_test.go @@ -36,17 +36,34 @@ func CreateS3Buckets(t *testing.T, client *s3.Client, buckets []model.Bucket) { } } -// DeleteAllS3BucketDelete deletes all S3 buckets. +// DeleteAllS3BucketDelete deletes all S3 buckets and objects. func DeleteAllS3BucketDelete(t *testing.T, client *s3.Client) { t.Helper() + ctx := context.Background() - buckets, err := client.ListBuckets(context.Background(), &s3.ListBucketsInput{}) + buckets, err := client.ListBuckets(ctx, &s3.ListBucketsInput{}) if err != nil { t.Fatal(err) } for _, bucket := range buckets.Buckets { - if _, err := client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{Bucket: bucket.Name}); err != nil { + output, err := client.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: bucket.Name, + }) + if err != nil { + t.Fatal(err) + } + + for _, object := range output.Contents { + if _, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: bucket.Name, + Key: object.Key, + }); err != nil { + t.Fatal(err) + } + } + + if _, err := client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: bucket.Name}); err != nil { t.Fatal(err) } } diff --git a/cmd/subcmd/s3hub/cp.go b/cmd/subcmd/s3hub/cp.go index a7332e3..5ef8eea 100644 --- a/cmd/subcmd/s3hub/cp.go +++ b/cmd/subcmd/s3hub/cp.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/fatih/color" + "github.com/gogf/gf/os/gfile" "github.com/nao1215/rainbow/app/domain/model" "github.com/nao1215/rainbow/app/usecase" "github.com/nao1215/rainbow/cmd/subcmd" @@ -65,15 +66,16 @@ type copyPathPair struct { From string // To is a path of destination. To string - // copyType is a type of copy. + // Type is represents a type of copy. + // Type: "from local to S3", "from S3 to local", "from S3 to S3". Type copyType } // newCopyPathPair returns a new copyPathPair. func newCopyPathPair(from, to string) *copyPathPair { pair := ©PathPair{ - From: filepath.Clean(from), - To: filepath.Clean(to), + From: from, + To: to, } pair.Type = pair.copyType() return pair @@ -121,22 +123,35 @@ func (c *cpCmd) Do() error { case copyTypeS3ToS3: return c.s3ToS3() case copyTypeUnknown: + fallthrough default: return fmt.Errorf("unsupported copy type. from=%s, to=%s", color.YellowString(c.pair.From), color.YellowString(c.pair.To)) } - return nil +} + +// copyTargetsInLocal returns a slice of target files in local. +func (c *cpCmd) copyTargetsInLocal() ([]string, error) { + if gfile.IsFile(c.pair.From) { + return []string{c.pair.From}, nil + } + targets, err := file.WalkDir(c.pair.From) + if err != nil { + return nil, err + } + return targets, nil } // localToS3 copies from local to S3. func (c *cpCmd) localToS3() error { - targets, err := file.WalkDir(c.pair.From) + targets, err := c.copyTargetsInLocal() if err != nil { return err } - toBucket, toKey := model.NewBucketWithoutProtocol(c.pair.To).Split() + toBucket, toKey := model.NewBucketWithoutProtocol(c.pair.To).Split() fileNum := len(targets) + for i, v := range targets { data, err := os.ReadFile(filepath.Clean(v)) if err != nil { @@ -145,7 +160,7 @@ func (c *cpCmd) localToS3() error { if _, err := c.s3hub.FileUploader.UploadFile(c.ctx, &usecase.FileUploaderInput{ Bucket: toBucket, Region: c.s3hub.region, - Key: toKey, + Key: model.S3Key(filepath.Join(toKey.String(), filepath.Base(v))), Data: data, }); err != nil { return err diff --git a/cmd/subcmd/s3hub/cp_test.go b/cmd/subcmd/s3hub/cp_test.go index 6dbf47a..e511a0e 100644 --- a/cmd/subcmd/s3hub/cp_test.go +++ b/cmd/subcmd/s3hub/cp_test.go @@ -3,6 +3,8 @@ package s3hub import ( "bytes" "testing" + + "github.com/google/go-cmp/cmp" ) func Test_cp(t *testing.T) { @@ -23,3 +25,101 @@ func Test_cp(t *testing.T) { } }) } + +func Test_newCopyPathPair(t *testing.T) { + t.Parallel() + + type args struct { + from string + to string + } + tests := []struct { + name string + args args + want *copyPathPair + }{ + { + name: "copyTypeLocalToS3", + args: args{ + from: "/path/to/file.txt", + to: "s3://mybucket/path/to/file.txt", + }, + want: ©PathPair{ + From: "/path/to/file.txt", + To: "s3://mybucket/path/to/file.txt", + Type: copyTypeLocalToS3, + }, + }, + { + name: "copyTypeS3ToLocal", + args: args{ + from: "s3://mybucket/path/to/file.txt", + to: "/path/to/file.txt", + }, + want: ©PathPair{ + From: "s3://mybucket/path/to/file.txt", + To: "/path/to/file.txt", + Type: copyTypeS3ToLocal, + }, + }, + { + name: "copyTypeS3ToS3", + args: args{ + from: "s3://mybucket1/path/to/file.txt", + to: "s3://mybucket2/path/to/file.txt", + }, + want: ©PathPair{ + From: "s3://mybucket1/path/to/file.txt", + To: "s3://mybucket2/path/to/file.txt", + Type: copyTypeS3ToS3, + }, + }, + { + name: "copyTypeUnknown: from local to local", + args: args{ + from: "/path/to/file.txt", + to: "/path/to/file.txt", + }, + want: ©PathPair{ + From: "/path/to/file.txt", + To: "/path/to/file.txt", + Type: copyTypeUnknown, + }, + }, + { + name: "copyTypeUnknown: from is empty", + args: args{ + from: "", + to: "/path/to/file.txt", + }, + want: ©PathPair{ + From: "", + To: "/path/to/file.txt", + Type: copyTypeUnknown, + }, + }, + { + name: "copyTypeUnknown: to is empty", + args: args{ + from: "/path/to/file.txt", + to: "", + }, + want: ©PathPair{ + From: "/path/to/file.txt", + To: "", + Type: copyTypeUnknown, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := newCopyPathPair(tt.args.from, tt.args.to) + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("newCopyPathPair() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/go.mod b/go.mod index 4e42101..81b1125 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/charmbracelet/lipgloss v0.9.1 github.com/charmbracelet/log v0.3.1 github.com/fatih/color v1.16.0 + github.com/gogf/gf v1.16.9 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/google/wire v0.5.0 @@ -51,6 +52,7 @@ require ( github.com/caarlos0/env/v9 v9.0.0 // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/google/subcommands v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -69,6 +71,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/spf13/pflag v1.0.5 // indirect + go.opentelemetry.io/otel v1.0.0 // indirect + go.opentelemetry.io/otel/trace v1.0.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect diff --git a/go.sum b/go.sum index e3bfde1..b04fa1c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= @@ -58,6 +60,8 @@ github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1 github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/charmbracelet/log v0.3.1 h1:TjuY4OBNbxmHWSwO3tosgqs5I3biyY8sQPny/eCMTYw= github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3zwHGWuuq8g= +github.com/clbanning/mxj v1.8.5-0.20200714211355-ff02cfb8ea28 h1:LdXxtjzvZYhhUaonAaAKArG3pyC67kGL3YY+6hGG8G4= +github.com/clbanning/mxj v1.8.5-0.20200714211355-ff02cfb8ea28/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -67,11 +71,21 @@ github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gogf/gf v1.16.9 h1:Q803UmmRo59+Ws08sMVFOcd8oNpkSWL9vS33hlo/Cyk= +github.com/gogf/gf v1.16.9/go.mod h1:8Q/kw05nlVRp+4vv7XASBsMe9L1tsVKiGoeP2AHnlkk= +github.com/gomodule/redigo v1.8.5 h1:nRAxCa+SVsyjSBrtZmG/cqb6VbTmuRzpg/PoTFlpumc= +github.com/gomodule/redigo v1.8.5/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k= @@ -80,6 +94,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0= +github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -101,14 +119,17 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -129,6 +150,8 @@ github.com/nao1215/gorky v0.2.1 h1:kxXYhCNBbtGru9CCSYx+QC0JZfZJ1csY3uLbb5n2WKA= github.com/nao1215/gorky v0.2.1/go.mod h1:fJNLiXzn3YkteARC8xghfHjkt+C5xtHOaRgmVnJEMOs= github.com/nao1215/spare v0.0.2 h1:bZNKutQZfg+v7KX+6w9EvvTvY0ySD4gZa+uXRHEQZMM= github.com/nao1215/spare v0.0.2/go.mod h1:2907GiSM1IWhSKt+kgLV4tUtlOI53dMEU1nwR0SLw0E= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -147,12 +170,18 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/wailsapp/mimetype v1.4.1 h1:pQN9ycO7uo4vsUUuPeHEYoUkLVkaRntMnHJxVwYhwHs= github.com/wailsapp/mimetype v1.4.1/go.mod h1:9aV5k31bBOv5z6u+QP8TltzvNGJPmNJD4XlAL3U+j3o= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI= +go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg= +go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4= +go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= @@ -164,6 +193,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -173,6 +203,9 @@ golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -203,12 +236,15 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From ea514ef796d50562c9c452eff9f60a62a8e41fb1 Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Fri, 26 Jan 2024 23:19:27 +0900 Subject: [PATCH 2/8] Add unit test --- cmd/subcmd/s3hub/cp_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/cmd/subcmd/s3hub/cp_test.go b/cmd/subcmd/s3hub/cp_test.go index e511a0e..a38973f 100644 --- a/cmd/subcmd/s3hub/cp_test.go +++ b/cmd/subcmd/s3hub/cp_test.go @@ -110,6 +110,42 @@ func Test_newCopyPathPair(t *testing.T) { Type: copyTypeUnknown, }, }, + { + name: "copyTypeUnknown: from and to are empty", + args: args{ + from: "", + to: "", + }, + want: ©PathPair{ + From: "", + To: "", + Type: copyTypeUnknown, + }, + }, + { + name: "copyTypeUnknown: use file:// protocol", + args: args{ + from: "file:///path/to/file.txt", + to: "file:///path/to/file.txt", + }, + want: ©PathPair{ + From: "file:///path/to/file.txt", + To: "file:///path/to/file.txt", + Type: copyTypeUnknown, + }, + }, + { + name: "copyTypeUnknown: use bad s3:// protocol", + args: args{ + from: "s3:/mybucket/path/to/file.txt", + to: "s3:/mybucket/path/to/file.txt", + }, + want: ©PathPair{ + From: "s3:/mybucket/path/to/file.txt", + To: "s3:/mybucket/path/to/file.txt", + Type: copyTypeUnknown, + }, + }, } for _, tt := range tests { tt := tt From 77a2e1d894157c88c018c354588e96c0b73c4f93 Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Fri, 26 Jan 2024 23:29:11 +0900 Subject: [PATCH 3/8] Fix coderabbitai --- app/domain/model/s3.go | 7 ++++++- app/domain/model/s3_test.go | 14 ++++++++++++-- cmd/subcmd/s3hub/cp.go | 3 +-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/app/domain/model/s3.go b/app/domain/model/s3.go index 007e0e2..70afa8e 100644 --- a/app/domain/model/s3.go +++ b/app/domain/model/s3.go @@ -229,7 +229,12 @@ func (b Bucket) Split() (Bucket, S3Key) { if len(s) == 1 { return b, "" } - return Bucket(s[0]), S3Key(strings.Join(s[1:], "/")) + + key := strings.Join(s[1:], "/") + if key == "" { + return Bucket(s[0]), S3Key("") + } + return Bucket(s[0]), S3Key(filepath.Clean(key)) } // Validate returns true if the Bucket is valid. diff --git a/app/domain/model/s3_test.go b/app/domain/model/s3_test.go index 185e4a6..c1fd621 100644 --- a/app/domain/model/s3_test.go +++ b/app/domain/model/s3_test.go @@ -659,11 +659,21 @@ func TestBucket_Split(t *testing.T) { want: Bucket("abc"), want1: S3Key(filepath.Join("def", "ghi")), }, + { + name: "If Bucket is 'abc/def/ghi/', Split() returns 'abc' and 'def/ghi/'", + b: Bucket(filepath.Join("abc", "def", "ghi/")), + want: Bucket("abc"), + want1: S3Key(filepath.Join("def", "ghi")), + }, + { + name: "If Bucket is 'abc/def/../ghi/jkl', Split() returns 'abc' and 'def/../ghi/jkl'", + b: Bucket(filepath.Join("abc", "def", "..", "ghi", "jkl")), + want: Bucket("abc"), + want1: S3Key(filepath.Join("ghi", "jkl")), + }, } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { - t.Parallel() got, got1 := tt.b.Split() if got != tt.want { t.Errorf("Bucket.Split() got = %v, want %v", got, tt.want) diff --git a/cmd/subcmd/s3hub/cp.go b/cmd/subcmd/s3hub/cp.go index 5ef8eea..c064296 100644 --- a/cmd/subcmd/s3hub/cp.go +++ b/cmd/subcmd/s3hub/cp.go @@ -66,8 +66,7 @@ type copyPathPair struct { From string // To is a path of destination. To string - // Type is represents a type of copy. - // Type: "from local to S3", "from S3 to local", "from S3 to S3". + // Type indicates the direction of the copy operation: from local to S3, from S3 to local, or within S3. Type copyType } From 0db9785958f1413797803ad7cc74c77633727dc4 Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Sun, 28 Jan 2024 00:15:57 +0900 Subject: [PATCH 4/8] Fix typo --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index e9ad0aa..10beb61 100644 --- a/LICENSE +++ b/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file From c79562b61166ce9b880b4fbe048a33dbaf4844be Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Sun, 28 Jan 2024 00:16:14 +0900 Subject: [PATCH 5/8] Fix: s3hub cp s3 to local --- app/external/s3.go | 1 + app/interactor/mock/s3.go | 15 +++++ app/interactor/s3.go | 1 + cmd/subcmd/s3hub/cp.go | 59 ++++++++++-------- cmd/subcmd/s3hub/cp_test.go | 118 ++++++++++++++++++++++++++++++++++++ 5 files changed, 169 insertions(+), 25 deletions(-) create mode 100644 app/interactor/mock/s3.go diff --git a/app/external/s3.go b/app/external/s3.go index 45940fd..8dbe028 100644 --- a/app/external/s3.go +++ b/app/external/s3.go @@ -368,6 +368,7 @@ func (c *S3ObjectUploader) UploadS3Object(ctx context.Context, input *service.S3 if err != nil { return nil, err } + return &service.S3ObjectUploaderOutput{ ContentType: input.S3Object.ContentType(), ContentLength: input.S3Object.ContentLength(), diff --git a/app/interactor/mock/s3.go b/app/interactor/mock/s3.go new file mode 100644 index 0000000..b1f4455 --- /dev/null +++ b/app/interactor/mock/s3.go @@ -0,0 +1,15 @@ +package mock + +import ( + "context" + + "github.com/nao1215/rainbow/app/usecase" +) + +// S3ObjectsLister is a mock of the S3ObjectLister interface. +type S3ObjectsLister func(ctx context.Context, input *usecase.S3ObjectsListerInput) (*usecase.S3ObjectsListerOutput, error) + +// ListS3Objects calls the ListS3ObjectsFunc. +func (m S3ObjectsLister) ListS3Objects(ctx context.Context, input *usecase.S3ObjectsListerInput) (*usecase.S3ObjectsListerOutput, error) { + return m(ctx, input) +} diff --git a/app/interactor/s3.go b/app/interactor/s3.go index afa77a2..39f1504 100644 --- a/app/interactor/s3.go +++ b/app/interactor/s3.go @@ -274,6 +274,7 @@ func (u *FileUploader) UploadFile(ctx context.Context, input *usecase.FileUpload if err != nil { return nil, err } + return &usecase.FileUploaderOutput{ ContentType: output.ContentType, ContentLength: output.ContentLength, diff --git a/cmd/subcmd/s3hub/cp.go b/cmd/subcmd/s3hub/cp.go index c064296..a607d76 100644 --- a/cmd/subcmd/s3hub/cp.go +++ b/cmd/subcmd/s3hub/cp.go @@ -154,15 +154,16 @@ func (c *cpCmd) localToS3() error { for i, v := range targets { data, err := os.ReadFile(filepath.Clean(v)) if err != nil { - return err + return fmt.Errorf("can not read file %s: %w", color.YellowString(v), err) } + if _, err := c.s3hub.FileUploader.UploadFile(c.ctx, &usecase.FileUploaderInput{ Bucket: toBucket, Region: c.s3hub.region, Key: model.S3Key(filepath.Join(toKey.String(), filepath.Base(v))), Data: data, }); err != nil { - return err + return fmt.Errorf("can not upload file %s: %w", color.YellowString(v), err) } c.printf("[%d/%d] copy %s to %s\n", i+1, @@ -177,27 +178,11 @@ func (c *cpCmd) localToS3() error { // s3ToLocal copies from S3 to local. func (c *cpCmd) s3ToLocal() error { fromBucket, fromKey := model.NewBucketWithoutProtocol(c.pair.From).Split() - _, toKey := model.NewBucketWithoutProtocol(c.pair.To).Split() - - listOutput, err := c.s3hub.ListS3Objects(c.ctx, &usecase.S3ObjectsListerInput{ - Bucket: fromBucket, - }) + targets, err := c.filterS3Objects(fromBucket, fromKey) if err != nil { return err } - targets := make([]model.S3Key, 0, len(listOutput.Objects)) - for _, v := range listOutput.Objects { - if strings.Contains(v.S3Key.String(), fromKey.String()) { - targets = append(targets, v.S3Key) - } - } - - if len(targets) == 0 { - return fmt.Errorf("no objects found. bucket=%s, key=%s", - color.YellowString(fromBucket.String()), color.YellowString(fromKey.String())) - } - fileNum := len(targets) for i, v := range targets { downloadOutput, err := c.s3hub.S3ObjectDownloader.DownloadS3Object(c.ctx, &usecase.S3ObjectDownloaderInput{ @@ -205,16 +190,17 @@ func (c *cpCmd) s3ToLocal() error { Key: v, }) if err != nil { - return err + return fmt.Errorf("can not download s3 object=%s: %w", + color.YellowString(fromBucket.Join(v).WithProtocol().String()), err) } - relativePath, err := filepath.Rel(fromKey.String(), v.String()) - if err != nil { - return err + destinationPath := filepath.Clean(filepath.Join(c.pair.To, fromKey.String())) + if err := os.MkdirAll(filepath.Dir(destinationPath), 0755); err != nil { + return fmt.Errorf("can not create directory %s: %w", color.YellowString(filepath.Dir(destinationPath)), err) } - destinationPath := filepath.Join(toKey.String(), relativePath) + if err := downloadOutput.S3Object.ToFile(destinationPath, 0644); err != nil { - return err + return fmt.Errorf("can not write file to %s: %w", color.YellowString(destinationPath), err) } c.printf("[%d/%d] copy %s to %s\n", @@ -227,6 +213,29 @@ func (c *cpCmd) s3ToLocal() error { return nil } +// filterS3Objects returns a slice of S3Key that matches the fromKey. +func (c *cpCmd) filterS3Objects(fromBucket model.Bucket, fromKey model.S3Key) ([]model.S3Key, error) { + listOutput, err := c.s3hub.ListS3Objects(c.ctx, &usecase.S3ObjectsListerInput{ + Bucket: fromBucket, + }) + if err != nil { + return nil, fmt.Errorf("%w: bucket=%s", err, color.YellowString(fromBucket.String())) + } + + targets := make([]model.S3Key, 0, len(listOutput.Objects)) + for _, v := range listOutput.Objects { + if strings.Contains(filepath.Join(fromBucket.String(), v.S3Key.String()), fromKey.String()) { + targets = append(targets, v.S3Key) + } + } + + if len(targets) == 0 { + return nil, fmt.Errorf("no objects found. bucket=%s, key=%s", + color.YellowString(fromBucket.String()), color.YellowString(fromKey.String())) + } + return targets, nil +} + // s3ToS3 copies from S3 to S3. func (c *cpCmd) s3ToS3() error { fromBucket, fromKey := model.NewBucketWithoutProtocol(c.pair.From).Split() diff --git a/cmd/subcmd/s3hub/cp_test.go b/cmd/subcmd/s3hub/cp_test.go index a38973f..fdd3df5 100644 --- a/cmd/subcmd/s3hub/cp_test.go +++ b/cmd/subcmd/s3hub/cp_test.go @@ -2,9 +2,15 @@ package s3hub import ( "bytes" + "context" + "errors" "testing" "github.com/google/go-cmp/cmp" + "github.com/nao1215/rainbow/app/di" + "github.com/nao1215/rainbow/app/domain/model" + "github.com/nao1215/rainbow/app/interactor/mock" + "github.com/nao1215/rainbow/app/usecase" ) func Test_cp(t *testing.T) { @@ -159,3 +165,115 @@ func Test_newCopyPathPair(t *testing.T) { }) } } + +func Test_cpCmd_filterS3Objects(t *testing.T) { + t.Parallel() + + t.Run("filterS3Objects ", func(t *testing.T) { + mockLister := mock.S3ObjectsLister(func(ctx context.Context, input *usecase.S3ObjectsListerInput) (*usecase.S3ObjectsListerOutput, error) { + want := &usecase.S3ObjectsListerInput{ + Bucket: model.NewBucketWithoutProtocol("mybucket"), + } + if diff := cmp.Diff(input, want); diff != "" { + t.Errorf("got %v, want %v", input, want) + } + return &usecase.S3ObjectsListerOutput{ + Objects: model.S3ObjectIdentifiers{ + {S3Key: model.S3Key("path/to/file1.txt")}, + {S3Key: model.S3Key("path/to/file2.txt")}, + {S3Key: model.S3Key("path/to/file3.txt")}, + }, + }, nil + }) + + cpCmd := &cpCmd{ + s3hub: &s3hub{ + S3App: &di.S3App{ + S3ObjectsLister: mockLister, + }, + ctx: context.Background(), + }, + } + + got, err := cpCmd.filterS3Objects(model.NewBucketWithoutProtocol("mybucket"), model.S3Key("path/to")) + if err != nil { + t.Errorf("got %v, want nil", err) + } + + want := []model.S3Key{ + model.S3Key("path/to/file1.txt"), + model.S3Key("path/to/file2.txt"), + model.S3Key("path/to/file3.txt"), + } + + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("filterS3Objects: no objects found", func(t *testing.T) { + mockLister := mock.S3ObjectsLister(func(ctx context.Context, input *usecase.S3ObjectsListerInput) (*usecase.S3ObjectsListerOutput, error) { + want := &usecase.S3ObjectsListerInput{ + Bucket: model.NewBucketWithoutProtocol("mybucket"), + } + if diff := cmp.Diff(input, want); diff != "" { + t.Errorf("got %v, want %v", input, want) + } + return &usecase.S3ObjectsListerOutput{ + Objects: model.S3ObjectIdentifiers{}, + }, nil + }) + + cpCmd := &cpCmd{ + s3hub: &s3hub{ + S3App: &di.S3App{ + S3ObjectsLister: mockLister, + }, + ctx: context.Background(), + }, + } + + _, err := cpCmd.filterS3Objects(model.NewBucketWithoutProtocol("mybucket"), model.S3Key("path/to")) + if err == nil { + t.Errorf("got nil, want error") + } + + want := "no objects found. bucket=mybucket, key=path/to" + got := err.Error() + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("filterS3Objects: ListS3Objects returns error", func(t *testing.T) { + mockLister := mock.S3ObjectsLister(func(ctx context.Context, input *usecase.S3ObjectsListerInput) (*usecase.S3ObjectsListerOutput, error) { + want := &usecase.S3ObjectsListerInput{ + Bucket: model.NewBucketWithoutProtocol("mybucket"), + } + if diff := cmp.Diff(input, want); diff != "" { + t.Errorf("got %v, want %v", input, want) + } + return nil, errors.New("dummy error") + }) + + cpCmd := &cpCmd{ + s3hub: &s3hub{ + S3App: &di.S3App{ + S3ObjectsLister: mockLister, + }, + ctx: context.Background(), + }, + } + + _, err := cpCmd.filterS3Objects(model.NewBucketWithoutProtocol("mybucket"), model.S3Key("path/to")) + if err == nil { + t.Errorf("got nil, want error") + } + + want := "dummy error: bucket=mybucket" + got := err.Error() + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("got %v, want %v", got, want) + } + }) +} From 36b0dda5fc59a4bc97ed3e0a8e9ad9e11fdf72fd Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Sun, 28 Jan 2024 00:26:46 +0900 Subject: [PATCH 6/8] Fix: gosec --- cmd/subcmd/s3hub/cp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/subcmd/s3hub/cp.go b/cmd/subcmd/s3hub/cp.go index a607d76..e781270 100644 --- a/cmd/subcmd/s3hub/cp.go +++ b/cmd/subcmd/s3hub/cp.go @@ -195,7 +195,7 @@ func (c *cpCmd) s3ToLocal() error { } destinationPath := filepath.Clean(filepath.Join(c.pair.To, fromKey.String())) - if err := os.MkdirAll(filepath.Dir(destinationPath), 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(destinationPath), 0750); err != nil { return fmt.Errorf("can not create directory %s: %w", color.YellowString(filepath.Dir(destinationPath)), err) } From ea7398011ce68d569189af374da8818a6ce7708c Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Sun, 28 Jan 2024 00:36:11 +0900 Subject: [PATCH 7/8] Fix: s3hub cp s3 to s3 --- cmd/subcmd/s3hub/cp.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/subcmd/s3hub/cp.go b/cmd/subcmd/s3hub/cp.go index e781270..c7c2c3b 100644 --- a/cmd/subcmd/s3hub/cp.go +++ b/cmd/subcmd/s3hub/cp.go @@ -261,11 +261,7 @@ func (c *cpCmd) s3ToS3() error { fileNum := len(targets) for i, v := range targets { - relativePath, err := filepath.Rel(fromKey.String(), v.String()) - if err != nil { - return err - } - destinationKey := model.S3Key(filepath.Join(toKey.String(), relativePath)) + destinationKey := model.S3Key(filepath.Clean(filepath.Join(toKey.String(), v.String()))) if _, err := c.s3hub.S3ObjectCopier.CopyS3Object(c.ctx, &usecase.S3ObjectCopierInput{ SourceBucket: fromBucket, From 28a9ab8b7df19d7f063818262e5f9a1f17f35976 Mon Sep 17 00:00:00 2001 From: CHIKAMATSU Naohiro Date: Sun, 28 Jan 2024 00:39:55 +0900 Subject: [PATCH 8/8] Update --- doc/img/cover.svg | 186 +++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/doc/img/cover.svg b/doc/img/cover.svg index da61209..5349403 100644 --- a/doc/img/cover.svg +++ b/doc/img/cover.svg @@ -7,7 +7,7 @@ > - + - + - + cmd/subcmd @@ -46,12 +46,12 @@ - + config/spare/config.go @@ -59,12 +59,12 @@ - + ui @@ -72,12 +72,12 @@ - + utils @@ -85,12 +85,12 @@ - + version/version.go @@ -98,12 +98,12 @@ - + domain/model @@ -111,7 +111,7 @@ - + - + interactor @@ -137,12 +137,12 @@ - + cfn @@ -150,12 +150,12 @@ - + common.go @@ -163,12 +163,12 @@ - + s3hub @@ -176,12 +176,12 @@ - + common.go @@ -189,12 +189,12 @@ - + s3hub @@ -202,12 +202,12 @@ - + errfmt/errfmt.go @@ -215,12 +215,12 @@ - + file/file.go @@ -228,12 +228,12 @@ - + xregex/xregex.go @@ -241,12 +241,12 @@ - + aws.go @@ -254,18 +254,18 @@ - + - + s3.go @@ -273,12 +273,12 @@ - + s3_policy.go @@ -286,12 +286,12 @@ - + cloudformation.go @@ -299,12 +299,12 @@ - + cloudfront.go @@ -312,12 +312,12 @@ - + mock @@ -325,7 +325,7 @@ - + - + s3_policy.go @@ -351,12 +351,12 @@ - + s3_retryer.go @@ -364,12 +364,12 @@ - + cloudformation.go @@ -377,12 +377,12 @@ - + cloudfront.go @@ -390,12 +390,12 @@ - + s3.go @@ -403,12 +403,12 @@ - + common.go @@ -416,12 +416,12 @@ - + ls.go @@ -429,12 +429,12 @@ - + root.go @@ -442,18 +442,18 @@ - + - + common.go @@ -461,31 +461,31 @@ - + cp.go - + - + ls.go @@ -493,12 +493,12 @@ - + mb.go @@ -506,12 +506,12 @@ - + rm.go @@ -519,12 +519,12 @@ - + root.go @@ -532,18 +532,18 @@ - + - + command.go @@ -551,12 +551,12 @@ - + create.go @@ -564,12 +564,12 @@ - + delete.go @@ -577,18 +577,18 @@ - + - + list.go @@ -596,12 +596,12 @@ - + root.go @@ -609,19 +609,19 @@ - + - + - +