From 3e52a6d4516be330d9c070613df2e75839891fe7 Mon Sep 17 00:00:00 2001 From: Tom Czarniecki Date: Tue, 8 Nov 2022 08:53:03 +1100 Subject: [PATCH] handle remote & local in any order --- README.md | 4 ++-- client/client.go | 23 ++++++++++++++++++----- client/client_test.go | 12 ++++++++++-- client/crypto/aes_test.go | 3 +-- client/crypto/rsa.go | 3 +-- client/crypto/rsa_test.go | 3 +-- client/mocks/store.go | 14 ++++++++++++++ client/store.go | 5 +++-- client/store/s3.go | 4 ++++ client/store/s3_test.go | 11 ++++++++--- cmd/s3backup/main.go | 4 ++-- utils/tempfile.go | 4 ++-- 12 files changed, 66 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2f4e008d..3add22c5 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ NAME: s3backup put - Upload file to S3 bucket using local credentials USAGE: - s3backup put [command options] s3://bucket/objectkey local_file_path + s3backup put [command options] local_file_path s3://bucket/objectkey OPTIONS: --symKey value Password to use for symmetric AES encryption (optional) @@ -102,7 +102,7 @@ NAME: s3backup vault-put - Upload file to S3 bucket using credentials from vault USAGE: - s3backup vault-put [command options] s3://bucket/objectkey local_file_path + s3backup vault-put [command options] local_file_path s3://bucket/objectkey OPTIONS: --role value Vault role_id to retrieve backup credentials (either role & secret, or token) diff --git a/client/client.go b/client/client.go index 09d57233..3ac7f54e 100644 --- a/client/client.go +++ b/client/client.go @@ -1,6 +1,7 @@ package client import ( + "errors" "log" "os" ) @@ -14,17 +15,23 @@ type Client struct { } func (c *Client) GetRemoteFile(remotePath, localPath string) error { - tempFile := localPath + if c.Store.IsRemote(remotePath) && c.Store.IsRemote(localPath) { + return errors.New("cannot have two remote paths") + } + if c.Store.IsRemote(localPath) { + localPath, remotePath = remotePath, localPath + } + tempFile := localPath if c.Cipher != nil { tempFile += tempFileSuffix defer remove(tempFile) } log.Println("Downloading", remotePath, "to", tempFile) - checksum, err := c.Store.DownloadFile(remotePath, tempFile, c.Hash != nil) - if err != nil { - return err + checksum, cerr := c.Store.DownloadFile(remotePath, tempFile, c.Hash != nil) + if cerr != nil { + return cerr } if c.Hash != nil { @@ -45,8 +52,14 @@ func (c *Client) GetRemoteFile(remotePath, localPath string) error { } func (c *Client) PutLocalFile(remotePath, localPath string) error { - tempFile := localPath + if c.Store.IsRemote(remotePath) && c.Store.IsRemote(localPath) { + return errors.New("cannot have two remote paths") + } + if c.Store.IsRemote(localPath) { + localPath, remotePath = remotePath, localPath + } + tempFile := localPath if c.Cipher != nil { tempFile += tempFileSuffix defer remove(tempFile) diff --git a/client/client_test.go b/client/client_test.go index c1932afd..6bd4a197 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -21,10 +21,12 @@ func TestGetRemoteFileWithoutDecryption(t *testing.T) { Store: store, } + store.EXPECT().IsRemote("bar.txt").Return(false).AnyTimes() + store.EXPECT().IsRemote("s3://foo/bar.txt").Return(true).AnyTimes() store.EXPECT().DownloadFile("s3://foo/bar.txt", "bar.txt", true).Return("muahahaha", nil) hash.EXPECT().Verify("bar.txt", "muahahaha").Return(nil) - assert.NoError(t, c.GetRemoteFile("s3://foo/bar.txt", "bar.txt")) + assert.NoError(t, c.GetRemoteFile("bar.txt", "s3://foo/bar.txt")) } func TestGetRemoteFileWithDecryption(t *testing.T) { @@ -41,6 +43,8 @@ func TestGetRemoteFileWithDecryption(t *testing.T) { Cipher: cipher, } + store.EXPECT().IsRemote("bar.txt").Return(false).AnyTimes() + store.EXPECT().IsRemote("s3://foo/bar.txt").Return(true).AnyTimes() store.EXPECT().DownloadFile("s3://foo/bar.txt", "bar.txt.tmp", true).Return("muahahaha", nil) hash.EXPECT().Verify("bar.txt.tmp", "muahahaha").Return(nil) cipher.EXPECT().Decrypt("bar.txt.tmp", "bar.txt").Return(nil) @@ -60,10 +64,12 @@ func TestPutLocalFileWithoutEncryption(t *testing.T) { Store: store, } + store.EXPECT().IsRemote("bar.txt").Return(false).AnyTimes() + store.EXPECT().IsRemote("s3://foo/bar.txt").Return(true).AnyTimes() hash.EXPECT().Calculate("bar.txt").Return("woahahaha", nil) store.EXPECT().UploadFile("s3://foo/bar.txt", "bar.txt", "woahahaha").Return(nil) - assert.NoError(t, c.PutLocalFile("s3://foo/bar.txt", "bar.txt")) + assert.NoError(t, c.PutLocalFile("bar.txt", "s3://foo/bar.txt")) } func TestPutLocalFileWithEncryption(t *testing.T) { @@ -80,6 +86,8 @@ func TestPutLocalFileWithEncryption(t *testing.T) { Cipher: cipher, } + store.EXPECT().IsRemote("bar.txt").Return(false).AnyTimes() + store.EXPECT().IsRemote("s3://foo/bar.txt").Return(true).AnyTimes() cipher.EXPECT().Encrypt("bar.txt", "bar.txt.tmp").Return(nil) hash.EXPECT().Calculate("bar.txt.tmp").Return("woahahaha", nil) store.EXPECT().UploadFile("s3://foo/bar.txt", "bar.txt.tmp", "woahahaha").Return(nil) diff --git a/client/crypto/aes_test.go b/client/crypto/aes_test.go index a0822703..cd5bb78d 100644 --- a/client/crypto/aes_test.go +++ b/client/crypto/aes_test.go @@ -1,7 +1,6 @@ package crypto import ( - "io/ioutil" "os" "testing" @@ -41,7 +40,7 @@ func testRoundTrip(t *testing.T, key string) { require.NoError(t, cipher.Encrypt(file, encryptedFile), "Cannot encrypt file") require.NoError(t, cipher.Decrypt(encryptedFile, decryptedFile), "Cannot decrypt file") - actual, err := ioutil.ReadFile(decryptedFile) + actual, err := os.ReadFile(decryptedFile) require.NoError(t, err, "Cannot read decrypted file") assert.Equal(t, expected, actual, "File contents are different") diff --git a/client/crypto/rsa.go b/client/crypto/rsa.go index 685e80fa..9f1b0b64 100644 --- a/client/crypto/rsa.go +++ b/client/crypto/rsa.go @@ -13,7 +13,6 @@ import ( "encoding/pem" "fmt" "io" - "io/ioutil" "os" "github.com/tomcz/s3backup/client" @@ -31,7 +30,7 @@ type rsaCipher struct { } func NewRSACipher(pemKeyFile string) (client.Cipher, error) { - buf, err := ioutil.ReadFile(pemKeyFile) + buf, err := os.ReadFile(pemKeyFile) if err != nil { return nil, err } diff --git a/client/crypto/rsa_test.go b/client/crypto/rsa_test.go index 48829088..5eadceec 100644 --- a/client/crypto/rsa_test.go +++ b/client/crypto/rsa_test.go @@ -1,7 +1,6 @@ package crypto import ( - "io/ioutil" "os" "testing" @@ -44,7 +43,7 @@ func TestRoundTripRSAEncryptDecrypt(t *testing.T) { require.NoError(t, pubCipher.Encrypt(file, encryptedFile), "Cannot encrypt file") require.NoError(t, privCipher.Decrypt(encryptedFile, decryptedFile), "Cannot decrypt file") - actual, err := ioutil.ReadFile(decryptedFile) + actual, err := os.ReadFile(decryptedFile) require.NoError(t, err, "Cannot read decrypted file") assert.Equal(t, expected, actual, "File contents are different") diff --git a/client/mocks/store.go b/client/mocks/store.go index 7f18ded8..262951ac 100644 --- a/client/mocks/store.go +++ b/client/mocks/store.go @@ -48,6 +48,20 @@ func (mr *MockStoreMockRecorder) DownloadFile(remotePath, localPath, readChecksu return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DownloadFile", reflect.TypeOf((*MockStore)(nil).DownloadFile), remotePath, localPath, readChecksum) } +// IsRemote mocks base method. +func (m *MockStore) IsRemote(path string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsRemote", path) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsRemote indicates an expected call of IsRemote. +func (mr *MockStoreMockRecorder) IsRemote(path interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsRemote", reflect.TypeOf((*MockStore)(nil).IsRemote), path) +} + // UploadFile mocks base method. func (m *MockStore) UploadFile(remotePath, localPath, checksum string) error { m.ctrl.T.Helper() diff --git a/client/store.go b/client/store.go index f491576a..6a32775f 100644 --- a/client/store.go +++ b/client/store.go @@ -1,8 +1,9 @@ package client +//go:generate mockgen --source=store.go --destination=mocks/store.go --package=mocks + type Store interface { + IsRemote(path string) bool UploadFile(remotePath, localPath, checksum string) error DownloadFile(remotePath, localPath string, readChecksum bool) (checksum string, err error) } - -//go:generate mockgen --source=store.go --destination=mocks/store.go --package=mocks diff --git a/client/store/s3.go b/client/store/s3.go index 6588f64d..c62eca40 100644 --- a/client/store/s3.go +++ b/client/store/s3.go @@ -51,6 +51,10 @@ func NewS3(awsAccessKey, awsSecretKey, awsToken, awsRegion, awsEndpoint string) return &s3store{s3.New(awsSession)}, nil } +func (s *s3store) IsRemote(path string) bool { + return s3PathPattern.MatchString(path) +} + func (s *s3store) UploadFile(remotePath, localPath, checksum string) error { bucket, objectKey, err := splitRemotePath(remotePath) if err != nil { diff --git a/client/store/s3_test.go b/client/store/s3_test.go index 21ce8639..afb8fc8b 100644 --- a/client/store/s3_test.go +++ b/client/store/s3_test.go @@ -1,7 +1,6 @@ package store import ( - "io/ioutil" "net/http/httptest" "os" "testing" @@ -33,6 +32,12 @@ func TestSplitRemotePath(t *testing.T) { assert.Error(t, err) } +func TestIsRemote(t *testing.T) { + store := &s3store{} + assert.True(t, store.IsRemote("s3://bucket/object.key")) + assert.False(t, store.IsRemote("wibble.txt")) +} + func TestRoundTripUploadDownload_withChecksum(t *testing.T) { backend := s3mem.New() faker := gofakes3.New(backend) @@ -63,7 +68,7 @@ func TestRoundTripUploadDownload_withChecksum(t *testing.T) { require.NoError(t, err, "failed to download file") defer os.Remove(downloadFile) - actual, err := ioutil.ReadFile(downloadFile) + actual, err := os.ReadFile(downloadFile) require.NoError(t, err, "Cannot read downloaded file") assert.Equal(t, "wibble", checksum) @@ -100,7 +105,7 @@ func TestRoundTripUploadDownload_withoutChecksum(t *testing.T) { require.NoError(t, err, "failed to download file") defer os.Remove(downloadFile) - actual, err := ioutil.ReadFile(downloadFile) + actual, err := os.ReadFile(downloadFile) require.NoError(t, err, "Cannot read downloaded file") assert.Equal(t, "", checksum) diff --git a/cmd/s3backup/main.go b/cmd/s3backup/main.go index 7c0092bf..bc613c3e 100644 --- a/cmd/s3backup/main.go +++ b/cmd/s3backup/main.go @@ -42,7 +42,7 @@ func main() { cmdBasicPut := &cli.Command{ Name: "put", Usage: "Upload file to S3 bucket using local credentials", - ArgsUsage: "s3://bucket/objectkey local_file_path", + ArgsUsage: "local_file_path s3://bucket/objectkey", Action: basicPut, Flags: basicFlags(true), } @@ -56,7 +56,7 @@ func main() { cmdVaultPut := &cli.Command{ Name: "vault-put", Usage: "Upload file to S3 bucket using credentials from vault", - ArgsUsage: "s3://bucket/objectkey local_file_path", + ArgsUsage: "local_file_path s3://bucket/objectkey", Action: vaultPut, Flags: vaultFlags(true), } diff --git a/utils/tempfile.go b/utils/tempfile.go index 360c9889..bba502b5 100644 --- a/utils/tempfile.go +++ b/utils/tempfile.go @@ -1,9 +1,9 @@ package utils -import "io/ioutil" +import "os" func CreateTempFile(prefix string, body []byte) (string, error) { - file, err := ioutil.TempFile("", prefix) + file, err := os.CreateTemp("", prefix) if err != nil { return "", err }