diff --git a/cache-cli/cmd/clear.go b/cache-cli/cmd/clear.go index aa2cbc3e..f1bcf97c 100644 --- a/cache-cli/cmd/clear.go +++ b/cache-cli/cmd/clear.go @@ -18,10 +18,10 @@ var clearCmd = &cobra.Command{ } func RunClear(cmd *cobra.Command, args []string) { - storage, err := storage.InitStorage() + storage, err := storage.InitStorage(cmd.Context()) utils.Check(err) - err = storage.Clear() + err = storage.Clear(cmd.Context()) utils.Check(err) log.Infof("Deleted all caches.") } diff --git a/cache-cli/cmd/clear_test.go b/cache-cli/cmd/clear_test.go index ac332e21..97a9ee47 100644 --- a/cache-cli/cmd/clear_test.go +++ b/cache-cli/cmd/clear_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -13,13 +14,14 @@ import ( ) func Test__Clear(t *testing.T) { + ctx := context.TODO() log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s no keys", backend), func(*testing.T) { - err := storage.Clear() + err := storage.Clear(ctx) assert.Nil(t, err) RunClear(clearCmd, []string{}) @@ -29,13 +31,13 @@ func Test__Clear(t *testing.T) { }) t.Run(fmt.Sprintf("%s with keys", backend), func(*testing.T) { - err := storage.Clear() + err := storage.Clear(ctx) assert.Nil(t, err) tempFile, _ := ioutil.TempFile(os.TempDir(), "*") - storage.Store("abc001", tempFile.Name()) + storage.Store(ctx, "abc001", tempFile.Name()) - RunClear(hasKeyCmd, []string{}) + RunClear(clearCmd, []string{}) output := readOutputFromFile(t) assert.Contains(t, output, "Deleted all caches.") diff --git a/cache-cli/cmd/delete.go b/cache-cli/cmd/delete.go index 84b8508f..769c44d4 100644 --- a/cache-cli/cmd/delete.go +++ b/cache-cli/cmd/delete.go @@ -24,14 +24,14 @@ func RunDelete(cmd *cobra.Command, args []string) { return } - storage, err := storage.InitStorage() + storage, err := storage.InitStorage(cmd.Context()) utils.Check(err) rawKey := args[0] key := NormalizeKey(rawKey) - if ok, _ := storage.HasKey(key); ok { - err := storage.Delete(key) + if ok, _ := storage.HasKey(cmd.Context(), key); ok { + err := storage.Delete(cmd.Context(), key) utils.Check(err) log.Infof("Key '%s' is deleted.", key) } else { diff --git a/cache-cli/cmd/delete_test.go b/cache-cli/cmd/delete_test.go index 7b14b029..5c90cc46 100644 --- a/cache-cli/cmd/delete_test.go +++ b/cache-cli/cmd/delete_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -13,11 +14,12 @@ import ( ) func Test__Delete(t *testing.T) { + ctx := context.TODO() log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s key is missing", backend), func(*testing.T) { RunDelete(deleteCmd, []string{"this-key-does-not-exist"}) output := readOutputFromFile(t) @@ -26,9 +28,9 @@ func Test__Delete(t *testing.T) { }) t.Run(fmt.Sprintf("%s key is present", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempFile, _ := ioutil.TempFile(os.TempDir(), "*") - storage.Store("abc001", tempFile.Name()) + storage.Store(ctx, "abc001", tempFile.Name()) RunDelete(deleteCmd, []string{"abc001"}) output := readOutputFromFile(t) @@ -37,7 +39,7 @@ func Test__Delete(t *testing.T) { }) t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempFile, _ := ioutil.TempFile(os.TempDir(), "*") RunStore(NewStoreCommand(), []string{"abc/00/33", tempFile.Name()}) diff --git a/cache-cli/cmd/has_key.go b/cache-cli/cmd/has_key.go index a5ee186e..80191b3b 100644 --- a/cache-cli/cmd/has_key.go +++ b/cache-cli/cmd/has_key.go @@ -28,12 +28,12 @@ func RunHasKey(cmd *cobra.Command, args []string) bool { return true } - storage, err := storage.InitStorage() + storage, err := storage.InitStorage(cmd.Context()) utils.Check(err) rawKey := args[0] key := NormalizeKey(rawKey) - exists, err := storage.HasKey(key) + exists, err := storage.HasKey(cmd.Context(), key) utils.Check(err) if exists { diff --git a/cache-cli/cmd/has_key_test.go b/cache-cli/cmd/has_key_test.go index 8f665b0b..10576815 100644 --- a/cache-cli/cmd/has_key_test.go +++ b/cache-cli/cmd/has_key_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -13,11 +14,12 @@ import ( ) func Test__HasKey(t *testing.T) { + ctx := context.TODO() log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s key is missing", backend), func(*testing.T) { RunHasKey(hasKeyCmd, []string{"this-key-does-not-exist"}) output := readOutputFromFile(t) @@ -26,9 +28,9 @@ func Test__HasKey(t *testing.T) { }) t.Run(fmt.Sprintf("%s key is present", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempFile, _ := ioutil.TempFile(os.TempDir(), "*") - storage.Store("abc001", tempFile.Name()) + storage.Store(ctx, "abc001", tempFile.Name()) RunHasKey(hasKeyCmd, []string{"abc001"}) output := readOutputFromFile(t) @@ -37,7 +39,7 @@ func Test__HasKey(t *testing.T) { }) t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempFile, _ := ioutil.TempFile(os.TempDir(), "*") RunStore(NewStoreCommand(), []string{"abc/00/33", tempFile.Name()}) diff --git a/cache-cli/cmd/is_not_empty.go b/cache-cli/cmd/is_not_empty.go index ac68ea13..2cead1a8 100644 --- a/cache-cli/cmd/is_not_empty.go +++ b/cache-cli/cmd/is_not_empty.go @@ -23,10 +23,10 @@ var isNotEmptyCmd = &cobra.Command{ } func RunIsNotEmpty(cmd *cobra.Command, args []string) bool { - storage, err := storage.InitStorage() + storage, err := storage.InitStorage(cmd.Context()) utils.Check(err) - isNotEmpty, err := storage.IsNotEmpty() + isNotEmpty, err := storage.IsNotEmpty(cmd.Context()) utils.Check(err) return isNotEmpty diff --git a/cache-cli/cmd/is_not_empty_test.go b/cache-cli/cmd/is_not_empty_test.go index c2310615..4ebc167e 100644 --- a/cache-cli/cmd/is_not_empty_test.go +++ b/cache-cli/cmd/is_not_empty_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -11,16 +12,17 @@ import ( ) func Test__IsNotEmpty(t *testing.T) { - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + ctx := context.TODO() + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s cache is empty", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) assert.False(t, RunIsNotEmpty(isNotEmptyCmd, []string{})) }) t.Run(fmt.Sprintf("%s cache is not empty", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempFile, _ := ioutil.TempFile(os.TempDir(), "*") - storage.Store("abc001", tempFile.Name()) + storage.Store(ctx, "abc001", tempFile.Name()) assert.True(t, RunIsNotEmpty(isNotEmptyCmd, []string{})) }) diff --git a/cache-cli/cmd/list.go b/cache-cli/cmd/list.go index 85fc1471..51f1f260 100644 --- a/cache-cli/cmd/list.go +++ b/cache-cli/cmd/list.go @@ -36,10 +36,10 @@ func RunList(cmd *cobra.Command, args []string) { sortBy, err := cmd.Flags().GetString("sort-by") utils.Check(err) - storage, err := storage.InitStorageWithConfig(storage.StorageConfig{SortKeysBy: sortBy}) + storage, err := storage.InitStorageWithConfig(cmd.Context(), storage.StorageConfig{SortKeysBy: sortBy}) utils.Check(err) - keys, err := storage.List() + keys, err := storage.List(cmd.Context()) utils.Check(err) if len(keys) == 0 { diff --git a/cache-cli/cmd/list_test.go b/cache-cli/cmd/list_test.go index 79fa1029..6fbc6f49 100644 --- a/cache-cli/cmd/list_test.go +++ b/cache-cli/cmd/list_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -13,14 +14,15 @@ import ( ) func Test__List(t *testing.T) { + ctx := context.TODO() listCmd := NewListCommand() log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s no keys", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) RunList(listCmd, []string{""}) output := readOutputFromFile(t) @@ -29,11 +31,11 @@ func Test__List(t *testing.T) { }) t.Run(fmt.Sprintf("%s with keys", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempFile, _ := ioutil.TempFile(os.TempDir(), "*") - storage.Store("abc001", tempFile.Name()) - storage.Store("abc002", tempFile.Name()) - storage.Store("abc003", tempFile.Name()) + storage.Store(ctx, "abc001", tempFile.Name()) + storage.Store(ctx, "abc002", tempFile.Name()) + storage.Store(ctx, "abc003", tempFile.Name()) RunList(listCmd, []string{}) output := readOutputFromFile(t) diff --git a/cache-cli/cmd/restore.go b/cache-cli/cmd/restore.go index 9dd9e87f..6efc9036 100644 --- a/cache-cli/cmd/restore.go +++ b/cache-cli/cmd/restore.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/fs" "os" @@ -34,7 +35,7 @@ func RunRestore(cmd *cobra.Command, args []string) { return } - storage, err := storage.InitStorage() + storage, err := storage.InitStorage(cmd.Context()) utils.Check(err) metricsManager, err := metrics.InitMetricsManager(metrics.LocalBackend) @@ -57,31 +58,31 @@ func RunRestore(cmd *cobra.Command, args []string) { log.Infof("Detected %s.", lookupResult.DetectedFile) for _, entry := range lookupResult.Entries { log.Infof("Fetching '%s' directory with cache keys '%s'...", entry.Path, strings.Join(entry.Keys, ",")) - downloadAndUnpack(storage, archiver, metricsManager, entry.Keys) + downloadAndUnpack(cmd.Context(), storage, archiver, metricsManager, entry.Keys) } } } else { keys := strings.Split(args[0], ",") - downloadAndUnpack(storage, archiver, metricsManager, keys) + downloadAndUnpack(cmd.Context(), storage, archiver, metricsManager, keys) } } -func downloadAndUnpack(storage storage.Storage, archiver archive.Archiver, metricsManager metrics.MetricsManager, keys []string) { +func downloadAndUnpack(ctx context.Context, storage storage.Storage, archiver archive.Archiver, metricsManager metrics.MetricsManager, keys []string) { for _, rawKey := range keys { key := NormalizeKey(rawKey) - if ok, _ := storage.HasKey(key); ok { + if ok, _ := storage.HasKey(ctx, key); ok { log.Infof("HIT: '%s', using key '%s'.", key, key) - downloadAndUnpackKey(storage, archiver, metricsManager, key) + downloadAndUnpackKey(ctx, storage, archiver, metricsManager, key) break } - availableKeys, err := storage.List() + availableKeys, err := storage.List(ctx) utils.Check(err) matchingKey := findMatchingKey(availableKeys, key) if matchingKey != "" { log.Infof("HIT: '%s', using key '%s'.", key, matchingKey) - downloadAndUnpackKey(storage, archiver, metricsManager, matchingKey) + downloadAndUnpackKey(ctx, storage, archiver, metricsManager, matchingKey) break } else { log.Infof("MISS: '%s'.", key) @@ -100,10 +101,10 @@ func findMatchingKey(availableKeys []storage.CacheKey, match string) string { return "" } -func downloadAndUnpackKey(storage storage.Storage, archiver archive.Archiver, metricsManager metrics.MetricsManager, key string) { +func downloadAndUnpackKey(ctx context.Context, storage storage.Storage, archiver archive.Archiver, metricsManager metrics.MetricsManager, key string) { downloadStart := time.Now() log.Infof("Downloading key '%s'...", key) - compressed, err := storage.Restore(key) + compressed, err := storage.Restore(ctx, key) utils.Check(err) downloadDuration := time.Since(downloadStart) @@ -114,7 +115,7 @@ func downloadAndUnpackKey(storage storage.Storage, archiver archive.Archiver, me unpackStart := time.Now() log.Infof("Unpacking '%s'...", compressed.Name()) - restorationPath, err := archiver.Decompress(compressed.Name()) + restorationPath, err := archiver.Decompress(ctx, compressed.Name()) utils.Check(err) unpackDuration := time.Since(unpackStart) diff --git a/cache-cli/cmd/restore_test.go b/cache-cli/cmd/restore_test.go index a3f20139..a0d5752f 100644 --- a/cache-cli/cmd/restore_test.go +++ b/cache-cli/cmd/restore_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -19,11 +20,12 @@ import ( ) func Test__Restore(t *testing.T) { + ctx := context.TODO() log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s wrong number of arguments", backend), func(t *testing.T) { RunRestore(restoreCmd, []string{"key", "extra-bad-argument"}) output := readOutputFromFile(t) @@ -32,7 +34,7 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s using single missing key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) RunRestore(restoreCmd, []string{"this-key-does-not-exist"}) output := readOutputFromFile(t) @@ -41,14 +43,14 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s using single exact key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") _ = tempFile.Close() archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) - compressAndStore(storage, archiver, "abc-001", tempDir) + compressAndStore(ctx, storage, archiver, "abc-001", tempDir) RunRestore(restoreCmd, []string{"abc-001"}) output := readOutputFromFile(t) @@ -61,14 +63,14 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") _ = tempFile.Close() archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) - compressAndStore(storage, archiver, "abc/00/22", tempDir) + compressAndStore(ctx, storage, archiver, "abc/00/22", tempDir) RunRestore(restoreCmd, []string{"abc/00/22"}) output := readOutputFromFile(t) @@ -82,14 +84,14 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s using single matching key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") _ = tempFile.Close() archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) - compressAndStore(storage, archiver, "abc-001", tempDir) + compressAndStore(ctx, storage, archiver, "abc-001", tempDir) RunRestore(restoreCmd, []string{"abc"}) output := readOutputFromFile(t) @@ -102,15 +104,15 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s only first matching key is used", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") _ = tempFile.Close() archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) - compressAndStore(storage, archiver, "abc-001", tempDir) - compressAndStore(storage, archiver, "abc-002", tempDir) + compressAndStore(ctx, storage, archiver, "abc-001", tempDir) + compressAndStore(ctx, storage, archiver, "abc-002", tempDir) RunRestore(restoreCmd, []string{"abc-001,abc-002"}) output := readOutputFromFile(t) @@ -124,14 +126,14 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s using fallback key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") _ = tempFile.Close() archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) - compressAndStore(storage, archiver, "abc", tempDir) + compressAndStore(ctx, storage, archiver, "abc", tempDir) RunRestore(restoreCmd, []string{"abc-001,abc"}) output := readOutputFromFile(t) @@ -145,14 +147,14 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s using regex key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") _ = tempFile.Close() archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) - compressAndStore(storage, archiver, "abc", tempDir) + compressAndStore(ctx, storage, archiver, "abc", tempDir) RunRestore(restoreCmd, []string{"^abc"}) output := readOutputFromFile(t) @@ -167,6 +169,7 @@ func Test__Restore(t *testing.T) { } func Test__AutomaticRestore(t *testing.T) { + ctx := context.TODO() _, file, _, _ := runtime.Caller(0) cmdPath := filepath.Dir(file) rootPath := filepath.Dir(cmdPath) @@ -175,9 +178,9 @@ func Test__AutomaticRestore(t *testing.T) { log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s nothing found", backend), func(t *testing.T) { - storage.Clear() + storage.Clear(ctx) os.Chdir(cmdPath) RunRestore(restoreCmd, []string{}) @@ -187,7 +190,7 @@ func Test__AutomaticRestore(t *testing.T) { }) t.Run(fmt.Sprintf("%s detects and restores using SEMAPHORE_GIT_BRANCH", backend), func(t *testing.T) { - storage.Clear() + storage.Clear(ctx) os.Chdir(fmt.Sprintf("%s/test/autocache/gems", rootPath)) os.Setenv("SEMAPHORE_GIT_BRANCH", "master") @@ -199,8 +202,8 @@ func Test__AutomaticRestore(t *testing.T) { key := fmt.Sprintf("gems-master-%s", checksum) compressedFile := filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d", key, time.Now().Nanosecond())) archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) - archiver.Compress(compressedFile, "vendor/bundle") - storage.Store(key, compressedFile) + archiver.Compress(ctx, compressedFile, "vendor/bundle") + storage.Store(ctx, key, compressedFile) // restoring RunRestore(restoreCmd, []string{}) @@ -215,7 +218,7 @@ func Test__AutomaticRestore(t *testing.T) { }) t.Run(fmt.Sprintf("%s detects and restores using SEMAPHORE_GIT_PR_BRANCH", backend), func(t *testing.T) { - storage.Clear() + storage.Clear(ctx) os.Chdir(fmt.Sprintf("%s/test/autocache/gems", rootPath)) os.Setenv("SEMAPHORE_GIT_BRANCH", "master") @@ -227,8 +230,8 @@ func Test__AutomaticRestore(t *testing.T) { key := fmt.Sprintf("gems-some-development-branch-%s", checksum) archiver := archive.NewShellOutArchiver(metrics.NewNoOpMetricsManager()) compressedFile := filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d", key, time.Now().Nanosecond())) - archiver.Compress(compressedFile, "vendor/bundle") - storage.Store(key, compressedFile) + archiver.Compress(ctx, compressedFile, "vendor/bundle") + storage.Store(ctx, key, compressedFile) // restoring RunRestore(restoreCmd, []string{}) diff --git a/cache-cli/cmd/root_test.go b/cache-cli/cmd/root_test.go index 6ff445e0..c872ee1d 100644 --- a/cache-cli/cmd/root_test.go +++ b/cache-cli/cmd/root_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "io" "io/ioutil" "os" @@ -38,7 +39,7 @@ var testBackends = map[string]TestBackend{ }, } -func runTestForAllBackends(t *testing.T, test func(string, storage.Storage)) { +func runTestForAllBackends(ctx context.Context, t *testing.T, test func(string, storage.Storage)) { for backendType, testBackend := range testBackends { if runtime.GOOS == "windows" && !testBackend.runInWindows { continue @@ -48,7 +49,7 @@ func runTestForAllBackends(t *testing.T, test func(string, storage.Storage)) { os.Setenv(envVarName, envVarValue) } - storage, err := storage.InitStorage() + storage, err := storage.InitStorage(ctx) assert.Nil(t, err) test(backendType, storage) } diff --git a/cache-cli/cmd/store.go b/cache-cli/cmd/store.go index d9517cf7..94c0f36c 100644 --- a/cache-cli/cmd/store.go +++ b/cache-cli/cmd/store.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "os" "path/filepath" @@ -48,7 +49,7 @@ func RunStore(cmd *cobra.Command, args []string) { cleanupBy, err := cmd.Flags().GetString("cleanup-by") utils.Check(err) - storage, err := storage.InitStorageWithConfig(storage.StorageConfig{SortKeysBy: cleanupBy}) + storage, err := storage.InitStorageWithConfig(cmd.Context(), storage.StorageConfig{SortKeysBy: cleanupBy}) utils.Check(err) metricsManager, err := metrics.InitMetricsManager(metrics.LocalBackend) @@ -72,24 +73,24 @@ func RunStore(cmd *cobra.Command, args []string) { for _, entry := range lookupResult.Entries { log.Infof("Using default cache path '%s'.", entry.Path) key := entry.Keys[0] - compressAndStore(storage, archiver, key, entry.Path) + compressAndStore(cmd.Context(), storage, archiver, key, entry.Path) } } } else { path := filepath.FromSlash(args[1]) - compressAndStore(storage, archiver, args[0], path) + compressAndStore(cmd.Context(), storage, archiver, args[0], path) } } -func compressAndStore(storage storage.Storage, archiver archive.Archiver, rawKey, path string) { +func compressAndStore(ctx context.Context, storage storage.Storage, archiver archive.Archiver, rawKey, path string) { key := NormalizeKey(rawKey) if _, err := os.Stat(path); err == nil { - if ok, _ := storage.HasKey(key); ok { + if ok, _ := storage.HasKey(ctx, key); ok { log.Infof("Key '%s' already exists.", key) return } - compressedFilePath, compressedFileSize, err := compress(archiver, key, path) + compressedFilePath, compressedFileSize, err := compress(ctx, archiver, key, path) if err != nil { log.Errorf("Error compressing %s: %v", path, err) return @@ -103,7 +104,7 @@ func compressAndStore(storage storage.Storage, archiver archive.Archiver, rawKey uploadStart := time.Now() log.Infof("Uploading '%s' with cache key '%s'...", path, key) - err = storage.Store(key, compressedFilePath) + err = storage.Store(ctx, key, compressedFilePath) utils.Check(err) uploadDuration := time.Since(uploadStart) @@ -118,12 +119,12 @@ func compressAndStore(storage storage.Storage, archiver archive.Archiver, rawKey } } -func compress(archiver archive.Archiver, key, path string) (string, int64, error) { +func compress(ctx context.Context, archiver archive.Archiver, key, path string) (string, int64, error) { compressingStart := time.Now() log.Infof("Compressing %s...", path) dst := filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d", key, time.Now().Nanosecond())) - err := archiver.Compress(dst, path) + err := archiver.Compress(ctx, dst, path) utils.Check(err) compressionDuration := time.Since(compressingStart) diff --git a/cache-cli/cmd/store_test.go b/cache-cli/cmd/store_test.go index 172ef1cc..bbeeb828 100644 --- a/cache-cli/cmd/store_test.go +++ b/cache-cli/cmd/store_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "io/ioutil" "os" @@ -16,12 +17,13 @@ import ( ) func Test__Store(t *testing.T) { + ctx := context.TODO() storeCmd := NewStoreCommand() log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s wrong number of arguments", backend), func(t *testing.T) { RunStore(storeCmd, []string{"key", "value", "extra-bad-argument"}) output := readOutputFromFile(t) @@ -37,7 +39,7 @@ func Test__Store(t *testing.T) { }) t.Run(fmt.Sprintf("%s using key and valid path", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") ioutil.TempFile(tempDir, "*") @@ -49,7 +51,7 @@ func Test__Store(t *testing.T) { }) t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") ioutil.TempFile(tempDir, "*") @@ -62,7 +64,7 @@ func Test__Store(t *testing.T) { }) t.Run(fmt.Sprintf("%s using duplicate key", backend), func(*testing.T) { - storage.Clear() + storage.Clear(ctx) tempDir, _ := ioutil.TempDir(os.TempDir(), "*") ioutil.TempFile(tempDir, "*") @@ -81,6 +83,7 @@ func Test__Store(t *testing.T) { } func Test__AutomaticStore(t *testing.T) { + ctx := context.TODO() storeCmd := NewStoreCommand() _, file, _, _ := runtime.Caller(0) cmdPath := filepath.Dir(file) @@ -90,7 +93,7 @@ func Test__AutomaticStore(t *testing.T) { log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s nothing found", backend), func(t *testing.T) { os.Chdir(cmdPath) @@ -110,7 +113,7 @@ func Test__AutomaticStore(t *testing.T) { }) t.Run(fmt.Sprintf("%s detects and stores using SEMAPHORE_GIT_BRANCH", backend), func(t *testing.T) { - storage.Clear() + storage.Clear(ctx) os.Chdir(fmt.Sprintf("%s/test/autocache/gems", rootPath)) os.Setenv("SEMAPHORE_GIT_BRANCH", "master") @@ -132,7 +135,7 @@ func Test__AutomaticStore(t *testing.T) { }) t.Run(fmt.Sprintf("%s detects and stores using SEMAPHORE_GIT_PR_BRANCH", backend), func(t *testing.T) { - storage.Clear() + storage.Clear(ctx) os.Chdir(fmt.Sprintf("%s/test/autocache/gems", rootPath)) os.Setenv("SEMAPHORE_GIT_BRANCH", "master") @@ -154,7 +157,7 @@ func Test__AutomaticStore(t *testing.T) { }) t.Run(fmt.Sprintf("%s does not store if key already exist", backend), func(t *testing.T) { - storage.Clear() + storage.Clear(ctx) os.Chdir(fmt.Sprintf("%s/test/autocache/gems", rootPath)) os.Setenv("SEMAPHORE_GIT_BRANCH", "master") @@ -165,7 +168,7 @@ func Test__AutomaticStore(t *testing.T) { tempFile, _ := ioutil.TempFile(os.TempDir(), "*") key := fmt.Sprintf("gems-master-%s", checksum) - err := storage.Store(key, tempFile.Name()) + err := storage.Store(ctx, key, tempFile.Name()) assert.Nil(t, err) RunStore(storeCmd, []string{}) diff --git a/cache-cli/cmd/usage.go b/cache-cli/cmd/usage.go index 0e759d76..52b2ecdc 100644 --- a/cache-cli/cmd/usage.go +++ b/cache-cli/cmd/usage.go @@ -19,10 +19,10 @@ var usageCmd = &cobra.Command{ } func RunUsage(cmd *cobra.Command, args []string) { - storage, err := storage.InitStorage() + storage, err := storage.InitStorage(cmd.Context()) utils.Check(err) - summary, err := storage.Usage() + summary, err := storage.Usage(cmd.Context()) utils.Check(err) if summary.Free == -1 { diff --git a/cache-cli/cmd/usage_test.go b/cache-cli/cmd/usage_test.go index ead91fad..064e34d8 100644 --- a/cache-cli/cmd/usage_test.go +++ b/cache-cli/cmd/usage_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "fmt" "testing" @@ -11,13 +12,14 @@ import ( ) func Test__Usage(t *testing.T) { + ctx := context.TODO() log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) log.SetOutput(openLogfileForTests(t)) - runTestForAllBackends(t, func(backend string, storage storage.Storage) { + runTestForAllBackends(ctx, t, func(backend string, storage storage.Storage) { t.Run(fmt.Sprintf("%s empty cache", backend), func(t *testing.T) { - storage.Clear() + storage.Clear(ctx) RunUsage(usageCmd, []string{}) output := readOutputFromFile(t) diff --git a/cache-cli/main.go b/cache-cli/main.go index ad227ba3..0d473dd8 100644 --- a/cache-cli/main.go +++ b/cache-cli/main.go @@ -1,8 +1,10 @@ package main import ( + "context" "io" "os" + "os/signal" "path/filepath" "github.com/semaphoreci/toolbox/cache-cli/cmd" @@ -15,7 +17,10 @@ func main() { log.SetOutput(logfile) log.SetFormatter(new(logging.CustomFormatter)) log.SetLevel(log.InfoLevel) - cmd.Execute() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + cmd.RootCmd.ExecuteContext(ctx) } func OpenLogfile() io.Writer { diff --git a/cache-cli/pkg/archive/archiver.go b/cache-cli/pkg/archive/archiver.go index 197cf316..8419802a 100644 --- a/cache-cli/pkg/archive/archiver.go +++ b/cache-cli/pkg/archive/archiver.go @@ -1,14 +1,15 @@ package archive import ( + "context" "os" "github.com/semaphoreci/toolbox/cache-cli/pkg/metrics" ) type Archiver interface { - Compress(dst, src string) error - Decompress(src string) (string, error) + Compress(ctx context.Context, dst, src string) error + Decompress(ctx context.Context, src string) (string, error) } func NewArchiver(metricsManager metrics.MetricsManager) Archiver { diff --git a/cache-cli/pkg/archive/archiver_test.go b/cache-cli/pkg/archive/archiver_test.go index 4626789e..c34e7213 100644 --- a/cache-cli/pkg/archive/archiver_test.go +++ b/cache-cli/pkg/archive/archiver_test.go @@ -1,6 +1,7 @@ package archive import ( + "context" "fmt" "io/fs" "io/ioutil" @@ -15,14 +16,15 @@ import ( ) func Test__Compress(t *testing.T) { + ctx := context.TODO() runTestForAllArchiverTypes(t, false, func(archiverType string, archiver Archiver) { t.Run(archiverType+" file to compress is not present", func(t *testing.T) { - err := archiver.Compress("???", "/tmp/this-file-does-not-exist") + err := archiver.Compress(ctx, "???", "/tmp/this-file-does-not-exist") assert.NotNil(t, err) }) t.Run(archiverType+" file to decompress is not present", func(t *testing.T) { - _, err := archiver.Decompress("/tmp/this-file-does-not-exist") + _, err := archiver.Decompress(ctx, "/tmp/this-file-does-not-exist") assert.NotNil(t, err) }) @@ -31,7 +33,7 @@ func Test__Compress(t *testing.T) { tempFile, _ := ioutil.TempFile(tempDir, "*") _ = tempFile.Close() - assertCompressAndUnpack(t, archiver, tempDir, []fileAssertion{ + assertCompressAndUnpack(ctx, t, archiver, tempDir, []fileAssertion{ { name: tempFile.Name(), mode: fs.FileMode(0600), @@ -47,7 +49,7 @@ func Test__Compress(t *testing.T) { tempDirBase := filepath.Base(tempDir) _ = tempFile.Close() - assertCompressAndUnpack(t, archiver, tempDirBase, []fileAssertion{ + assertCompressAndUnpack(ctx, t, archiver, tempDirBase, []fileAssertion{ { name: tempFile.Name(), mode: fs.FileMode(0600), @@ -65,7 +67,7 @@ func Test__Compress(t *testing.T) { symlinkName := tempFile.Name() + "-link" assert.NoError(t, os.Symlink(tempFile.Name(), symlinkName)) - assertCompressAndUnpack(t, archiver, tempDirBase, []fileAssertion{ + assertCompressAndUnpack(ctx, t, archiver, tempDirBase, []fileAssertion{ { name: tempFile.Name(), mode: fs.FileMode(0600), @@ -87,7 +89,7 @@ func Test__Compress(t *testing.T) { assert.NoError(t, os.Chmod(tempFile.Name(), 0700)) tempDirBase := filepath.Base(tempDir) - assertCompressAndUnpack(t, archiver, tempDirBase, []fileAssertion{ + assertCompressAndUnpack(ctx, t, archiver, tempDirBase, []fileAssertion{ { name: tempFile.Name(), mode: fs.FileMode(0700), @@ -109,7 +111,7 @@ func Test__Compress(t *testing.T) { // compressing compressedFileName := tmpFileNameWithPrefix("abc0003") - assert.NoError(t, archiver.Compress(compressedFileName, tempDirBase)) + assert.NoError(t, archiver.Compress(ctx, compressedFileName, tempDirBase)) assert.Contains(t, compressedFileName, filepath.Join(os.TempDir(), "abc0003")) _, err := os.Stat(compressedFileName) assert.Nil(t, err) @@ -120,7 +122,7 @@ func Test__Compress(t *testing.T) { assert.NoError(t, os.RemoveAll(tempDirBase)) // unpacking - unpackedAt, err := archiver.Decompress(compressedFileName) + unpackedAt, err := archiver.Decompress(ctx, compressedFileName) assert.Nil(t, err) assert.Equal(t, tempDirBase+string(os.PathSeparator), unpackedAt) @@ -151,7 +153,7 @@ func Test__Compress(t *testing.T) { // compressing compressedFileName := tmpFileNameWithPrefix("abc0003") - assert.NoError(t, archiver.Compress(compressedFileName, tempDirBase)) + assert.NoError(t, archiver.Compress(ctx, compressedFileName, tempDirBase)) assert.Contains(t, compressedFileName, filepath.Join(os.TempDir(), "abc0003")) _, err := os.Stat(compressedFileName) assert.Nil(t, err) @@ -162,7 +164,7 @@ func Test__Compress(t *testing.T) { assert.NoError(t, os.RemoveAll(tempDirBase)) // unpacking - unpackedAt, err := archiver.Decompress(compressedFileName) + unpackedAt, err := archiver.Decompress(ctx, compressedFileName) if !assert.Nil(t, err) { return } @@ -195,7 +197,7 @@ func Test__Compress(t *testing.T) { // compressing compressedFileName := tmpFileNameWithPrefix("abc0003") - err := archiver.Compress(compressedFileName, tempFile.Name()) + err := archiver.Compress(ctx, compressedFileName, tempFile.Name()) assert.Nil(t, err) _, err = os.Stat(compressedFileName) @@ -203,7 +205,7 @@ func Test__Compress(t *testing.T) { assert.NoError(t, os.Remove(tempFile.Name())) // unpacking - unpackedAt, err := archiver.Decompress(compressedFileName) + unpackedAt, err := archiver.Decompress(ctx, compressedFileName) assert.Nil(t, err) assert.Equal(t, tempFile.Name(), unpackedAt) @@ -222,7 +224,7 @@ func Test__Compress(t *testing.T) { // compressing compressedFileName := tmpFileNameWithPrefix("abc0007") - err := archiver.Compress(compressedFileName, tempFile.Name()) + err := archiver.Compress(ctx, compressedFileName, tempFile.Name()) assert.Nil(t, err) // compressed file is created @@ -231,7 +233,7 @@ func Test__Compress(t *testing.T) { assert.NoError(t, os.RemoveAll(tempDir)) // unpacking - unpackedAt, err := archiver.Decompress(compressedFileName) + unpackedAt, err := archiver.Decompress(ctx, compressedFileName) assert.Nil(t, err) assert.Equal(t, tempFile.Name(), unpackedAt) @@ -253,7 +255,7 @@ func Test__Compress(t *testing.T) { // compressing compressedFileName := tmpFileNameWithPrefix("abc0007") - err := archiver.Compress(compressedFileName, tempFile.Name()) + err := archiver.Compress(ctx, compressedFileName, tempFile.Name()) assert.Nil(t, err) // compressed file is created @@ -262,7 +264,7 @@ func Test__Compress(t *testing.T) { assert.NoError(t, os.Remove(tempFile.Name())) // unpacking - unpackedAt, err := archiver.Decompress(compressedFileName) + unpackedAt, err := archiver.Decompress(ctx, compressedFileName) assert.Nil(t, err) assert.Equal(t, tempFile.Name(), unpackedAt) @@ -277,13 +279,14 @@ func Test__Compress(t *testing.T) { } func Test__Decompress(t *testing.T) { + ctx := context.TODO() runTestForAllArchiverTypes(t, true, func(archiverType string, archiver Archiver) { t.Run(archiverType+" sends metric on failure", func(t *testing.T) { tempFile, _ := ioutil.TempFile(os.TempDir(), "*") tempFile.WriteString("this is not a proper archive") _ = tempFile.Close() - _, err := archiver.Decompress(tempFile.Name()) + _, err := archiver.Decompress(ctx, tempFile.Name()) assert.NotNil(t, err) metricsFile := filepath.Join(os.TempDir(), "toolbox_metrics") @@ -307,10 +310,10 @@ type fileAssertion struct { symlink bool } -func assertCompressAndUnpack(t *testing.T, archiver Archiver, tempDirectory string, assertions []fileAssertion) { +func assertCompressAndUnpack(ctx context.Context, t *testing.T, archiver Archiver, tempDirectory string, assertions []fileAssertion) { // compressing compressedFileName := tmpFileNameWithPrefix("abc0003") - assert.NoError(t, archiver.Compress(compressedFileName, tempDirectory)) + assert.NoError(t, archiver.Compress(ctx, compressedFileName, tempDirectory)) assert.Contains(t, compressedFileName, filepath.Join(os.TempDir(), "abc0003")) _, err := os.Stat(compressedFileName) assert.Nil(t, err) @@ -319,7 +322,7 @@ func assertCompressAndUnpack(t *testing.T, archiver Archiver, tempDirectory stri assert.NoError(t, os.RemoveAll(tempDirectory)) // unpacking - unpackedAt, err := archiver.Decompress(compressedFileName) + unpackedAt, err := archiver.Decompress(ctx, compressedFileName) assert.Nil(t, err) assert.Equal(t, tempDirectory+string(os.PathSeparator), unpackedAt) diff --git a/cache-cli/pkg/archive/native_archiver.go b/cache-cli/pkg/archive/native_archiver.go index 38a778c4..bce14819 100644 --- a/cache-cli/pkg/archive/native_archiver.go +++ b/cache-cli/pkg/archive/native_archiver.go @@ -3,6 +3,7 @@ package archive import ( "archive/tar" "compress/gzip" + "context" "errors" "fmt" "io" @@ -28,7 +29,7 @@ func NewNativeArchiver(metricsManager metrics.MetricsManager, useParallelism boo } } -func (a *NativeArchiver) Compress(dst, src string) error { +func (a *NativeArchiver) Compress(ctx context.Context, dst, src string) error { if _, err := os.Stat(src); err != nil { return fmt.Errorf("error finding '%s': %v", src, err) } @@ -115,7 +116,7 @@ type directoryStat struct { mode fs.FileMode } -func (a *NativeArchiver) Decompress(src string) (string, error) { +func (a *NativeArchiver) Decompress(ctx context.Context, src string) (string, error) { // #nosec srcFile, err := os.Open(src) if err != nil { diff --git a/cache-cli/pkg/archive/shell_out_archiver.go b/cache-cli/pkg/archive/shell_out_archiver.go index a7bcefd5..60da904e 100644 --- a/cache-cli/pkg/archive/shell_out_archiver.go +++ b/cache-cli/pkg/archive/shell_out_archiver.go @@ -3,6 +3,7 @@ package archive import ( "archive/tar" "compress/gzip" + "context" "fmt" "io" "os" @@ -21,12 +22,12 @@ func NewShellOutArchiver(metricsManager metrics.MetricsManager) *ShellOutArchive return &ShellOutArchiver{metricsManager: metricsManager} } -func (a *ShellOutArchiver) Compress(dst, src string) error { +func (a *ShellOutArchiver) Compress(ctx context.Context, dst, src string) error { if _, err := os.Stat(src); err != nil { return fmt.Errorf("error finding '%s': %v", src, err) } - cmd := a.compressionCommand(dst, src) + cmd := a.compressionCommand(ctx, dst, src) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("error compressing %s: %s, %v", src, output, err) @@ -35,7 +36,7 @@ func (a *ShellOutArchiver) Compress(dst, src string) error { return nil } -func (a *ShellOutArchiver) Decompress(src string) (string, error) { +func (a *ShellOutArchiver) Decompress(ctx context.Context, src string) (string, error) { restorationPath, err := a.findRestorationPath(src) if err != nil { if metricErr := a.metricsManager.Publish(metrics.Metric{Name: metrics.CacheCorruptionRate, Value: "1"}); metricErr != nil { @@ -45,7 +46,7 @@ func (a *ShellOutArchiver) Decompress(src string) (string, error) { return "", fmt.Errorf("error finding restoration path: %v", err) } - cmd := a.decompressionCmd(restorationPath, src) + cmd := a.decompressionCmd(ctx, restorationPath, src) output, err := cmd.CombinedOutput() if err != nil { if metricErr := a.metricsManager.Publish(metrics.Metric{Name: metrics.CacheCorruptionRate, Value: "1"}); metricErr != nil { @@ -58,20 +59,20 @@ func (a *ShellOutArchiver) Decompress(src string) (string, error) { return restorationPath, nil } -func (a *ShellOutArchiver) compressionCommand(dst, src string) *exec.Cmd { +func (a *ShellOutArchiver) compressionCommand(ctx context.Context, dst, src string) *exec.Cmd { if filepath.IsAbs(src) { - return exec.Command("tar", "czPf", dst, src) + return exec.CommandContext(ctx, "tar", "czPf", dst, src) } - return exec.Command("tar", "czf", dst, src) + return exec.CommandContext(ctx, "tar", "czf", dst, src) } -func (a *ShellOutArchiver) decompressionCmd(dst, tempFile string) *exec.Cmd { +func (a *ShellOutArchiver) decompressionCmd(ctx context.Context, dst, tempFile string) *exec.Cmd { if filepath.IsAbs(dst) { - return exec.Command("tar", "xzPf", tempFile, "-C", ".") + return exec.CommandContext(ctx, "tar", "xzPf", tempFile, "-C", ".") } - return exec.Command("tar", "xzf", tempFile, "-C", ".") + return exec.CommandContext(ctx, "tar", "xzf", tempFile, "-C", ".") } func (a *ShellOutArchiver) findRestorationPath(src string) (string, error) { diff --git a/cache-cli/pkg/storage/clear_test.go b/cache-cli/pkg/storage/clear_test.go index b5dfdad6..ee245fc9 100644 --- a/cache-cli/pkg/storage/clear_test.go +++ b/cache-cli/pkg/storage/clear_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -10,9 +11,10 @@ import ( ) func Test__Clear(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { setup := func(storage Storage) []string { - _ = storage.Clear() + _ = storage.Clear(ctx) file1, _ := ioutil.TempFile(os.TempDir(), "*") file1.WriteString("something, something") @@ -20,8 +22,8 @@ func Test__Clear(t *testing.T) { file2, _ := ioutil.TempFile(os.TempDir(), "*") file2.WriteString("else, else") - _ = storage.Store("abc001", file1.Name()) - _ = storage.Store("abc002", file2.Name()) + _ = storage.Store(ctx, "abc001", file1.Name()) + _ = storage.Store(ctx, "abc002", file2.Name()) return []string{file1.Name(), file2.Name()} } @@ -33,28 +35,28 @@ func Test__Clear(t *testing.T) { } t.Run(fmt.Sprintf("%s no keys", storageType), func(t *testing.T) { - err := storage.Clear() + err := storage.Clear(ctx) assert.Nil(t, err) - keys, err := storage.List() + keys, err := storage.List(ctx) assert.Nil(t, err) assert.Len(t, keys, 0) - err = storage.Clear() + err = storage.Clear(ctx) assert.Nil(t, err) }) t.Run(fmt.Sprintf("%s with keys", storageType), func(t *testing.T) { filesToCleanup := setup(storage) - keys, err := storage.List() + keys, err := storage.List(ctx) assert.Nil(t, err) assert.Len(t, keys, 2) - err = storage.Clear() + err = storage.Clear(ctx) assert.Nil(t, err) - keys, err = storage.List() + keys, err = storage.List(ctx) assert.Nil(t, err) assert.Len(t, keys, 0) diff --git a/cache-cli/pkg/storage/delete_test.go b/cache-cli/pkg/storage/delete_test.go index e4a779f0..2be84fde 100644 --- a/cache-cli/pkg/storage/delete_test.go +++ b/cache-cli/pkg/storage/delete_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -10,27 +11,28 @@ import ( ) func Test__Delete(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s non-existing key", storageType), func(t *testing.T) { - _ = storage.Clear() - err := storage.Delete("this-key-does-not-exist") + _ = storage.Clear(ctx) + err := storage.Delete(ctx, "this-key-does-not-exist") assert.Nil(t, err) }) t.Run(fmt.Sprintf("%s existing key", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) file, _ := ioutil.TempFile(os.TempDir(), "*") - _ = storage.Store("abc001", file.Name()) + _ = storage.Store(ctx, "abc001", file.Name()) - keys, err := storage.List() + keys, err := storage.List(ctx) assert.Nil(t, err) assert.Len(t, keys, 1) - err = storage.Delete("abc001") + err = storage.Delete(ctx, "abc001") assert.Nil(t, err) - keys, err = storage.List() + keys, err = storage.List(ctx) assert.Nil(t, err) assert.Len(t, keys, 0) diff --git a/cache-cli/pkg/storage/gcs.go b/cache-cli/pkg/storage/gcs.go index 95ad7b51..8fe32c0f 100644 --- a/cache-cli/pkg/storage/gcs.go +++ b/cache-cli/pkg/storage/gcs.go @@ -19,12 +19,12 @@ type GCSStorageOptions struct { Config StorageConfig } -func NewGCSStorage(options GCSStorageOptions) (*GCSStorage, error) { - return createDefaultGCSStorage(options.Bucket, options.Project, options.Config) +func NewGCSStorage(ctx context.Context, options GCSStorageOptions) (*GCSStorage, error) { + return createDefaultGCSStorage(ctx, options.Bucket, options.Project, options.Config) } -func createDefaultGCSStorage(gcsBucket string, project string, storageConfig StorageConfig) (*GCSStorage, error) { - client, err := storage.NewClient(context.TODO()) +func createDefaultGCSStorage(ctx context.Context, gcsBucket string, project string, storageConfig StorageConfig) (*GCSStorage, error) { + client, err := storage.NewClient(ctx) if err != nil { return nil, err } diff --git a/cache-cli/pkg/storage/gcs_clear.go b/cache-cli/pkg/storage/gcs_clear.go index 1cf9f068..def08eab 100644 --- a/cache-cli/pkg/storage/gcs_clear.go +++ b/cache-cli/pkg/storage/gcs_clear.go @@ -7,8 +7,8 @@ import ( "google.golang.org/api/iterator" ) -func (s *GCSStorage) Clear() error { - it := s.Bucket.Objects(context.TODO(), &storage.Query{Prefix: s.Project}) +func (s *GCSStorage) Clear(ctx context.Context) error { + it := s.Bucket.Objects(ctx, &storage.Query{Prefix: s.Project}) for { attrs, err := it.Next() if err == iterator.Done { @@ -18,7 +18,7 @@ func (s *GCSStorage) Clear() error { return err } - err = s.Bucket.Object(attrs.Name).Delete(context.TODO()) + err = s.Bucket.Object(attrs.Name).Delete(ctx) if err != nil { return err } diff --git a/cache-cli/pkg/storage/gcs_delete.go b/cache-cli/pkg/storage/gcs_delete.go index 0fe05a10..3f3e5fdd 100644 --- a/cache-cli/pkg/storage/gcs_delete.go +++ b/cache-cli/pkg/storage/gcs_delete.go @@ -8,9 +8,9 @@ import ( gcs "cloud.google.com/go/storage" ) -func (s *GCSStorage) Delete(key string) error { +func (s *GCSStorage) Delete(ctx context.Context, key string) error { bucketKey := fmt.Sprintf("%s/%s", s.Project, key) - err := s.Bucket.Object(bucketKey).Delete(context.TODO()) + err := s.Bucket.Object(bucketKey).Delete(ctx) if errors.Is(err, gcs.ErrObjectNotExist) { return nil } diff --git a/cache-cli/pkg/storage/gcs_has_key.go b/cache-cli/pkg/storage/gcs_has_key.go index 607b7c2a..55b5ac68 100644 --- a/cache-cli/pkg/storage/gcs_has_key.go +++ b/cache-cli/pkg/storage/gcs_has_key.go @@ -8,9 +8,9 @@ import ( "cloud.google.com/go/storage" ) -func (s *GCSStorage) HasKey(key string) (bool, error) { +func (s *GCSStorage) HasKey(ctx context.Context, key string) (bool, error) { gcsKey := fmt.Sprintf("%s/%s", s.Project, key) - _, err := s.Bucket.Object(gcsKey).Attrs(context.TODO()) + _, err := s.Bucket.Object(gcsKey).Attrs(ctx) if err != nil { if errors.Is(err, storage.ErrObjectNotExist) { return false, nil diff --git a/cache-cli/pkg/storage/gcs_is_not_empty.go b/cache-cli/pkg/storage/gcs_is_not_empty.go index 5f42c886..f241f685 100644 --- a/cache-cli/pkg/storage/gcs_is_not_empty.go +++ b/cache-cli/pkg/storage/gcs_is_not_empty.go @@ -7,8 +7,8 @@ import ( "google.golang.org/api/iterator" ) -func (s *GCSStorage) IsNotEmpty() (bool, error) { - it := s.Bucket.Objects(context.TODO(), &storage.Query{Prefix: s.Project}) +func (s *GCSStorage) IsNotEmpty(ctx context.Context) (bool, error) { + it := s.Bucket.Objects(ctx, &storage.Query{Prefix: s.Project}) _, err := it.Next() if err == iterator.Done { diff --git a/cache-cli/pkg/storage/gcs_list.go b/cache-cli/pkg/storage/gcs_list.go index 019fc14e..68691bd3 100644 --- a/cache-cli/pkg/storage/gcs_list.go +++ b/cache-cli/pkg/storage/gcs_list.go @@ -10,8 +10,8 @@ import ( "google.golang.org/api/iterator" ) -func (s *GCSStorage) List() ([]CacheKey, error) { - it := s.Bucket.Objects(context.TODO(), &storage.Query{Prefix: s.Project}) +func (s *GCSStorage) List(ctx context.Context) ([]CacheKey, error) { + it := s.Bucket.Objects(ctx, &storage.Query{Prefix: s.Project}) keys := make([]CacheKey, 0) for { diff --git a/cache-cli/pkg/storage/gcs_restore.go b/cache-cli/pkg/storage/gcs_restore.go index 3f696a16..c1a1e94a 100644 --- a/cache-cli/pkg/storage/gcs_restore.go +++ b/cache-cli/pkg/storage/gcs_restore.go @@ -8,14 +8,14 @@ import ( "os" ) -func (s *GCSStorage) Restore(key string) (*os.File, error) { +func (s *GCSStorage) Restore(ctx context.Context, key string) (*os.File, error) { tempFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) if err != nil { return nil, err } bucketKey := fmt.Sprintf("%s/%s", s.Project, key) - reader, err := s.Bucket.Object(bucketKey).NewReader(context.TODO()) + reader, err := s.Bucket.Object(bucketKey).NewReader(ctx) if err != nil { _ = tempFile.Close() return nil, err diff --git a/cache-cli/pkg/storage/gcs_store.go b/cache-cli/pkg/storage/gcs_store.go index fe9b5810..e1319084 100644 --- a/cache-cli/pkg/storage/gcs_store.go +++ b/cache-cli/pkg/storage/gcs_store.go @@ -9,14 +9,14 @@ import ( log "github.com/sirupsen/logrus" ) -func (s *GCSStorage) Store(key, path string) error { +func (s *GCSStorage) Store(ctx context.Context, key, path string) error { // #nosec file, err := os.Open(path) if err != nil { return err } - ctx, cancel := context.WithCancel(context.TODO()) + ctx, cancel := context.WithCancel(ctx) defer cancel() destination := fmt.Sprintf("%s/%s", s.Project, key) diff --git a/cache-cli/pkg/storage/gcs_usage.go b/cache-cli/pkg/storage/gcs_usage.go index c7ad5a63..f053bce9 100644 --- a/cache-cli/pkg/storage/gcs_usage.go +++ b/cache-cli/pkg/storage/gcs_usage.go @@ -1,7 +1,9 @@ package storage -func (s *GCSStorage) Usage() (*UsageSummary, error) { - keys, err := s.List() +import "context" + +func (s *GCSStorage) Usage(ctx context.Context) (*UsageSummary, error) { + keys, err := s.List(ctx) if err != nil { return nil, err } diff --git a/cache-cli/pkg/storage/has_key_test.go b/cache-cli/pkg/storage/has_key_test.go index 42f32869..8d9b6622 100644 --- a/cache-cli/pkg/storage/has_key_test.go +++ b/cache-cli/pkg/storage/has_key_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -10,21 +11,22 @@ import ( ) func Test__HasKey(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s non-existing key", storageType), func(t *testing.T) { - _ = storage.Clear() - exists, err := storage.HasKey("this-key-does-not-exist") + _ = storage.Clear(ctx) + exists, err := storage.HasKey(ctx, "this-key-does-not-exist") assert.Nil(t, err) assert.False(t, exists) }) t.Run(fmt.Sprintf("%s existing key", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) file, _ := ioutil.TempFile(os.TempDir(), "*") - _ = storage.Store("abc001", file.Name()) + _ = storage.Store(ctx, "abc001", file.Name()) - exists, err := storage.HasKey("abc001") + exists, err := storage.HasKey(ctx, "abc001") assert.Nil(t, err) assert.True(t, exists) diff --git a/cache-cli/pkg/storage/is_not_empty_test.go b/cache-cli/pkg/storage/is_not_empty_test.go index 395b855e..8c95092c 100644 --- a/cache-cli/pkg/storage/is_not_empty_test.go +++ b/cache-cli/pkg/storage/is_not_empty_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -10,21 +11,22 @@ import ( ) func Test__IsNotEmpty(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s empty cache", storageType), func(t *testing.T) { - _ = storage.Clear() - isNotEmpty, err := storage.IsNotEmpty() + _ = storage.Clear(ctx) + isNotEmpty, err := storage.IsNotEmpty(ctx) assert.Nil(t, err) assert.False(t, isNotEmpty) }) t.Run(fmt.Sprintf("%s non-empty cache", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) file, _ := ioutil.TempFile(os.TempDir(), "*") - _ = storage.Store("abc001", file.Name()) + _ = storage.Store(ctx, "abc001", file.Name()) - isNotEmpty, err := storage.IsNotEmpty() + isNotEmpty, err := storage.IsNotEmpty(ctx) assert.Nil(t, err) assert.True(t, isNotEmpty) diff --git a/cache-cli/pkg/storage/list_test.go b/cache-cli/pkg/storage/list_test.go index 2fa7bad7..9e72035c 100644 --- a/cache-cli/pkg/storage/list_test.go +++ b/cache-cli/pkg/storage/list_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -12,29 +13,30 @@ import ( ) func Test__List(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s no keys", storageType), func(t *testing.T) { - _ = storage.Clear() - keys, err := storage.List() + _ = storage.Clear(ctx) + keys, err := storage.List(ctx) assert.Nil(t, err) assert.Len(t, keys, 0) }) t.Run(fmt.Sprintf("%s keys are ordered by store time", storageType), func(t *testing.T) { - err := storage.Clear() + err := storage.Clear(ctx) assert.Nil(t, err) file1, _ := ioutil.TempFile(os.TempDir(), "*") - err = storage.Store("abc001", file1.Name()) + err = storage.Store(ctx, "abc001", file1.Name()) assert.Nil(t, err) time.Sleep(time.Second) file2, _ := ioutil.TempFile(os.TempDir(), "*") - err = storage.Store("abc002", file2.Name()) + err = storage.Store(ctx, "abc002", file2.Name()) assert.Nil(t, err) - keys, err := storage.List() + keys, err := storage.List(ctx) assert.Nil(t, err) if assert.Len(t, keys, 2) { @@ -58,13 +60,13 @@ func Test__List(t *testing.T) { runTestForAllStorageTypes(t, SortBySize, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s keys are ordered by size", storageType), func(t *testing.T) { - err := storage.Clear() + err := storage.Clear(ctx) assert.Nil(t, err) biggerFile := fmt.Sprintf("%s/bigger.tmp", os.TempDir()) err = createBigTempFile(biggerFile, 100*1000*1000) // 100M assert.Nil(t, err) - err = storage.Store("bigger", biggerFile) + err = storage.Store(ctx, "bigger", biggerFile) assert.Nil(t, err) // Just to make sure things are really being sorted by size @@ -73,10 +75,10 @@ func Test__List(t *testing.T) { smallerFile := fmt.Sprintf("%s/smaller.tmp", os.TempDir()) err = createBigTempFile(smallerFile, 50*1000*1000) // 50M assert.Nil(t, err) - err = storage.Store("smaller", smallerFile) + err = storage.Store(ctx, "smaller", smallerFile) assert.Nil(t, err) - keys, err := storage.List() + keys, err := storage.List(ctx) assert.Nil(t, err) if assert.Len(t, keys, 2) { @@ -102,25 +104,25 @@ func Test__List(t *testing.T) { // s3 does not support access time sorting runTestForSingleStorageType("sftp", 1024, SortByAccessTime, t, func(storage Storage) { t.Run("sftp keys are ordered by access time", func(t *testing.T) { - err := storage.Clear() + err := storage.Clear(ctx) assert.Nil(t, err) // store first key tmpFile, _ := ioutil.TempFile(os.TempDir(), "*") - err = storage.Store("abc001", tmpFile.Name()) + err = storage.Store(ctx, "abc001", tmpFile.Name()) assert.Nil(t, err) // wait a little bit, and then store second key time.Sleep(2 * time.Second) - err = storage.Store("abc002", tmpFile.Name()) + err = storage.Store(ctx, "abc002", tmpFile.Name()) assert.Nil(t, err) // wait a little bit, and then restore first key (access time will be updated) time.Sleep(2 * time.Second) - _, err = storage.Restore("abc001") + _, err = storage.Restore(ctx, "abc001") assert.Nil(t, err) - keys, err := storage.List() + keys, err := storage.List(ctx) assert.Nil(t, err) if assert.Len(t, keys, 2) { diff --git a/cache-cli/pkg/storage/restore_test.go b/cache-cli/pkg/storage/restore_test.go index 2c943c4d..7b0cb85f 100644 --- a/cache-cli/pkg/storage/restore_test.go +++ b/cache-cli/pkg/storage/restore_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -10,17 +11,18 @@ import ( ) func Test__Restore(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s key exists", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) file, _ := ioutil.TempFile(os.TempDir(), "*") file.WriteString("restore - key exists") - err := storage.Store("abc001", file.Name()) + err := storage.Store(ctx, "abc001", file.Name()) assert.Nil(t, err) - restoredFile, err := storage.Restore("abc001") + restoredFile, err := storage.Restore(ctx, "abc001") assert.Nil(t, err) content, err := ioutil.ReadFile(restoredFile.Name()) @@ -32,9 +34,9 @@ func Test__Restore(t *testing.T) { }) t.Run(fmt.Sprintf("%s key does not exist", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) - _, err := storage.Restore("abc002") + _, err := storage.Restore(ctx, "abc002") assert.NotNil(t, err) }) }) diff --git a/cache-cli/pkg/storage/s3.go b/cache-cli/pkg/storage/s3.go index 47adf797..ca4f0231 100644 --- a/cache-cli/pkg/storage/s3.go +++ b/cache-cli/pkg/storage/s3.go @@ -26,15 +26,15 @@ type S3StorageOptions struct { Config StorageConfig } -func NewS3Storage(options S3StorageOptions) (*S3Storage, error) { +func NewS3Storage(ctx context.Context, options S3StorageOptions) (*S3Storage, error) { if options.URL != "" { - return createS3StorageUsingEndpoint(options.Bucket, options.Project, options.URL, options.Config) + return createS3StorageUsingEndpoint(ctx, options.Bucket, options.Project, options.URL, options.Config) } - return createDefaultS3Storage(options.Bucket, options.Project, options.Config) + return createDefaultS3Storage(ctx, options.Bucket, options.Project, options.Config) } -func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfig) (*S3Storage, error) { +func createDefaultS3Storage(ctx context.Context, s3Bucket, project string, storageConfig StorageConfig) (*S3Storage, error) { var config aws.Config var err error @@ -42,7 +42,7 @@ func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfi if os.Getenv("SEMAPHORE_CACHE_USE_EC2_INSTANCE_PROFILE") == "true" { log.Infof("Using EC2 instance profile.") config, err = awsConfig.LoadDefaultConfig( - context.TODO(), + ctx, awsConfig.WithCredentialsProvider(ec2rolecreds.New()), awsConfig.WithEC2IMDSRegion(), ) @@ -63,7 +63,7 @@ func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfi profile := os.Getenv("SEMAPHORE_CACHE_AWS_PROFILE") if profile != "" { log.Infof("Using '%s' AWS profile.", profile) - config, err = awsConfig.LoadDefaultConfig(context.TODO(), awsConfig.WithSharedConfigProfile(profile)) + config, err = awsConfig.LoadDefaultConfig(ctx, awsConfig.WithSharedConfigProfile(profile)) if err != nil { return nil, err } @@ -77,7 +77,7 @@ func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfi } // No special configuration, just follow the default chain - config, err = awsConfig.LoadDefaultConfig(context.TODO()) + config, err = awsConfig.LoadDefaultConfig(ctx) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfi }, nil } -func createS3StorageUsingEndpoint(s3Bucket, project, s3Url string, storageConfig StorageConfig) (*S3Storage, error) { +func createS3StorageUsingEndpoint(ctx context.Context, s3Bucket, project, s3Url string, storageConfig StorageConfig) (*S3Storage, error) { resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { return aws.Endpoint{ URL: s3Url, @@ -98,7 +98,7 @@ func createS3StorageUsingEndpoint(s3Bucket, project, s3Url string, storageConfig }) creds := credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "") - cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), + cfg, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithCredentialsProvider(creds), awsConfig.WithEndpointResolver(resolver), ) diff --git a/cache-cli/pkg/storage/s3_clear.go b/cache-cli/pkg/storage/s3_clear.go index f5e8b7b5..814aadeb 100644 --- a/cache-cli/pkg/storage/s3_clear.go +++ b/cache-cli/pkg/storage/s3_clear.go @@ -8,8 +8,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" ) -func (s *S3Storage) Clear() error { - keys, err := s.List() +func (s *S3Storage) Clear(ctx context.Context) error { + keys, err := s.List(ctx) if err != nil { return err } @@ -22,7 +22,7 @@ func (s *S3Storage) Clear() error { chunks := createChunks(keys, 1000) for _, chunk := range chunks { - err := s.deleteChunk(chunk) + err := s.deleteChunk(ctx, chunk) if err != nil { return err } @@ -31,8 +31,8 @@ func (s *S3Storage) Clear() error { return nil } -func (s *S3Storage) deleteChunk(keys []CacheKey) error { - output, err := s.Client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ +func (s *S3Storage) deleteChunk(ctx context.Context, keys []CacheKey) error { + output, err := s.Client.DeleteObjects(ctx, &s3.DeleteObjectsInput{ Bucket: &s.Bucket, Delete: &types.Delete{ Objects: cacheKeysToObjectIdentifiers(s.Project, keys), diff --git a/cache-cli/pkg/storage/s3_delete.go b/cache-cli/pkg/storage/s3_delete.go index a03885bd..4c72cb31 100644 --- a/cache-cli/pkg/storage/s3_delete.go +++ b/cache-cli/pkg/storage/s3_delete.go @@ -7,10 +7,10 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) -func (s *S3Storage) Delete(key string) error { +func (s *S3Storage) Delete(ctx context.Context, key string) error { bucketKey := fmt.Sprintf("%s/%s", s.Project, key) - _, err := s.Client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + _, err := s.Client.DeleteObject(ctx, &s3.DeleteObjectInput{ Bucket: &s.Bucket, Key: &bucketKey, }) diff --git a/cache-cli/pkg/storage/s3_has_key.go b/cache-cli/pkg/storage/s3_has_key.go index 19e6d50c..6b2f7db7 100644 --- a/cache-cli/pkg/storage/s3_has_key.go +++ b/cache-cli/pkg/storage/s3_has_key.go @@ -9,14 +9,14 @@ import ( "github.com/aws/smithy-go" ) -func (s *S3Storage) HasKey(key string) (bool, error) { +func (s *S3Storage) HasKey(ctx context.Context, key string) (bool, error) { s3Key := fmt.Sprintf("%s/%s", s.Project, key) input := s3.HeadObjectInput{ Bucket: &s.Bucket, Key: &s3Key, } - _, err := s.Client.HeadObject(context.TODO(), &input) + _, err := s.Client.HeadObject(ctx, &input) if err != nil { var apiErr *smithy.GenericAPIError if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotFound" { diff --git a/cache-cli/pkg/storage/s3_is_not_empty.go b/cache-cli/pkg/storage/s3_is_not_empty.go index 14da6e33..418270ac 100644 --- a/cache-cli/pkg/storage/s3_is_not_empty.go +++ b/cache-cli/pkg/storage/s3_is_not_empty.go @@ -1,7 +1,9 @@ package storage -func (s *S3Storage) IsNotEmpty() (bool, error) { - keys, err := s.List() +import "context" + +func (s *S3Storage) IsNotEmpty(ctx context.Context) (bool, error) { + keys, err := s.List(ctx) if err != nil { return false, err } diff --git a/cache-cli/pkg/storage/s3_list.go b/cache-cli/pkg/storage/s3_list.go index b0af5129..0aad8ec9 100644 --- a/cache-cli/pkg/storage/s3_list.go +++ b/cache-cli/pkg/storage/s3_list.go @@ -10,8 +10,8 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" ) -func (s *S3Storage) List() ([]CacheKey, error) { - output, err := s.Client.ListObjects(context.TODO(), s.listObjectsInput(nil)) +func (s *S3Storage) List(ctx context.Context) ([]CacheKey, error) { + output, err := s.Client.ListObjects(ctx, s.listObjectsInput(nil)) if err != nil { return nil, err } @@ -21,7 +21,7 @@ func (s *S3Storage) List() ([]CacheKey, error) { for output.IsTruncated { nextMarker := findNextMarker(output) - output, err = s.Client.ListObjects(context.TODO(), s.listObjectsInput(&nextMarker)) + output, err = s.Client.ListObjects(ctx, s.listObjectsInput(&nextMarker)) if err != nil { return nil, err } diff --git a/cache-cli/pkg/storage/s3_restore.go b/cache-cli/pkg/storage/s3_restore.go index 37938bf4..e7c93cec 100644 --- a/cache-cli/pkg/storage/s3_restore.go +++ b/cache-cli/pkg/storage/s3_restore.go @@ -10,7 +10,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" ) -func (s *S3Storage) Restore(key string) (*os.File, error) { +func (s *S3Storage) Restore(ctx context.Context, key string) (*os.File, error) { tempFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) if err != nil { return nil, err @@ -18,7 +18,7 @@ func (s *S3Storage) Restore(key string) (*os.File, error) { bucketKey := fmt.Sprintf("%s/%s", s.Project, key) downloader := manager.NewDownloader(s.Client) - _, err = downloader.Download(context.TODO(), tempFile, &s3.GetObjectInput{ + _, err = downloader.Download(ctx, tempFile, &s3.GetObjectInput{ Bucket: &s.Bucket, Key: &bucketKey, }) diff --git a/cache-cli/pkg/storage/s3_store.go b/cache-cli/pkg/storage/s3_store.go index 9180f73c..345ac123 100644 --- a/cache-cli/pkg/storage/s3_store.go +++ b/cache-cli/pkg/storage/s3_store.go @@ -10,7 +10,7 @@ import ( log "github.com/sirupsen/logrus" ) -func (s *S3Storage) Store(key, path string) error { +func (s *S3Storage) Store(ctx context.Context, key, path string) error { // #nosec file, err := os.Open(path) if err != nil { @@ -19,7 +19,7 @@ func (s *S3Storage) Store(key, path string) error { destination := fmt.Sprintf("%s/%s", s.Project, key) uploader := manager.NewUploader(s.Client) - _, err = uploader.Upload(context.TODO(), &s3.PutObjectInput{ + _, err = uploader.Upload(ctx, &s3.PutObjectInput{ Bucket: &s.Bucket, Key: &destination, Body: file, diff --git a/cache-cli/pkg/storage/s3_usage.go b/cache-cli/pkg/storage/s3_usage.go index cc5eca24..0b3a04e6 100644 --- a/cache-cli/pkg/storage/s3_usage.go +++ b/cache-cli/pkg/storage/s3_usage.go @@ -1,7 +1,9 @@ package storage -func (s *S3Storage) Usage() (*UsageSummary, error) { - keys, err := s.List() +import "context" + +func (s *S3Storage) Usage(ctx context.Context) (*UsageSummary, error) { + keys, err := s.List(ctx) if err != nil { return nil, err } diff --git a/cache-cli/pkg/storage/sftp.go b/cache-cli/pkg/storage/sftp.go index e6812336..0547c098 100644 --- a/cache-cli/pkg/storage/sftp.go +++ b/cache-cli/pkg/storage/sftp.go @@ -1,6 +1,7 @@ package storage import ( + "context" "io/ioutil" "os" "strings" @@ -24,7 +25,7 @@ type SFTPStorageOptions struct { Config StorageConfig } -func NewSFTPStorage(options SFTPStorageOptions) (*SFTPStorage, error) { +func NewSFTPStorage(ctx context.Context, options SFTPStorageOptions) (*SFTPStorage, error) { sshClient, err := createSSHClient(options) if err != nil { return nil, err diff --git a/cache-cli/pkg/storage/sftp_clear.go b/cache-cli/pkg/storage/sftp_clear.go index eedf4630..bd49e1ea 100644 --- a/cache-cli/pkg/storage/sftp_clear.go +++ b/cache-cli/pkg/storage/sftp_clear.go @@ -1,7 +1,9 @@ package storage -func (s *SFTPStorage) Clear() error { - keys, err := s.List() +import "context" + +func (s *SFTPStorage) Clear(ctx context.Context) error { + keys, err := s.List(ctx) if err != nil { return err } diff --git a/cache-cli/pkg/storage/sftp_delete.go b/cache-cli/pkg/storage/sftp_delete.go index ca0de0eb..d54a7fd5 100644 --- a/cache-cli/pkg/storage/sftp_delete.go +++ b/cache-cli/pkg/storage/sftp_delete.go @@ -1,8 +1,11 @@ package storage -import "strings" +import ( + "context" + "strings" +) -func (s *SFTPStorage) Delete(key string) error { +func (s *SFTPStorage) Delete(ctx context.Context, key string) error { err := s.SFTPClient.Remove(key) if err != nil && strings.Contains(err.Error(), "file does not exist") { return nil diff --git a/cache-cli/pkg/storage/sftp_has_key.go b/cache-cli/pkg/storage/sftp_has_key.go index 109a03f8..44bda706 100644 --- a/cache-cli/pkg/storage/sftp_has_key.go +++ b/cache-cli/pkg/storage/sftp_has_key.go @@ -1,8 +1,11 @@ package storage -import "strings" +import ( + "context" + "strings" +) -func (s *SFTPStorage) HasKey(key string) (bool, error) { +func (s *SFTPStorage) HasKey(ctx context.Context, key string) (bool, error) { file, err := s.SFTPClient.Stat(key) if file == nil { if err != nil && strings.Contains(err.Error(), "file does not exist") { diff --git a/cache-cli/pkg/storage/sftp_is_not_empty.go b/cache-cli/pkg/storage/sftp_is_not_empty.go index bf2d8f71..247c0686 100644 --- a/cache-cli/pkg/storage/sftp_is_not_empty.go +++ b/cache-cli/pkg/storage/sftp_is_not_empty.go @@ -1,7 +1,9 @@ package storage -func (s *SFTPStorage) IsNotEmpty() (bool, error) { - keys, err := s.List() +import "context" + +func (s *SFTPStorage) IsNotEmpty(ctx context.Context) (bool, error) { + keys, err := s.List(ctx) if err != nil { return false, err } diff --git a/cache-cli/pkg/storage/sftp_list.go b/cache-cli/pkg/storage/sftp_list.go index 84bab757..28aee431 100644 --- a/cache-cli/pkg/storage/sftp_list.go +++ b/cache-cli/pkg/storage/sftp_list.go @@ -1,6 +1,7 @@ package storage import ( + "context" "io/fs" "sort" "time" @@ -8,7 +9,7 @@ import ( "github.com/pkg/sftp" ) -func (s *SFTPStorage) List() ([]CacheKey, error) { +func (s *SFTPStorage) List(ctx context.Context) ([]CacheKey, error) { files, err := s.SFTPClient.ReadDir(".") if err != nil { return nil, err diff --git a/cache-cli/pkg/storage/sftp_restore.go b/cache-cli/pkg/storage/sftp_restore.go index d037b805..353d6b7b 100644 --- a/cache-cli/pkg/storage/sftp_restore.go +++ b/cache-cli/pkg/storage/sftp_restore.go @@ -1,12 +1,13 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" ) -func (s *SFTPStorage) Restore(key string) (*os.File, error) { +func (s *SFTPStorage) Restore(ctx context.Context, key string) (*os.File, error) { localFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) if err != nil { return nil, err diff --git a/cache-cli/pkg/storage/sftp_store.go b/cache-cli/pkg/storage/sftp_store.go index 0c07008e..4c47e9aa 100644 --- a/cache-cli/pkg/storage/sftp_store.go +++ b/cache-cli/pkg/storage/sftp_store.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "os" "time" @@ -8,7 +9,7 @@ import ( log "github.com/sirupsen/logrus" ) -func (s *SFTPStorage) Store(key, path string) error { +func (s *SFTPStorage) Store(ctx context.Context, key, path string) error { epochNanos := time.Now().UnixNano() tmpKey := fmt.Sprintf("%s-%d", os.Getenv("SEMAPHORE_JOB_ID"), epochNanos) @@ -17,7 +18,7 @@ func (s *SFTPStorage) Store(key, path string) error { return err } - err = s.allocateSpace(localFileInfo.Size()) + err = s.allocateSpace(ctx, localFileInfo.Size()) if err != nil { return err } @@ -66,8 +67,8 @@ func (s *SFTPStorage) Store(key, path string) error { return localFile.Close() } -func (s *SFTPStorage) allocateSpace(space int64) error { - usage, err := s.Usage() +func (s *SFTPStorage) allocateSpace(ctx context.Context, space int64) error { + usage, err := s.Usage(ctx) if err != nil { return err } @@ -75,14 +76,14 @@ func (s *SFTPStorage) allocateSpace(space int64) error { freeSpace := usage.Free if freeSpace < space { fmt.Printf("Not enough space, deleting keys based on %s...\n", s.Config().SortKeysBy) - keys, err := s.List() + keys, err := s.List(ctx) if err != nil { return err } for freeSpace < space { lastKey := keys[len(keys)-1] - err = s.Delete(lastKey.Name) + err = s.Delete(ctx, lastKey.Name) if err != nil { return err } diff --git a/cache-cli/pkg/storage/sftp_usage.go b/cache-cli/pkg/storage/sftp_usage.go index ef9a23cc..4dfe651e 100644 --- a/cache-cli/pkg/storage/sftp_usage.go +++ b/cache-cli/pkg/storage/sftp_usage.go @@ -1,6 +1,8 @@ package storage -func (s *SFTPStorage) Usage() (*UsageSummary, error) { +import "context" + +func (s *SFTPStorage) Usage(ctx context.Context) (*UsageSummary, error) { files, err := s.SFTPClient.ReadDir(".") if err != nil { return nil, err diff --git a/cache-cli/pkg/storage/storage.go b/cache-cli/pkg/storage/storage.go index 135d23b0..08a0b853 100644 --- a/cache-cli/pkg/storage/storage.go +++ b/cache-cli/pkg/storage/storage.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "math" "os" @@ -11,14 +12,14 @@ import ( ) type Storage interface { - List() ([]CacheKey, error) - HasKey(key string) (bool, error) - Store(key, path string) error - Restore(key string) (*os.File, error) - Delete(key string) error - Clear() error - Usage() (*UsageSummary, error) - IsNotEmpty() (bool, error) + List(ctx context.Context) ([]CacheKey, error) + HasKey(ctx context.Context, key string) (bool, error) + Store(ctx context.Context, key, path string) error + Restore(ctx context.Context, key string) (*os.File, error) + Delete(ctx context.Context, key string) error + Clear(ctx context.Context) error + Usage(ctx context.Context) (*UsageSummary, error) + IsNotEmpty(ctx context.Context) (bool, error) Config() StorageConfig } @@ -53,11 +54,11 @@ type UsageSummary struct { Used int64 } -func InitStorage() (Storage, error) { - return InitStorageWithConfig(StorageConfig{SortKeysBy: SortByStoreTime}) +func InitStorage(ctx context.Context) (Storage, error) { + return InitStorageWithConfig(ctx, StorageConfig{SortKeysBy: SortByStoreTime}) } -func InitStorageWithConfig(config StorageConfig) (Storage, error) { +func InitStorageWithConfig(ctx context.Context, config StorageConfig) (Storage, error) { err := config.Validate() if err != nil { return nil, err @@ -80,7 +81,7 @@ func InitStorageWithConfig(config StorageConfig) (Storage, error) { return nil, fmt.Errorf("no SEMAPHORE_CACHE_S3_BUCKET set") } - return NewS3Storage(S3StorageOptions{ + return NewS3Storage(ctx, S3StorageOptions{ URL: os.Getenv("SEMAPHORE_CACHE_S3_URL"), Bucket: s3Bucket, Project: project, @@ -102,7 +103,7 @@ func InitStorageWithConfig(config StorageConfig) (Storage, error) { return nil, fmt.Errorf("no SEMAPHORE_CACHE_PRIVATE_KEY_PATH set") } - return NewSFTPStorage(SFTPStorageOptions{ + return NewSFTPStorage(ctx, SFTPStorageOptions{ URL: url, Username: username, PrivateKeyPath: privateKeyPath, @@ -119,7 +120,7 @@ func InitStorageWithConfig(config StorageConfig) (Storage, error) { return nil, fmt.Errorf("no SEMAPHORE_CACHE_GCS_BUCKET set") } - return NewGCSStorage(GCSStorageOptions{ + return NewGCSStorage(ctx, GCSStorageOptions{ Bucket: gcsBucket, Project: project, Config: StorageConfig{MaxSpace: math.MaxInt64, SortKeysBy: config.SortKeysBy}, diff --git a/cache-cli/pkg/storage/storage_test.go b/cache-cli/pkg/storage/storage_test.go index 8b60ce1c..63274292 100644 --- a/cache-cli/pkg/storage/storage_test.go +++ b/cache-cli/pkg/storage/storage_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "math" "os" "runtime" @@ -18,7 +19,7 @@ var testStorageTypes = map[string]TestStorageType{ "s3": { runInWindows: true, initializer: func(storageSize int64, sortBy string) (Storage, error) { - return NewS3Storage(S3StorageOptions{ + return NewS3Storage(context.TODO(), S3StorageOptions{ URL: os.Getenv("SEMAPHORE_CACHE_S3_URL"), Bucket: "semaphore-cache", Project: "cache-cli", @@ -29,7 +30,7 @@ var testStorageTypes = map[string]TestStorageType{ "sftp": { runInWindows: false, initializer: func(storageSize int64, sortBy string) (Storage, error) { - return NewSFTPStorage(SFTPStorageOptions{ + return NewSFTPStorage(context.TODO(), SFTPStorageOptions{ URL: "sftp-server:22", Username: "tester", PrivateKeyPath: "/root/.ssh/semaphore_cache_key", @@ -40,7 +41,7 @@ var testStorageTypes = map[string]TestStorageType{ "gcs": { runInWindows: false, initializer: func(storageSize int64, sortBy string) (Storage, error) { - return NewGCSStorage(GCSStorageOptions{ + return NewGCSStorage(context.TODO(), GCSStorageOptions{ Bucket: "semaphore-cache", Project: "cache-cli", Config: StorageConfig{MaxSpace: math.MaxInt64, SortKeysBy: sortBy}, diff --git a/cache-cli/pkg/storage/store_test.go b/cache-cli/pkg/storage/store_test.go index ee31b122..1c5a8202 100644 --- a/cache-cli/pkg/storage/store_test.go +++ b/cache-cli/pkg/storage/store_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -15,15 +16,16 @@ import ( ) func Test__Store(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s stored objects can be listed", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) file, _ := ioutil.TempFile(os.TempDir(), "*") - err := storage.Store("abc001", file.Name()) + err := storage.Store(ctx, "abc001", file.Name()) assert.Nil(t, err) - keys, err := storage.List() + keys, err := storage.List(ctx) assert.Nil(t, err) if assert.Len(t, keys, 1) { @@ -37,15 +39,15 @@ func Test__Store(t *testing.T) { }) t.Run(fmt.Sprintf("%s stored objects can be restored", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) file, _ := ioutil.TempFile(os.TempDir(), "*") file.WriteString("stored objects can be restored") - err := storage.Store("abc002", file.Name()) + err := storage.Store(ctx, "abc002", file.Name()) assert.Nil(t, err) - restoredFile, err := storage.Restore("abc002") + restoredFile, err := storage.Restore(ctx, "abc002") assert.Nil(t, err) content, err := ioutil.ReadFile(restoredFile.Name()) @@ -70,7 +72,7 @@ func Test__Store(t *testing.T) { t.Skip() } - _ = storage.Clear() + _ = storage.Clear(ctx) smallerFile := fmt.Sprintf("%s/smaller.tmp", os.TempDir()) err := createBigTempFile(smallerFile, 300*1000*1000) // 300M @@ -82,12 +84,12 @@ func Test__Store(t *testing.T) { assert.Nil(t, err) go func() { - _ = storage.Store("abc003", smallerFile) + _ = storage.Store(ctx, "abc003", smallerFile) }() - _ = storage.Store("abc003", biggerFile) + _ = storage.Store(ctx, "abc003", biggerFile) - restoredFile, err := storage.Restore("abc003") + restoredFile, err := storage.Restore(ctx, "abc003") assert.Nil(t, err) assert.Zero(t, countLines(restoredFile.Name(), smallerFile)) @@ -100,22 +102,22 @@ func Test__Store(t *testing.T) { if runtime.GOOS != "windows" { runTestForSingleStorageType("sftp", 1024, SortByStoreTime, t, func(storage Storage) { t.Run("sftp least recently stored keys are deleted when no space", func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) // store first key tmpFile, _ := ioutil.TempFile(os.TempDir(), "*") tmpFile.WriteString(strings.Repeat("x", 400)) - storage.Store("abc001", tmpFile.Name()) + storage.Store(ctx, "abc001", tmpFile.Name()) // wait a little bit, then store second key time.Sleep(2 * time.Second) - storage.Store("abc002", tmpFile.Name()) + storage.Store(ctx, "abc002", tmpFile.Name()) // wait a little bit, then store third key time.Sleep(2 * time.Second) - storage.Store("abc003", tmpFile.Name()) + storage.Store(ctx, "abc003", tmpFile.Name()) - keys, _ := storage.List() + keys, _ := storage.List(ctx) assert.Len(t, keys, 2) firstKey := keys[0] @@ -129,26 +131,26 @@ func Test__Store(t *testing.T) { runTestForSingleStorageType("sftp", 1024, SortByAccessTime, t, func(storage Storage) { t.Run("sftp least recently accessed keys are deleted when no space", func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) // store first key tmpFile, _ := ioutil.TempFile(os.TempDir(), "*") tmpFile.WriteString(strings.Repeat("x", 400)) - storage.Store("abc001", tmpFile.Name()) + storage.Store(ctx, "abc001", tmpFile.Name()) // wait a little bit, then store second key time.Sleep(2 * time.Second) - storage.Store("abc002", tmpFile.Name()) + storage.Store(ctx, "abc002", tmpFile.Name()) // wait a little bit, then restore first key (access time will be updated) time.Sleep(2 * time.Second) - storage.Store("abc001", tmpFile.Name()) + storage.Store(ctx, "abc001", tmpFile.Name()) // wait a little bit, then store third key time.Sleep(2 * time.Second) - storage.Store("abc003", tmpFile.Name()) + storage.Store(ctx, "abc003", tmpFile.Name()) - keys, _ := storage.List() + keys, _ := storage.List(ctx) assert.Len(t, keys, 2) firstKey := keys[0] @@ -162,7 +164,7 @@ func Test__Store(t *testing.T) { runTestForSingleStorageType("sftp", 150*1024*1024, SortBySize, t, func(storage Storage) { t.Run("sftp smaller keys are deleted when no space", func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) smallerFile := fmt.Sprintf("%s/smaller.tmp", os.TempDir()) err := createBigTempFile(smallerFile, 50*1000*1000) // 50M @@ -173,13 +175,13 @@ func Test__Store(t *testing.T) { assert.Nil(t, err) // store both keys - storage.Store("smaller-key", smallerFile) - storage.Store("bigger-key", biggerFile) + storage.Store(ctx, "smaller-key", smallerFile) + storage.Store(ctx, "bigger-key", biggerFile) // store third key, this should cleanup the smaller key - storage.Store("another-smaller-key", smallerFile) + storage.Store(ctx, "another-smaller-key", smallerFile) - keys, _ := storage.List() + keys, _ := storage.List(ctx) assert.Len(t, keys, 2) firstKey := keys[0] diff --git a/cache-cli/pkg/storage/usage_test.go b/cache-cli/pkg/storage/usage_test.go index 92d2c434..1d3d08e3 100644 --- a/cache-cli/pkg/storage/usage_test.go +++ b/cache-cli/pkg/storage/usage_test.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "io/ioutil" "os" @@ -10,10 +11,11 @@ import ( ) func Test__Usage(t *testing.T) { + ctx := context.TODO() runTestForAllStorageTypes(t, SortByStoreTime, func(storageType string, storage Storage) { t.Run(fmt.Sprintf("%s no usage", storageType), func(t *testing.T) { - _ = storage.Clear() - usage, err := storage.Usage() + _ = storage.Clear(ctx) + usage, err := storage.Usage(ctx) assert.Nil(t, err) assert.Equal(t, int64(0), usage.Used) @@ -26,14 +28,14 @@ func Test__Usage(t *testing.T) { }) t.Run(fmt.Sprintf("%s some usage", storageType), func(t *testing.T) { - _ = storage.Clear() + _ = storage.Clear(ctx) fileContents := "usage - some usage" file, _ := ioutil.TempFile(os.TempDir(), "*") file.WriteString(fileContents) - _ = storage.Store("abc001", file.Name()) + _ = storage.Store(ctx, "abc001", file.Name()) - usage, err := storage.Usage() + usage, err := storage.Usage(ctx) assert.Nil(t, err) assert.Equal(t, int64(len(fileContents)), usage.Used)