diff --git a/cmd/add.go b/cmd/add.go index 28c3444..0448c6e 100644 --- a/cmd/add.go +++ b/cmd/add.go @@ -4,6 +4,9 @@ package cmd import ( + "fmt" + "os" + "github.com/cisco-open/grabit/internal" "github.com/spf13/cobra" ) @@ -18,6 +21,7 @@ func addAdd(cmd *cobra.Command) { addCmd.Flags().String("algo", internal.RecommendedAlgo, "Integrity algorithm") addCmd.Flags().String("filename", "", "Target file name to use when downloading the resource") addCmd.Flags().StringArray("tag", []string{}, "Resource tags") + addCmd.Flags().String("artifactory-cache-url", "", "Artifactory cache URL") cmd.AddCommand(addCmd) } @@ -26,6 +30,17 @@ func runAdd(cmd *cobra.Command, args []string) error { if err != nil { return err } + // Get cache URL + ArtifactoryCacheURL, err := cmd.Flags().GetString("artifactory-cache-url") + if err != nil { + return err + } + if ArtifactoryCacheURL != "" { + token := os.Getenv(internal.GRABIT_ARTIFACTORY_TOKEN_ENV_VAR) + if token == "" { + return fmt.Errorf("%s environment variable is not set", internal.GRABIT_ARTIFACTORY_TOKEN_ENV_VAR) + } + } lock, err := internal.NewLock(lockFile, true) if err != nil { return err @@ -42,7 +57,7 @@ func runAdd(cmd *cobra.Command, args []string) error { if err != nil { return err } - err = lock.AddResource(args, algo, tags, filename) + err = lock.AddResource(args, algo, tags, filename, ArtifactoryCacheURL) if err != nil { return err } diff --git a/cmd/add_test.go b/cmd/add_test.go index a7ba460..cdb97a6 100644 --- a/cmd/add_test.go +++ b/cmd/add_test.go @@ -2,24 +2,27 @@ package cmd import ( "fmt" - "net/http" "testing" + "github.com/cisco-open/grabit/internal" "github.com/cisco-open/grabit/test" "github.com/stretchr/testify/assert" ) func TestRunAdd(t *testing.T) { - handler := func(w http.ResponseWriter, r *http.Request) { - _, err := w.Write([]byte(`abcdef`)) - if err != nil { - t.Fatal(err) - } - } - port, server := test.HttpHandler(handler) - defer server.Close() + port := test.TestHttpHandler("abcdef", t) cmd := NewRootCmd() cmd.SetArgs([]string{"-f", test.TmpFile(t, ""), "add", fmt.Sprintf("http://localhost:%d/test.html", port)}) err := cmd.Execute() assert.Nil(t, err) } + +func TestRunAddWithArtifactoryCache(t *testing.T) { + t.Setenv(internal.GRABIT_ARTIFACTORY_TOKEN_ENV_VAR, "artifactory-token") + port := test.TestHttpHandler("abcdef", t) + artPort := test.TestHttpHandler("abcdef", t) + cmd := NewRootCmd() + cmd.SetArgs([]string{"-f", test.TmpFile(t, ""), "add", fmt.Sprintf("http://localhost:%d/test.html", port), "--artifactory-cache-url", fmt.Sprintf("http://localhost:%d/artifactory", artPort)}) + err := cmd.Execute() + assert.Nil(t, err) +} diff --git a/cmd/delete.go b/cmd/delete.go index 27c5722..769bd45 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -28,7 +28,10 @@ func runDel(cmd *cobra.Command, args []string) error { return err } for _, r := range args { - lock.DeleteResource(r) + err = lock.DeleteResource(r) + if err != nil { + return err + } } err = lock.Save() if err != nil { diff --git a/cmd/delete_test.go b/cmd/delete_test.go index 7f312be..496820c 100644 --- a/cmd/delete_test.go +++ b/cmd/delete_test.go @@ -15,6 +15,7 @@ func TestRunDelete(t *testing.T) { Tags = ['tag1', 'tag2'] `) cmd := NewRootCmd() + cmd.Flags().String("cache", "", "Artifactory URL for caching") cmd.SetArgs([]string{"-f", testfilepath, "delete", "http://localhost:123456/test.html"}) err := cmd.Execute() assert.Nil(t, err) diff --git a/internal/lock.go b/internal/lock.go index f525507..66a5120 100644 --- a/internal/lock.go +++ b/internal/lock.go @@ -49,28 +49,36 @@ func NewLock(path string, newOk bool) (*Lock, error) { return &Lock{path: path, conf: conf}, nil } -func (l *Lock) AddResource(paths []string, algo string, tags []string, filename string) error { +func (l *Lock) AddResource(paths []string, algo string, tags []string, filename string, cacheURL string) error { for _, u := range paths { if l.Contains(u) { return fmt.Errorf("resource '%s' is already present", u) } } - r, err := NewResourceFromUrl(paths, algo, tags, filename) + r, err := NewResourceFromUrl(paths, algo, tags, filename, cacheURL) + if err != nil { return err } + l.conf.Resource = append(l.conf.Resource, *r) return nil } -func (l *Lock) DeleteResource(path string) { +func (l *Lock) DeleteResource(path string) error { newStatements := []Resource{} for _, r := range l.conf.Resource { if !r.Contains(path) { newStatements = append(newStatements, r) + } else { + err := r.Delete() + if err != nil { + return fmt.Errorf("Failed to delete resource '%s': %w", path, err) + } } } l.conf.Resource = newStatements + return nil } const NoFileMode = os.FileMode(0) diff --git a/internal/lock_test.go b/internal/lock_test.go index eb6c810..11a92f7 100644 --- a/internal/lock_test.go +++ b/internal/lock_test.go @@ -51,12 +51,13 @@ func TestLockManipulations(t *testing.T) { port, server := test.HttpHandler(handler) defer server.Close() resource := fmt.Sprintf("http://localhost:%d/test2.html", port) - err = lock.AddResource([]string{resource}, "sha512", []string{}, "") + err = lock.AddResource([]string{resource}, "sha512", []string{}, "", "") assert.Nil(t, err) assert.Equal(t, 2, len(lock.conf.Resource)) err = lock.Save() assert.Nil(t, err) - lock.DeleteResource(resource) + err = lock.DeleteResource(resource) + assert.Nil(t, err) assert.Equal(t, 1, len(lock.conf.Resource)) } @@ -68,7 +69,7 @@ func TestDuplicateResource(t *testing.T) { Integrity = 'sha256-asdasdasd'`, url)) lock, err := NewLock(path, false) assert.Nil(t, err) - err = lock.AddResource([]string{url}, "sha512", []string{}, "") + err = lock.AddResource([]string{url}, "sha512", []string{}, "", "") assert.NotNil(t, err) assert.Contains(t, err.Error(), "already present") } diff --git a/internal/resource.go b/internal/resource.go index d40993e..dbe5c3d 100644 --- a/internal/resource.go +++ b/internal/resource.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "net/http" "net/url" "os" "path" @@ -20,19 +21,26 @@ import ( // Resource represents an external resource to be downloaded. type Resource struct { - Urls []string - Integrity string - Tags []string `toml:",omitempty"` - Filename string `toml:",omitempty"` + Urls []string + Integrity string + Tags []string `toml:",omitempty"` + Filename string `toml:",omitempty"` + ArtifactoryCacheURL string `toml:",omitempty"` } -func NewResourceFromUrl(urls []string, algo string, tags []string, filename string) (*Resource, error) { +const GRABIT_ARTIFACTORY_TOKEN_ENV_VAR = "GRABIT_ARTIFACTORY_TOKEN" + +func getArtifactoryToken() string { + return os.Getenv(GRABIT_ARTIFACTORY_TOKEN_ENV_VAR) +} + +func NewResourceFromUrl(urls []string, algo string, tags []string, filename string, ArtifactoryCacheURL string) (*Resource, error) { if len(urls) < 1 { return nil, fmt.Errorf("empty url list") } url := urls[0] ctx := context.Background() - path, err := GetUrltoTempFile(url, ctx) + path, err := GetUrltoTempFile(url, "", ctx) if err != nil { return nil, fmt.Errorf("failed to get url: %s", err) } @@ -41,53 +49,136 @@ func NewResourceFromUrl(urls []string, algo string, tags []string, filename stri if err != nil { return nil, fmt.Errorf("failed to compute ressource integrity: %s", err) } - return &Resource{Urls: urls, Integrity: integrity, Tags: tags, Filename: filename}, nil + resource := &Resource{Urls: urls, Integrity: integrity, Tags: tags, Filename: filename, ArtifactoryCacheURL: ArtifactoryCacheURL} + // If cache URL is provided, upload file to Artifactory. + if resource.ArtifactoryCacheURL != "" { + err := resource.AddToCache(path) + if err != nil { + return nil, fmt.Errorf("failed to upload to cache: %s", err) + } + } + return resource, nil +} + +func (l *Resource) getCacheFullURL() string { + url, err := url.JoinPath(l.ArtifactoryCacheURL, l.Integrity) + if err != nil { + log.Fatal().Err(err) + } + return url +} + +func (l *Resource) AddToCache(filePath string) error { + token := getArtifactoryToken() + if token == "" { + return fmt.Errorf("%s environment variable is not set and is needed to upload to cache", GRABIT_ARTIFACTORY_TOKEN_ENV_VAR) + } + + err := requests. + URL(l.getCacheFullURL()). + Method(http.MethodPut). + Header("Authorization", fmt.Sprintf("Bearer %s", token)). + BodyFile(filePath). + Fetch(context.Background()) + if err != nil { + return fmt.Errorf("failed to upload to cache: %v", err) + } + return nil +} + +func (l *Resource) Delete() error { + token := getArtifactoryToken() + if token == "" { + log.Warn().Msgf("%s environment variable is not set and is needed to delete the file from the cache", GRABIT_ARTIFACTORY_TOKEN_ENV_VAR) + return nil + } + url := l.getCacheFullURL() + err := requests. + URL(url). + Method(http.MethodDelete). + Header("Authorization", fmt.Sprintf("Bearer %s", token)). + Fetch(context.Background()) + log.Warn().Msgf("Error deleting file from cache (%s): %v", url, err) + return nil } // getUrl downloads the given resource and returns the path to it. -func getUrl(u string, fileName string, ctx context.Context) (string, error) { +func getUrl(u string, fileName string, bearer string, ctx context.Context) (string, error) { _, err := url.Parse(u) if err != nil { return "", fmt.Errorf("invalid url '%s': %s", u, err) } log.Debug().Str("URL", u).Msg("Downloading") - err = requests. + + req := requests. URL(u). Header("Accept", "*/*"). - ToFile(fileName). - Fetch(ctx) + ToFile(fileName) + + if bearer != "" { + req.Header("Authorization", fmt.Sprintf("Bearer %s", bearer)) + } + + err = req.Fetch(ctx) if err != nil { return "", fmt.Errorf("failed to download '%s': %s", u, err) } - log.Debug().Str("URL", u).Msg("Downloaded") + return fileName, nil } // GetUrlToDir downloads the given resource to the given directory and returns the path to it. -func GetUrlToDir(u string, targetDir string, ctx context.Context) (string, error) { +func GetUrlToDir(u string, targetDir string, bearer string, ctx context.Context) (string, error) { // create temporary name in the target directory. h := sha256.New() h.Write([]byte(u)) fileName := filepath.Join(targetDir, fmt.Sprintf(".%s", hex.EncodeToString(h.Sum(nil)))) - return getUrl(u, fileName, ctx) + return getUrl(u, fileName, bearer, ctx) } // GetUrlWithDir downloads the given resource to a temporary file and returns the path to it. -func GetUrltoTempFile(u string, ctx context.Context) (string, error) { +func GetUrltoTempFile(u string, bearer string, ctx context.Context) (string, error) { file, err := os.CreateTemp("", "prefix") if err != nil { log.Fatal().Err(err) } fileName := file.Name() - return getUrl(u, fileName, ctx) + return getUrl(u, fileName, "", ctx) } func (l *Resource) Download(dir string, mode os.FileMode, ctx context.Context) error { - ok := false algo, err := getAlgoFromIntegrity(l.Integrity) if err != nil { return err } + // Check if a cache URL exists to use Artifactory first. + if l.ArtifactoryCacheURL != "" { + token := getArtifactoryToken() + if token != "" { + artifactoryURL := fmt.Sprintf("%s/%s", l.ArtifactoryCacheURL, l.Integrity) + localName := l.Filename + if localName == "" { + localName = path.Base(l.Urls[0]) + } + resPath := filepath.Join(dir, localName) + + tmpPath, err := getUrl(artifactoryURL, resPath, token, ctx) + if err == nil { + if mode != NoFileMode { + err = os.Chmod(tmpPath, mode.Perm()) + if err != nil { + return fmt.Errorf("error changing target file permission: '%v'", err) + } + } + err = checkIntegrityFromFile(resPath, algo, l.Integrity, artifactoryURL) + if err != nil { + return fmt.Errorf("cache file at '%s' with incorrect integrity: '%v'", artifactoryURL, err) + } + } + log.Warn().Msgf("Failed to download from Artifactory cache, falling back to original URL: %v\n", err) + } + } + ok := false var downloadError error = nil for _, u := range l.Urls { @@ -121,7 +212,7 @@ func (l *Resource) Download(dir string, mode os.FileMode, ctx context.Context) e // Download file in the target directory so that the call to // os.Rename is atomic. - lpath, err := GetUrlToDir(u, dir, ctx) + lpath, err := GetUrlToDir(u, dir, "", ctx) if err != nil { downloadError = err continue @@ -137,15 +228,19 @@ func (l *Resource) Download(dir string, mode os.FileMode, ctx context.Context) e if mode != NoFileMode { err = os.Chmod(resPath, mode.Perm()) if err != nil { - return err + return fmt.Errorf("error changing target file permission: '%v'", err) } } ok = true break } if !ok { - if downloadError != nil { - return downloadError + if err == nil { + if downloadError != nil { + return downloadError + } else { + panic("no error but no file downloaded") + } } return err } diff --git a/internal/resource_test.go b/internal/resource_test.go index 5e9b69e..f05c73d 100644 --- a/internal/resource_test.go +++ b/internal/resource_test.go @@ -13,6 +13,7 @@ import ( "github.com/cisco-open/grabit/test" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNewResourceFromUrl(t *testing.T) { @@ -34,7 +35,7 @@ func TestNewResourceFromUrl(t *testing.T) { { urls: []string{fmt.Sprintf("http://localhost:%d/test.html", port)}, valid: true, - res: Resource{Urls: []string{fmt.Sprintf("http://localhost:%d/test.html", port)}, Integrity: fmt.Sprintf("%s-vvV+x/U6bUC+tkCngKY5yDvCmsipgW8fxsXG3Nk8RyE=", algo), Tags: []string{}, Filename: ""}, + res: Resource{Urls: []string{fmt.Sprintf("http://localhost:%d/test.html", port)}, Integrity: fmt.Sprintf("%s-vvV+x/U6bUC+tkCngKY5yDvCmsipgW8fxsXG3Nk8RyE=", algo), Tags: []string{}, Filename: "", ArtifactoryCacheURL: ""}, }, { urls: []string{"invalid url"}, @@ -42,9 +43,8 @@ func TestNewResourceFromUrl(t *testing.T) { errorContains: "failed to download", }, } - for _, data := range tests { - resource, err := NewResourceFromUrl(data.urls, algo, []string{}, "") + resource, err := NewResourceFromUrl(data.urls, algo, []string{}, "", "") assert.Equal(t, data.valid, err == nil) if err != nil { assert.Contains(t, err.Error(), data.errorContains) @@ -84,3 +84,87 @@ func TestResourceDownloadWithInValidFileAlreadyPresent(t *testing.T) { assert.Contains(t, err.Error(), "integrity mismatch") assert.Contains(t, err.Error(), "existing file") } + +func TestUseResourceWithCache(t *testing.T) { + content := `abcdef` + token := "test-token" + port, server := test.TestHttpHandlerWithServer(content, t) + fileName := "test.txt" + sourceURL := fmt.Sprintf("http://localhost:%d", port) + + artServer, artPort := test.NewRecorderHttpServer(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + _, err := w.Write([]byte(content)) + if err != nil { + t.Fatal(err) + } + } + }, t) + baseCacheURL := fmt.Sprintf("http://localhost:%d/", artPort) + + // Create resource, download it, upload it to cache. + t.Setenv(GRABIT_ARTIFACTORY_TOKEN_ENV_VAR, token) + resource, err := NewResourceFromUrl([]string{sourceURL}, "sha256", []string{}, fileName, baseCacheURL) + require.Nil(t, err) + server.Close() // Close origin server: file will be served from cache. + outputDir := test.TmpDir(t) + + // Download resource from cache. + err = resource.Download(outputDir, 0644, context.Background()) + require.Nil(t, err) + for _, file := range []string{"test.txt"} { + test.AssertFileContains(t, fmt.Sprintf("%s/%s", outputDir, file), content) + } + assert.Equal(t, 2, len(*artServer.Requests)) + assert.Equal(t, "PUT", (*artServer.Requests)[0].Method) + assert.Equal(t, []byte(content), (*artServer.Requests)[0].Body) + assert.Equal(t, []string([]string{fmt.Sprintf("Bearer %s", token)}), (*artServer.Requests)[0].Headers["Authorization"]) + assert.Equal(t, "GET", (*artServer.Requests)[1].Method) + + // Delete resource, deleting it from cache. + err = resource.Delete() + require.Nil(t, err) + assert.Equal(t, 3, len(*artServer.Requests)) + assert.Equal(t, "DELETE", (*artServer.Requests)[2].Method) + assert.Equal(t, []string([]string{fmt.Sprintf("Bearer %s", token)}), (*artServer.Requests)[2].Headers["Authorization"]) +} + +func TestResourceWithCacheCorruptedCache(t *testing.T) { + content := `abcdef` + token := "test-token" + port, server := test.TestHttpHandlerWithServer(content, t) + fileName := "test.txt" + sourceURL := fmt.Sprintf("http://localhost:%d", port) + + _, artPort := test.NewRecorderHttpServer(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + _, err := w.Write([]byte("invalid-content")) + if err != nil { + t.Fatal(err) + } + } + }, t) + baseCacheURL := fmt.Sprintf("http://localhost:%d/", artPort) + + // Create resource, download it, upload it to cache. + t.Setenv(GRABIT_ARTIFACTORY_TOKEN_ENV_VAR, token) + resource, err := NewResourceFromUrl([]string{sourceURL}, "sha256", []string{}, fileName, baseCacheURL) + require.Nil(t, err) + server.Close() // Close origin server: file will be served from cache. + outputDir := test.TmpDir(t) + + // Download resource from cache. + err = resource.Download(outputDir, 0644, context.Background()) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "cache file at") + assert.Contains(t, err.Error(), "with incorrect integrity") +} + +func TestResourceWithCacheNoToken(t *testing.T) { + t.Setenv(GRABIT_ARTIFACTORY_TOKEN_ENV_VAR, "") + fileName := "test.txt" + port := 33 + sourceURL := fmt.Sprintf("http://localhost:%d", port) + _, err := NewResourceFromUrl([]string{sourceURL}, "sha256", []string{}, fileName, "http://localhost:8080/") + assert.NotNil(t, err) +} diff --git a/test/utils.go b/test/utils.go index 3653641..5fcb1de 100644 --- a/test/utils.go +++ b/test/utils.go @@ -4,9 +4,11 @@ package test import ( + "bytes" "crypto/sha256" "encoding/base64" "fmt" + "io" "log" "net" "net/http" @@ -58,9 +60,51 @@ func HttpHandler(handler http.HandlerFunc) (int, *httptest.Server) { return l.Addr().(*net.TCPAddr).Port, s } +type RecordedRequest struct { + Method string + Url string + Body []byte + Headers map[string][]string +} + +func NewRecordedRequest(r *http.Request) *RecordedRequest { + var bodyBytes []byte + if r.Body != nil { + bodyBytes, _ = io.ReadAll(r.Body) + } + r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + return &RecordedRequest{ + Method: r.Method, + Url: r.URL.String(), + Body: bodyBytes, + Headers: r.Header} +} + +type RecorderHttpServer struct { + *httptest.Server + Requests *[]RecordedRequest +} + +func NewRecorderHttpServer(handler http.HandlerFunc, t *testing.T) (*RecorderHttpServer, int) { + requests := make([]RecordedRequest, 0) + + outerHandler := func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, *NewRecordedRequest(r)) + handler(w, r) + } + port, server := HttpHandler(outerHandler) + t.Cleanup(func() { server.Close() }) + return &RecorderHttpServer{Server: server, Requests: &requests}, port +} + // TestHttpHandler creates a new HTTP server and returns the port and serves // the given content. Its lifetime is tied to the given testing.T object. func TestHttpHandler(content string, t *testing.T) int { + port, _ := TestHttpHandlerWithServer(content, t) + return port +} + +func TestHttpHandlerWithServer(content string, t *testing.T) (int, *httptest.Server) { handler := func(w http.ResponseWriter, r *http.Request) { _, err := w.Write([]byte(content)) if err != nil { @@ -69,7 +113,7 @@ func TestHttpHandler(content string, t *testing.T) int { } port, server := HttpHandler(handler) t.Cleanup(func() { server.Close() }) - return port + return port, server } // AssertFileContains asserts that the file at the given path exists and