From 8a2231969c11420b7ceaddb53b56ccd98517c9d2 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Fri, 5 Feb 2021 11:45:59 +0800 Subject: [PATCH] Add a proper logger to fetch (#90) --- .gitignore | 3 -- checksum.go | 6 ++-- checksum_test.go | 10 +++--- file.go | 6 +++- file_test.go | 19 ++++++---- github.go | 5 +-- github_test.go | 6 ++-- go.mod | 5 +++ go.sum | 38 ++++++++++++++++++++ integration_test.go | 67 +++++++--------------------------- logger.go | 22 ++++++++++++ main.go | 88 ++++++++++++++++++++++++++++++--------------- main_test.go | 6 ++-- 13 files changed, 175 insertions(+), 106 deletions(-) create mode 100644 logger.go diff --git a/.gitignore b/.gitignore index 6ccdf3d..d28c001 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,3 @@ fetch fetch.exe bin/* - -# Don't upload dependencies -vendor diff --git a/checksum.go b/checksum.go index df82c48..ae175d1 100644 --- a/checksum.go +++ b/checksum.go @@ -9,9 +9,11 @@ import ( "io" "os" "reflect" + + "github.com/sirupsen/logrus" ) -func verifyChecksumOfReleaseAsset(assetPath string, checksumMap map[string]bool, algorithm string) *FetchError { +func verifyChecksumOfReleaseAsset(logger *logrus.Logger, assetPath string, checksumMap map[string]bool, algorithm string) *FetchError { computedChecksum, err := computeChecksum(assetPath, algorithm) if err != nil { return newError(errorWhileComputingChecksum, err.Error()) @@ -20,7 +22,7 @@ func verifyChecksumOfReleaseAsset(assetPath string, checksumMap map[string]bool, keys := reflect.ValueOf(checksumMap).MapKeys() return newError(checksumDoesNotMatch, fmt.Sprintf("Expected to checksum value to be one of %s, but instead got %s for Release Asset at %s. This means that either you are using the wrong checksum value in your call to fetch, (e.g. did you update the version of the module you're installing but not the checksum?) or that someone has replaced the asset with a potentially dangerous one and you should be very careful about proceeding.", keys, computedChecksum, assetPath)) } - fmt.Printf("Release asset checksum verified for %s\n", assetPath) + logger.Infof("Release asset checksum verified for %s\n", assetPath) return nil } diff --git a/checksum_test.go b/checksum_test.go index 82627ad..2c3d346 100644 --- a/checksum_test.go +++ b/checksum_test.go @@ -27,6 +27,7 @@ var SAMPLE_RELEASE_ASSET_CHECKSUMS_SHA256_NO_MATCH = map[string]bool{ func TestVerifyReleaseAsset(t *testing.T) { tmpDir := mkTempDir(t) + logger := GetProjectLogger() testInst := GitHubInstance{ BaseUrl: "github.com", ApiUrl: "api.github.com", @@ -37,7 +38,7 @@ func TestVerifyReleaseAsset(t *testing.T) { t.Fatalf("Failed to parse sample release asset GitHub URL into Fetch GitHubRepo struct: %s", err) } - assetPaths, fetchErr := downloadReleaseAssets(SAMPLE_RELEASE_ASSET_NAME, tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) + assetPaths, fetchErr := downloadReleaseAssets(logger, SAMPLE_RELEASE_ASSET_NAME, tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) if fetchErr != nil { t.Fatalf("Failed to download release asset: %s", fetchErr) } @@ -62,6 +63,7 @@ func TestVerifyReleaseAsset(t *testing.T) { func TestVerifyChecksumOfReleaseAsset(t *testing.T) { tmpDir := mkTempDir(t) + logger := GetProjectLogger() testInst := GitHubInstance{ BaseUrl: "github.com", ApiUrl: "api.github.com", @@ -72,7 +74,7 @@ func TestVerifyChecksumOfReleaseAsset(t *testing.T) { t.Fatalf("Failed to parse sample release asset GitHub URL into Fetch GitHubRepo struct: %s", err) } - assetPaths, fetchErr := downloadReleaseAssets(SAMPLE_RELEASE_ASSET_REGEX, tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) + assetPaths, fetchErr := downloadReleaseAssets(logger, SAMPLE_RELEASE_ASSET_REGEX, tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) if fetchErr != nil { t.Fatalf("Failed to download release asset: %s", fetchErr) } @@ -82,14 +84,14 @@ func TestVerifyChecksumOfReleaseAsset(t *testing.T) { } for _, assetPath := range assetPaths { - checksumErr := verifyChecksumOfReleaseAsset(assetPath, SAMPLE_RELEASE_ASSET_CHECKSUMS_SHA256, "sha256") + checksumErr := verifyChecksumOfReleaseAsset(logger, assetPath, SAMPLE_RELEASE_ASSET_CHECKSUMS_SHA256, "sha256") if checksumErr != nil { t.Fatalf("Expected downloaded asset to match one of %d checksums: %s", len(SAMPLE_RELEASE_ASSET_CHECKSUMS_SHA256), checksumErr) } } for _, assetPath := range assetPaths { - checksumErr := verifyChecksumOfReleaseAsset(assetPath, SAMPLE_RELEASE_ASSET_CHECKSUMS_SHA256_NO_MATCH, "sha256") + checksumErr := verifyChecksumOfReleaseAsset(logger, assetPath, SAMPLE_RELEASE_ASSET_CHECKSUMS_SHA256_NO_MATCH, "sha256") if checksumErr == nil { t.Fatalf("Expected downloaded asset to not match any checksums") } diff --git a/file.go b/file.go index 48b0f39..8662797 100644 --- a/file.go +++ b/file.go @@ -9,12 +9,14 @@ import ( "os" "path/filepath" "strings" + + "github.com/sirupsen/logrus" ) // Download the zip file at the given URL to a temporary local directory. // Returns the absolute path to the downloaded zip file. // IMPORTANT: You must call "defer os.RemoveAll(dir)" in the calling function when done with the downloaded zip file! -func downloadGithubZipFile(gitHubCommit GitHubCommit, gitHubToken string, instance GitHubInstance) (string, *FetchError) { +func downloadGithubZipFile(logger *logrus.Logger, gitHubCommit GitHubCommit, gitHubToken string, instance GitHubInstance) (string, *FetchError) { var zipFilePath string @@ -33,6 +35,7 @@ func downloadGithubZipFile(gitHubCommit GitHubCommit, gitHubToken string, instan return zipFilePath, wrapError(err) } + logger.Debugf("Performing HTTP request to download GitHub ZIP Archive: %s", req.URL) resp, err := httpClient.Do(req) if err != nil { return zipFilePath, wrapError(err) @@ -51,6 +54,7 @@ func downloadGithubZipFile(gitHubCommit GitHubCommit, gitHubToken string, instan return zipFilePath, wrapError(err) } + logger.Debugf("Writing ZIP Archive to temporary path: %s", tempDir) err = ioutil.WriteFile(filepath.Join(tempDir, "repo.zip"), respBodyBuffer.Bytes(), 0644) if err != nil { return zipFilePath, wrapError(err) diff --git a/file_test.go b/file_test.go index c04e7b6..efa9555 100644 --- a/file_test.go +++ b/file_test.go @@ -44,6 +44,7 @@ func TestDownloadGitTagZipFile(t *testing.T) { } for _, tc := range cases { + logger := GetProjectLogger() gitHubCommits := []GitHubCommit{ // Test as a GitTag GitHubCommit{ @@ -63,7 +64,7 @@ func TestDownloadGitTagZipFile(t *testing.T) { }, } for _, gitHubCommit := range gitHubCommits { - zipFilePath, err := downloadGithubZipFile(gitHubCommit, tc.githubToken, tc.instance) + zipFilePath, err := downloadGithubZipFile(logger, gitHubCommit, tc.githubToken, tc.instance) defer os.RemoveAll(zipFilePath) @@ -118,6 +119,7 @@ func TestDownloadGitBranchZipFile(t *testing.T) { } for _, tc := range cases { + logger := GetProjectLogger() gitHubCommits := []GitHubCommit{ GitHubCommit{ Repo: GitHubRepo{ @@ -135,7 +137,7 @@ func TestDownloadGitBranchZipFile(t *testing.T) { }, } for _, gitHubCommit := range gitHubCommits { - zipFilePath, err := downloadGithubZipFile(gitHubCommit, tc.githubToken, tc.instance) + zipFilePath, err := downloadGithubZipFile(logger, gitHubCommit, tc.githubToken, tc.instance) defer os.RemoveAll(zipFilePath) if err != nil { t.Fatalf("Failed to download file: %s", err) @@ -167,6 +169,7 @@ func TestDownloadBadGitBranchZipFile(t *testing.T) { } for _, tc := range cases { + logger := GetProjectLogger() gitHubCommits := []GitHubCommit{ GitHubCommit{ Repo: GitHubRepo{ @@ -184,7 +187,7 @@ func TestDownloadBadGitBranchZipFile(t *testing.T) { }, } for _, gitHubCommit := range gitHubCommits { - zipFilePath, err := downloadGithubZipFile(gitHubCommit, tc.githubToken, tc.instance) + zipFilePath, err := downloadGithubZipFile(logger, gitHubCommit, tc.githubToken, tc.instance) defer os.RemoveAll(zipFilePath) if err == nil { t.Fatalf("Expected that attempt to download repo %s/%s for branch \"%s\" would fail, but received no error.", tc.repoOwner, tc.repoName, tc.branchName) @@ -215,6 +218,7 @@ func TestDownloadGitCommitFile(t *testing.T) { } for _, tc := range cases { + logger := GetProjectLogger() GitHubCommits := []GitHubCommit{ GitHubCommit{ Repo: GitHubRepo{ @@ -232,7 +236,7 @@ func TestDownloadGitCommitFile(t *testing.T) { }, } for _, gitHubCommit := range GitHubCommits { - zipFilePath, err := downloadGithubZipFile(gitHubCommit, tc.githubToken, tc.instance) + zipFilePath, err := downloadGithubZipFile(logger, gitHubCommit, tc.githubToken, tc.instance) defer os.RemoveAll(zipFilePath) if err != nil { t.Fatalf("Failed to download file: %s", err) @@ -269,7 +273,7 @@ func TestDownloadBadGitCommitFile(t *testing.T) { } for _, tc := range cases { - + logger := GetProjectLogger() gitHubCommits := []GitHubCommit{ GitHubCommit{ Repo: GitHubRepo{ @@ -287,7 +291,7 @@ func TestDownloadBadGitCommitFile(t *testing.T) { }, } for _, gitHubCommit := range gitHubCommits { - zipFilePath, err := downloadGithubZipFile(gitHubCommit, tc.githubToken, tc.instance) + zipFilePath, err := downloadGithubZipFile(logger, gitHubCommit, tc.githubToken, tc.instance) defer os.RemoveAll(zipFilePath) if err == nil { t.Fatalf("Expected that attempt to download repo %s/%s at commmit sha \"%s\" would fail, but received no error.", tc.repoOwner, tc.repoName, tc.commitSha) @@ -315,6 +319,7 @@ func TestDownloadZipFileWithBadRepoValues(t *testing.T) { } for _, tc := range cases { + logger := GetProjectLogger() gitHubCommits := []GitHubCommit{ GitHubCommit{ Repo: GitHubRepo{ @@ -333,7 +338,7 @@ func TestDownloadZipFileWithBadRepoValues(t *testing.T) { } for _, gitHubCommit := range gitHubCommits { - _, err := downloadGithubZipFile(gitHubCommit, tc.githubToken, tc.instance) + _, err := downloadGithubZipFile(logger, gitHubCommit, tc.githubToken, tc.instance) if err == nil && err.errorCode != 500 { t.Fatalf("Expected error for bad repo values: %s/%s:%s", tc.repoOwner, tc.repoName, tc.gitTag) } diff --git a/github.go b/github.go index 08125aa..d9cf032 100644 --- a/github.go +++ b/github.go @@ -13,6 +13,7 @@ import ( "github.com/dustin/go-humanize" "github.com/hashicorp/go-version" + "github.com/sirupsen/logrus" ) type GitHubRepo struct { @@ -75,7 +76,7 @@ type GitHubReleaseAsset struct { Name string } -func ParseUrlIntoGithubInstance(repoUrl string, apiv string) (GitHubInstance, *FetchError) { +func ParseUrlIntoGithubInstance(logger *logrus.Logger, repoUrl string, apiv string) (GitHubInstance, *FetchError) { var instance GitHubInstance u, err := url.Parse(repoUrl) @@ -86,7 +87,7 @@ func ParseUrlIntoGithubInstance(repoUrl string, apiv string) (GitHubInstance, *F baseUrl := u.Host apiUrl := "api.github.com" if baseUrl != "github.com" && baseUrl != "www.github.com" { - fmt.Printf("Assuming GitHub Enterprise since the provided url (%s) does not appear to be for GitHub.com\n", repoUrl) + logger.Infof("Assuming GitHub Enterprise since the provided url (%s) does not appear to be for GitHub.com\n", repoUrl) apiUrl = baseUrl + "/api/" + apiv } diff --git a/github_test.go b/github_test.go index 6ea6302..06c43e9 100644 --- a/github_test.go +++ b/github_test.go @@ -1,11 +1,12 @@ package main import ( - "github.com/stretchr/testify/require" "io/ioutil" "os" "reflect" "testing" + + "github.com/stretchr/testify/require" ) func TestGetListOfReleasesFromGitHubRepo(t *testing.T) { @@ -146,7 +147,8 @@ func TestParseUrlIntoGithubInstance(t *testing.T) { } for _, tc := range cases { - inst, err := ParseUrlIntoGithubInstance(tc.repoUrl, tc.apiv) + logger := GetProjectLogger() + inst, err := ParseUrlIntoGithubInstance(logger, tc.repoUrl, tc.apiv) if err != nil { t.Fatalf("error extracting url %s into a GitHubRepo struct: %s", tc.repoUrl, err) } diff --git a/go.mod b/go.mod index 805a81a..7189259 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,12 @@ go 1.14 require ( github.com/dustin/go-humanize v1.0.0 + github.com/gruntwork-io/gruntwork-cli v0.7.2 github.com/hashicorp/go-version v1.2.1 + github.com/kr/pretty v0.2.0 // indirect + github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.6.1 + golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/urfave/cli.v1 v1.20.0 ) diff --git a/go.sum b/go.sum index 50d54c4..85aca48 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,55 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/gruntwork-io/gruntwork-cli v0.7.2 h1:aZTztE9vVxUnpNFBecOPuqk1QYl5fPPIriE15Sp3ATs= +github.com/gruntwork-io/gruntwork-cli v0.7.2/go.mod h1:jp6Z7NcLF2avpY8v71fBx6hds9eOFPELSuD/VPv7w00= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/integration_test.go b/integration_test.go index 02b6c71..d05ff68 100644 --- a/integration_test.go +++ b/integration_test.go @@ -15,9 +15,6 @@ import ( ) func TestFetchWithBranchOption(t *testing.T) { - // Note: we don't run these tests in parallel as runFetchCommandWithOutput currently overrides the - // stdout and stderr variables which may result in unstable tests. - tmpDownloadPath := createTempDir(t, "fetch-branch-test") cases := []struct { @@ -40,12 +37,14 @@ func TestFetchWithBranchOption(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { + t.Parallel() + cmd := fmt.Sprintf("fetch --repo %s --branch %s --source-path %s %s", tc.repoUrl, tc.branchName, tc.sourcePath, tmpDownloadPath) - output, _, err := runFetchCommandWithOutput(t, cmd) + _, erroutput, err := runFetchCommandWithOutput(t, cmd) require.NoError(t, err) // When --branch is specified, ensure the latest commit is fetched - assert.Contains(t, output, "Downloading latest commit from branch") + assert.Contains(t, erroutput, "Downloading latest commit from branch") // Ensure the expected file was downloaded assert.FileExists(t, JoinPath(tmpDownloadPath, tc.expectedFile)) @@ -54,58 +53,14 @@ func TestFetchWithBranchOption(t *testing.T) { } func runFetchCommandWithOutput(t *testing.T, command string) (string, string, error) { - // Note: As most of fetch writes directly to stdout and stderr using the fmt package, we need to temporarily override - // the OS pipes. This is based loosely on https://stackoverflow.com/questions/10473800/in-go-how-do-i-capture-stdout-of-a-function-into-a-string/10476304#10476304 - // Our goal eventually is to remove this, but we'll need to introduce a logger as mentioned in: https://github.com/gruntwork-io/fetch/issues/89 stdout := bytes.Buffer{} stderr := bytes.Buffer{} - stdoutReader, stdoutWriter, err1 := os.Pipe() - if err1 != nil { - return "", "", err1 - } - - stderrReader, stderrWriter, err2 := os.Pipe() - if err2 != nil { - return "", "", err2 - } - - oldStdout := os.Stdout - oldStderr := os.Stderr - - // override the pipes to capture output - os.Stdout = stdoutWriter - os.Stderr = stderrWriter - - // execute the fetch command which produces output - err := runFetchCommand(t, command) + err := runFetchCommand(t, command, &stdout, &stderr) if err != nil { return "", "", err } - // copy the output to the buffers in separate goroutines so printing can't block indefinitely - stdoutC := make(chan bytes.Buffer) - stderrC := make(chan bytes.Buffer) - go func() { - var buf bytes.Buffer - io.Copy(&buf, stdoutReader) - stdoutC <- buf - }() - - go func() { - var buf bytes.Buffer - io.Copy(&buf, stderrReader) - stderrC <- buf - }() - - // reset the pipes back to normal - stdoutWriter.Close() - stderrWriter.Close() - os.Stdout = oldStdout - os.Stderr = oldStderr - stdout = <-stdoutC - stderr = <-stderrC - // log the buffers for easier debugging. this is inspired by the integration tests in Terragrunt. // For more information, see: https://github.com/gruntwork-io/terragrunt/blob/master/test/integration_test.go. logBufferContentsLineByLine(t, stdout, "stdout") @@ -113,11 +68,11 @@ func runFetchCommandWithOutput(t *testing.T, command string) (string, string, er return stdout.String(), stderr.String(), nil } -func runFetchCommand(t *testing.T, command string) error { +func runFetchCommand(t *testing.T, command string, writer io.Writer, errwriter io.Writer) error { args := strings.Split(command, " ") - app := CreateFetchCli(VERSION) - app.Action = runFetchWrapper + app := CreateFetchCli(VERSION, writer, errwriter) + app.Action = runFetchTestWrapper return app.Run(args) } @@ -138,8 +93,10 @@ func createTempDir(t *testing.T, prefix string) string { return dir } -// We want to call runFetch() using the app.Action wrapper like the main CLI handler, but we don't want to write to strerr +// We want to call runFetch() using the app.Action wrapper like the main CLI handler, but we don't want to write to stderr // and suddenly exit using os.Exit(1), so we use a separate wrapper method in the integration tests. func runFetchTestWrapper(c *cli.Context) error { - return runFetch(c) + // initialize the logger + logger := GetProjectLoggerWithWriter(c.App.ErrWriter) + return runFetch(c, logger) } diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..3857ab5 --- /dev/null +++ b/logger.go @@ -0,0 +1,22 @@ +package main + +import ( + "io" + + "github.com/gruntwork-io/gruntwork-cli/logging" + "github.com/sirupsen/logrus" +) + +const DEFAULT_LOG_LEVEL = logrus.InfoLevel + +// GetProjectLogger returns a logging instance for this project +func GetProjectLogger() *logrus.Logger { + return logging.GetLogger("fetch") +} + +// GetProjectLoggerWithWriter creates a logger around the given output stream +func GetProjectLoggerWithWriter(writer io.Writer) *logrus.Logger { + logger := GetProjectLogger() + logger.SetOutput(writer) + return logger +} diff --git a/main.go b/main.go index 819cb23..f97aadb 100644 --- a/main.go +++ b/main.go @@ -3,12 +3,15 @@ package main import ( "errors" "fmt" + "io" "os" "path" "regexp" "strings" "sync" + "github.com/gruntwork-io/gruntwork-cli/logging" + "github.com/sirupsen/logrus" cli "gopkg.in/urfave/cli.v1" ) @@ -30,6 +33,9 @@ type FetchOptions struct { LocalDownloadPath string GithubApiVersion string WithProgress bool + + // Project logger + Logger *logrus.Logger } type AssetDownloadResult struct { @@ -49,11 +55,12 @@ const optionReleaseAssetChecksum = "release-asset-checksum" const optionReleaseAssetChecksumAlgo = "release-asset-checksum-algo" const optionGithubAPIVersion = "github-api-version" const optionWithProgress = "progress" +const optionLogLevel = "log-level" const envVarGithubToken = "GITHUB_OAUTH_TOKEN" // Create the Fetch CLI App -func CreateFetchCli(version string) *cli.App { +func CreateFetchCli(version string, writer io.Writer, errwriter io.Writer) *cli.App { cli.OsExiter = func(exitCode int) { // Do nothing. We just need to override this function, as the default value calls os.Exit, which // kills the app (or any automated test) dead in its tracks. @@ -65,6 +72,8 @@ func CreateFetchCli(version string) *cli.App { app.UsageText = "fetch [global options] \n (See https://github.com/gruntwork-io/fetch for examples, argument definitions, and additional docs.)" app.Author = "Gruntwork " app.Version = version + app.Writer = writer + app.ErrWriter = errwriter app.Flags = []cli.Flag{ cli.StringFlag{ @@ -117,36 +126,57 @@ func CreateFetchCli(version string) *cli.App { Name: optionWithProgress, Usage: "Display progress on file downloads, especially useful for large files", }, + cli.StringFlag{ + Name: optionLogLevel, + Value: logrus.InfoLevel.String(), + Usage: "The logging level of the command. Acceptable values\n\tare \"trace\", \"debug\", \"info\", \"warn\", \"error\", \"fatal\" and \"panic\".", + }, } return app } func main() { - app := CreateFetchCli(VERSION) + app := CreateFetchCli(VERSION, os.Stdout, os.Stderr) + app.Before = initLogger app.Action = runFetchWrapper // Run the definition of App.Action app.Run(os.Args) } +// initLogger initializes the Logger before any command is actually executed. This function will handle all the setup +// code, such as setting up the logger with the appropriate log level. +func initLogger(cliContext *cli.Context) error { + // Set logging level + logLevel := cliContext.String(optionLogLevel) + level, err := logrus.ParseLevel(logLevel) + if err != nil { + return fmt.Errorf("Error: %s\n", err) + } + logging.SetGlobalLogLevel(level) + return nil +} + // We just want to call runFetch(), but app.Action won't permit us to return an error, so call a wrapper function instead. func runFetchWrapper(c *cli.Context) { - err := runFetch(c) + // initialize the logger + logger := GetProjectLogger() + err := runFetch(c, logger) if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) + logger.Errorf("%s\n", err) os.Exit(1) } } // Run the fetch program -func runFetch(c *cli.Context) error { - options := parseOptions(c) +func runFetch(c *cli.Context, logger *logrus.Logger) error { + options := parseOptions(c, logger) if err := validateOptions(options); err != nil { return err } - instance, fetchErr := ParseUrlIntoGithubInstance(options.RepoUrl, options.GithubApiVersion) + instance, fetchErr := ParseUrlIntoGithubInstance(logger, options.RepoUrl, options.GithubApiVersion) if fetchErr != nil { return fetchErr } @@ -200,12 +230,12 @@ func runFetch(c *cli.Context) error { } // Download any requested source files - if err := downloadSourcePaths(options.SourcePaths, options.LocalDownloadPath, repo, desiredTag, options.BranchName, options.CommitSha, instance); err != nil { + if err := downloadSourcePaths(logger, options.SourcePaths, options.LocalDownloadPath, repo, desiredTag, options.BranchName, options.CommitSha, instance); err != nil { return err } // Download the requested release assets - assetPaths, err := downloadReleaseAssets(options.ReleaseAsset, options.LocalDownloadPath, repo, desiredTag, options.WithProgress) + assetPaths, err := downloadReleaseAssets(logger, options.ReleaseAsset, options.LocalDownloadPath, repo, desiredTag, options.WithProgress) if err != nil { return err } @@ -213,7 +243,7 @@ func runFetch(c *cli.Context) error { // If applicable, verify the release asset if len(options.ReleaseAssetChecksums) > 0 { for _, assetPath := range assetPaths { - fetchErr = verifyChecksumOfReleaseAsset(assetPath, options.ReleaseAssetChecksums, options.ReleaseAssetChecksumAlgo) + fetchErr = verifyChecksumOfReleaseAsset(logger, assetPath, options.ReleaseAssetChecksums, options.ReleaseAssetChecksumAlgo) if fetchErr != nil { return fetchErr } @@ -223,7 +253,7 @@ func runFetch(c *cli.Context) error { return nil } -func parseOptions(c *cli.Context) FetchOptions { +func parseOptions(c *cli.Context, logger *logrus.Logger) FetchOptions { localDownloadPath := c.Args().First() sourcePaths := c.StringSlice(optionSourcePath) assetChecksums := c.StringSlice(optionReleaseAssetChecksum) @@ -232,7 +262,7 @@ func parseOptions(c *cli.Context) FetchOptions { // Maintain backwards compatibility with older versions of fetch that passed source paths as an optional first // command-line arg if c.NArg() == 2 { - fmt.Printf("DEPRECATION WARNING: passing source paths via command-line args is deprecated. Please use the --%s option instead!\n", optionSourcePath) + logger.Warnf("DEPRECATION WARNING: passing source paths via command-line args is deprecated. Please use the --%s option instead!\n", optionSourcePath) sourcePaths = []string{c.Args().First()} localDownloadPath = c.Args().Get(1) } @@ -255,6 +285,7 @@ func parseOptions(c *cli.Context) FetchOptions { LocalDownloadPath: localDownloadPath, GithubApiVersion: c.String(optionGithubAPIVersion), WithProgress: c.IsSet(optionWithProgress), + Logger: logger, } } @@ -283,7 +314,7 @@ func validateOptions(options FetchOptions) error { } // Download the specified source files from the given repo -func downloadSourcePaths(sourcePaths []string, destPath string, githubRepo GitHubRepo, latestTag string, branchName string, commitSha string, instance GitHubInstance) error { +func downloadSourcePaths(logger *logrus.Logger, sourcePaths []string, destPath string, githubRepo GitHubRepo, latestTag string, branchName string, commitSha string, instance GitHubInstance) error { if len(sourcePaths) == 0 { return nil } @@ -306,18 +337,18 @@ func downloadSourcePaths(sourcePaths []string, destPath string, githubRepo GitHu // GitRef needs to be the fallback and therefore must be last // See https://github.com/gruntwork-io/fetch/issues/87 for an example if gitHubCommit.CommitSha != "" { - fmt.Printf("Downloading git commit \"%s\" of %s ...\n", gitHubCommit.CommitSha, githubRepo.Url) + logger.Infof("Downloading git commit \"%s\" of %s ...\n", gitHubCommit.CommitSha, githubRepo.Url) } else if gitHubCommit.BranchName != "" { - fmt.Printf("Downloading latest commit from branch \"%s\" of %s ...\n", gitHubCommit.BranchName, githubRepo.Url) + logger.Infof("Downloading latest commit from branch \"%s\" of %s ...\n", gitHubCommit.BranchName, githubRepo.Url) } else if gitHubCommit.GitTag != "" { - fmt.Printf("Downloading tag \"%s\" of %s ...\n", latestTag, githubRepo.Url) + logger.Infof("Downloading tag \"%s\" of %s ...\n", latestTag, githubRepo.Url) } else if gitHubCommit.GitRef != "" { - fmt.Printf("Downloading git reference \"%s\" of %s ...\n", gitHubCommit.GitRef, githubRepo.Url) + logger.Infof("Downloading git reference \"%s\" of %s ...\n", gitHubCommit.GitRef, githubRepo.Url) } else { - return fmt.Errorf("The commit sha, tag, and branch name are all empty.") + return fmt.Errorf("The commit sha, tag, and branch name are all empty") } - localZipFilePath, err := downloadGithubZipFile(gitHubCommit, githubRepo.Token, instance) + localZipFilePath, err := downloadGithubZipFile(logger, gitHubCommit, githubRepo.Token, instance) if err != nil { return fmt.Errorf("Error occurred while downloading zip file from GitHub repo: %s", err) } @@ -325,20 +356,21 @@ func downloadSourcePaths(sourcePaths []string, destPath string, githubRepo GitHu // Unzip and move the files we need to our destination for _, sourcePath := range sourcePaths { - fmt.Printf("Extracting files from %s to %s ... ", sourcePath, destPath) + logger.Infof("Extracting files from %s to %s ...\n", sourcePath, destPath) + fileCount, err := extractFiles(localZipFilePath, sourcePath, destPath) plural := "" if fileCount != 1 { plural = "s" } - fmt.Printf("%d file%s extracted\n", fileCount, plural) + logger.Infof("%d file%s extracted\n", fileCount, plural) if err != nil { return fmt.Errorf("Error occurred while extracting files from GitHub zip file: %s", err.Error()) } } - fmt.Println("Download and file extraction complete.") + logger.Infof("Download and file extraction complete.\n") return nil } @@ -348,7 +380,7 @@ func downloadSourcePaths(sourcePaths []string, destPath string, githubRepo GitHu // were downloaded. For those that succeeded, the path they were downloaded to will be passed back // along with the error. // Returns the paths where the release assets were downloaded. -func downloadReleaseAssets(assetRegex string, destPath string, githubRepo GitHubRepo, tag string, withProgress bool) ([]string, error) { +func downloadReleaseAssets(logger *logrus.Logger, assetRegex string, destPath string, githubRepo GitHubRepo, tag string, withProgress bool) ([]string, error) { var err error var assetPaths []string @@ -379,12 +411,12 @@ func downloadReleaseAssets(assetRegex string, destPath string, githubRepo GitHub defer wg.Done() assetPath := path.Join(destPath, asset.Name) - fmt.Printf("Downloading release asset %s to %s\n", asset.Name, assetPath) + logger.Infof("Downloading release asset %s to %s\n", asset.Name, assetPath) if downloadErr := DownloadReleaseAsset(githubRepo, asset.Id, assetPath, withProgress); downloadErr == nil { - fmt.Printf("Downloaded %s\n", assetPath) + logger.Infof("Downloaded %s\n", assetPath) results <- AssetDownloadResult{assetPath, nil} } else { - fmt.Printf("Download failed for %s: %s\n", asset.Name, downloadErr) + logger.Infof("Download failed for %s: %s\n", asset.Name, downloadErr) results <- AssetDownloadResult{assetPath, downloadErr} } }(asset, results) @@ -392,7 +424,7 @@ func downloadReleaseAssets(assetRegex string, destPath string, githubRepo GitHub wg.Wait() close(results) - fmt.Println("Download of release assets complete") + logger.Infof("Download of release assets complete\n") var errorStrs []string for result := range results { @@ -404,7 +436,7 @@ func downloadReleaseAssets(assetRegex string, destPath string, githubRepo GitHub } if numErrors := len(errorStrs); numErrors > 0 { - err = fmt.Errorf("%d errors while downloading assets:\n\t%s", numErrors, strings.Join(errorStrs, "\n\t")) + logger.Errorf("%d errors while downloading assets:\n\t%s", numErrors, strings.Join(errorStrs, "\n\t")) } return assetPaths, err diff --git a/main_test.go b/main_test.go index 2a7fae7..825e5ff 100644 --- a/main_test.go +++ b/main_test.go @@ -13,6 +13,7 @@ const SAMPLE_RELEASE_ASSET_REGEX = "health-checker_linux_[a-z0-9]+" func TestDownloadReleaseAssets(t *testing.T) { tmpDir := mkTempDir(t) + logger := GetProjectLogger() testInst := GitHubInstance{ BaseUrl: "github.com", ApiUrl: "api.github.com", @@ -23,7 +24,7 @@ func TestDownloadReleaseAssets(t *testing.T) { t.Fatalf("Failed to parse sample release asset GitHub URL into Fetch GitHubRepo struct: %s", err) } - assetPaths, fetchErr := downloadReleaseAssets(SAMPLE_RELEASE_ASSET_REGEX, tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) + assetPaths, fetchErr := downloadReleaseAssets(logger, SAMPLE_RELEASE_ASSET_REGEX, tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) if fetchErr != nil { t.Fatalf("Failed to download release asset: %s", fetchErr) } @@ -43,6 +44,7 @@ func TestDownloadReleaseAssets(t *testing.T) { func TestInvalidReleaseAssetsRegex(t *testing.T) { tmpDir := mkTempDir(t) + logger := GetProjectLogger() testInst := GitHubInstance{ BaseUrl: "github.com", ApiUrl: "api.github.com", @@ -53,7 +55,7 @@ func TestInvalidReleaseAssetsRegex(t *testing.T) { t.Fatalf("Failed to parse sample release asset GitHub URL into Fetch GitHubRepo struct: %s", err) } - _, fetchErr := downloadReleaseAssets("*", tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) + _, fetchErr := downloadReleaseAssets(logger, "*", tmpDir, githubRepo, SAMPLE_RELEASE_ASSET_VERSION, false) if fetchErr == nil { t.Fatalf("Expected error for invalid regex") }