Merge pull request #13 from gruntwork-io/release-asset

Release asset

README.md

@@ -1,12 +1,14 @@
 # fetch
 
-fetch downloads all or a subset of files or folders from a specific git commit, branch or tag of a GitHub repo.
+fetch downloads all or a subset of files or folders from a specific git commit, branch, tag, or release of a GitHub
+repo.
 
 #### Features
 
 - Download from a specific git commit SHA.
 - Download from a specific git tag.
 - Download from a specific git branch.
+- Download a binary asset from a specific release.
 - Download from public repos.
 - Download from private repos by specifying a [GitHub Personal Access Token](https://help.github.com/articles/creating-an-access-token-for-command-line-use/).
 - Download a single file, a subset of files, or all files from the repo.
@@ -15,40 +17,52 @@ fetch downloads all or a subset of files or folders from a specific git commit,
 ## Motivation
 
 [Gruntwork](http://gruntwork.io) helps software teams get up and running on AWS with DevOps best practices and world-class 
-infrastructure in about 2 weeks. Sometimes we publish scripts that clients use in their infrastructure, and we want clients
-to auto-download the latest non-breaking version of a script when we publish updates. In addition, for security reasons,
-we wish to verify the integrity of the git commit being downloaded.
+infrastructure in about 2 weeks. Sometimes we publish scripts and binaries that clients use in their infrastructure,
+and we want clients to auto-download the latest non-breaking version of that script or binary when we publish updates.
+In addition, for security reasons, we wish to verify the integrity of the git commit being downloaded.
 
 ## Installation
 
-Download the binary from the [GitHub Releases](https://github.com/gruntwork-io/fetch/releases) tab. 
+Download the fetch binary from the [GitHub Releases](https://github.com/gruntwork-io/fetch/releases) tab.
 
 ## Assumptions
 
-fetch assumes that a repo's tags are in the format `vX.Y.Z` or `X.Y.Z` to support Semantic Versioning parsing. Repos that
-use git tags not in this format cannot be used with fetch.
+fetch assumes that a repo's tags are in the format `vX.Y.Z` or `X.Y.Z` to support Semantic Versioning parsing. Repos
+that use git tags not in this format cannot currently be used with fetch.
 
 ## Usage
 
 #### General Usage
 
 ```
-fetch --repo=<github-repo-url> --tag=<version-constraint> [<repo-download-filter>] <local-download-path>
+fetch [OPTIONS] <local-download-path>
 ```
 
-- `<repo-download-filter>` 
-  **Optional**.
-  If blank, all files in the repo will be downloaded. Otherwise, only files located in the given path
-  will be downloaded. 
-  - Example: `/` Download all files in the repo
-  - Example: `/folder` Download only files in the `/folder` path and below from the repo
-- `<local-download-path>`
-  **Required**.
-  The local path where all files should be downloaded.
-  - Example: `/tmp` Download all files to `/tmp`
+The supported options are:
 
-Run `fetch --help` to see more information about the flags, some of which are required. See [Tag Constraint Expressions](#tag-constraint-expressions)
-for examples of tag constraints you can use.
+- `--repo` (**Required**): The fully qualified URL of the GitHub repo to download from (e.g. https://github.com/foo/bar).
+- `--tag` (**Optional**): The git tag to download. Can be a specific tag or a [Tag Constraint
+  Expression](#tag-constraint-expressions).
+- `--branch` (**Optional**): The git branch from which to download; the latest commit in the branch will be used. If
+  specified, will override `--tag`.
+- `--commit` (**Optional**): The SHA of a git commit to download. If specified, will override `--branch` and `--tag`.
+- `--source-path` (**Optional**): The source path to download from the repo (e.g. `--source-path=/folder` will download
+  the `/folder` path and all files below it). By default, all files are downloaded from the repo unless `--source-path`
+  or `--release-asset` is specified. This option can be specified more than once.
+- `--release-asset` (**Optional**): The name of a release asset--that is, a binary uploaded to a [GitHub
+  Release](https://help.github.com/articles/creating-releases/)--to download. This option can be specified more than
+  once. It only works with the `--tag` option.
+- `--github-oauth-token` (**Optional**): A [GitHub Personal Access
+  Token](https://help.github.com/articles/creating-an-access-token-for-command-line-use/). Required if you're
+  downloading from private GitHub repos. **NOTE:** fetch will also look for this token using the `GITHUB_OAUTH_TOKEN`
+  environment variable, which we recommend using instead of the command line option to ensure the token doesn't get
+  saved in bash history.
+
+The supported arguments are:
+
+- `<local-download-path>` (**Required**): The local path where all files should be downloaded (e.g. `/tmp`).
+
+Run `fetch --help` to see more information about the flags.
 
 #### Usage Example 1
 
@@ -58,7 +72,7 @@ Download `/modules/foo/bar.sh` from a GitHub release where the tag is the latest
 fetch \
 --repo="https://github.com/gruntwork-io/script-modules" \
 --tag="~>0.1.5" \
-/modules/foo/bar.sh \
+--source-path="/modules/foo/bar.sh" \
 /tmp/bar
 ```
 
@@ -70,7 +84,7 @@ Download all files in `/modules/foo` from a GitHub release where the tag is exac
 fetch \
 --repo="https://github.com/gruntwork-io/script-modules" \
 --tag="0.1.5" \
-/modules/foo \
+--source-path="/modules/foo" \
 /tmp
 
 ```
@@ -80,10 +94,11 @@ fetch \
 Download all files from a private GitHub repo using the GitHUb oAuth Token `123`. Get the release whose tag is exactly `0.1.5`, and save the files to `/tmp`:
 
 ```
+GITHUB_OAUTH_TOKEN=123
+
 fetch \
 --repo="https://github.com/gruntwork-io/script-modules" \
 --tag="0.1.5" \
---github-oauth-token="123" \
 /tmp
 
 ```
@@ -112,6 +127,17 @@ fetch \
 
 ```
 
+#### Usage Example 6
+
+Download the release asset `foo.exe` from a GitHub release where the tag is exactly `0.1.5`, and save it to `/tmp`:
+
+```
+fetch \
+--repo="https://github.com/gruntwork-io/script-modules" \
+--tag="0.1.5" \
+--release-asset="foo.exe" \
+/tmp
+
 
 #### Tag Constraint Expressions
 

fetch_error.go

@@ -23,6 +23,9 @@ func newError(errorCode int, details string) *FetchError {
 }
 
 func wrapError(err error) *FetchError {
+	if err == nil {
+		return nil
+	}
 	return &FetchError{
 		errorCode: -1,
 		details: err.Error(),

github.go

@@ -6,11 +6,15 @@ import (
 	"bytes"
 	"encoding/json"
 	"regexp"
+	"os"
+	"io"
 )
 
 type GitHubRepo struct {
+	Url   string // The URL of the GitHub repo
 	Owner string // The GitHub account name under which the repo exists
 	Name  string // The GitHub repo name
+	Token string // The personal access token to access this repo (if it's a private repo)
 }
 
 // Represents a specific git commit.
@@ -40,35 +44,37 @@ type GitHubTagsCommitApiResponse struct {
 	Url string // The URL at which additional API information can be found for the given commit
 }
 
+// Modeled directly after the api.github.com response (but only includes the fields we care about). For more info, see:
+// https://developer.github.com/v3/repos/releases/#get-a-release-by-tag-name
+type GitHubReleaseApiResponse struct {
+	Id      int
+	Url     string
+	Name    string
+	Assets  []GitHubReleaseAsset
+}
+
+// The "assets" portion of the GitHubReleaseApiResponse. Modeled directly after the api.github.com response (but only
+// includes the fields we care about). For more info, see:
+// https://developer.github.com/v3/repos/releases/#get-a-release-by-tag-name
+type GitHubReleaseAsset struct {
+	Id   int
+	Url  string
+	Name string
+}
+
 // Fetch all tags from the given GitHub repo
 func FetchTags(githubRepoUrl string, githubToken string) ([]string, *FetchError) {
 	var tagsString []string
 
-	repo, err := ParseUrlIntoGitHubRepo(githubRepoUrl)
+	repo, err := ParseUrlIntoGitHubRepo(githubRepoUrl, githubToken)
 	if err != nil {
 		return tagsString, wrapError(err)
 	}
 
-	// Make an HTTP request, possibly with the gitHubOAuthToken in the header
-	httpClient := &http.Client{}
-
-	req, err := MakeGitHubTagsRequest(repo, githubToken)
+	url := createGitHubRepoUrlForPath(repo, "tags")
+	resp, err := callGitHubApi(repo, url, map[string]string{})
 	if err != nil {
-		return tagsString, wrapError(err)
-	}
-
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return tagsString, wrapError(err)
-	}
-	if resp.StatusCode != 200 {
-		// Convert the resp.Body to a string
-		buf := new(bytes.Buffer)
-		buf.ReadFrom(resp.Body)
-		respBody := buf.String()
-
-		// We leverage the HTTP Response Code as our ErrorCode here.
-		return tagsString, newError(resp.StatusCode, fmt.Sprintf("Received HTTP Response %d while fetching releases for GitHub URL %s. Full HTTP response: %s", resp.StatusCode, githubRepoUrl, respBody))
+		return tagsString, err
 	}
 
 	// Convert the response body to a byte array
@@ -78,8 +84,7 @@ func FetchTags(githubRepoUrl string, githubToken string) ([]string, *FetchError)
 
 	// Extract the JSON into our array of gitHubTagsCommitApiResponse's
 	var tags []GitHubTagsApiResponse
-	err = json.Unmarshal(jsonResp, &tags)
-	if err != nil {
+	if err := json.Unmarshal(jsonResp, &tags); err != nil {
 		return tagsString, wrapError(err)
 	}
 
@@ -91,7 +96,7 @@ func FetchTags(githubRepoUrl string, githubToken string) ([]string, *FetchError)
 }
 
 // Convert a URL into a GitHubRepo struct
-func ParseUrlIntoGitHubRepo(url string) (GitHubRepo, error) {
+func ParseUrlIntoGitHubRepo(url string, token string) (GitHubRepo, *FetchError) {
 	var gitHubRepo GitHubRepo
 
 	regex, regexErr := regexp.Compile("https?://(?:www\\.)?github.com/(.+?)/(.+?)(?:$|\\?|#|/)")
@@ -105,26 +110,97 @@ func ParseUrlIntoGitHubRepo(url string) (GitHubRepo, error) {
 	}
 
 	gitHubRepo = GitHubRepo{
+		Url: url,
 		Owner: matches[1],
 		Name: matches[2],
+		Token: token,
 	}
 
 	return gitHubRepo, nil
 }
 
+// Download the release asset with the given id and return its body
+func DownloadReleaseAsset(repo GitHubRepo, assetId int, destPath string) *FetchError {
+	url := createGitHubRepoUrlForPath(repo, fmt.Sprintf("releases/assets/%d", assetId))
+	resp, err := callGitHubApi(repo, url, map[string]string{"Accept": "application/octet-stream"})
+	if err != nil {
+		return err
+	}
+
+	return writeResonseToDisk(resp, destPath)
+}
 
-// Return an HTTP request that will fetch the given GitHub repo's tags, possibly with the gitHubOAuthToken in the header
-func MakeGitHubTagsRequest(repo GitHubRepo, gitHubToken string) (*http.Request, error) {
-	var request *http.Request
+// Get information about the GitHub release with the given tag
+func GetGitHubReleaseInfo(repo GitHubRepo, tag string) (GitHubReleaseApiResponse, *FetchError) {
+	release := GitHubReleaseApiResponse{}
 
-	request, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s/%s/tags", repo.Owner, repo.Name), nil)
+	url := createGitHubRepoUrlForPath(repo, fmt.Sprintf("releases/tags/%s", tag))
+	resp, err := callGitHubApi(repo, url, map[string]string{})
 	if err != nil {
-		return request, wrapError(err)
+		return release, err
 	}
 
-	if gitHubToken != "" {
-		request.Header.Set("Authorization", fmt.Sprintf("token %s", gitHubToken))
+	// Convert the response body to a byte array
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(resp.Body)
+	jsonResp := buf.Bytes()
+
+	if err := json.Unmarshal(jsonResp, &release); err != nil {
+		return release, wrapError(err)
 	}
 
-	return request, nil
+	return release, nil
 }
+
+// Craft a URL for the GitHub repos API of the form repos/:owner/:repo/:path
+func createGitHubRepoUrlForPath(repo GitHubRepo, path string) string {
+	return fmt.Sprintf("repos/%s/%s/%s", repo.Owner, repo.Name, path)
+}
+
+// Call the GitHub API at the given path and return the HTTP response
+func callGitHubApi(repo GitHubRepo, path string, customHeaders map[string]string) (*http.Response, *FetchError) {
+	httpClient := &http.Client{}
+
+	request, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/%s", path), nil)
+	if err != nil {
+		return nil, wrapError(err)
+	}
+
+	if repo.Token != "" {
+		request.Header.Set("Authorization", fmt.Sprintf("token %s", repo.Token))
+	}
+
+	for headerName, headerValue := range customHeaders {
+		request.Header.Set(headerName, headerValue)
+	}
+
+	resp, err := httpClient.Do(request)
+	if err != nil {
+		return nil, wrapError(err)
+	}
+	if resp.StatusCode != 200 {
+		// Convert the resp.Body to a string
+		buf := new(bytes.Buffer)
+		buf.ReadFrom(resp.Body)
+		respBody := buf.String()
+
+		// We leverage the HTTP Response Code as our ErrorCode here.
+		return nil, newError(resp.StatusCode, fmt.Sprintf("Received HTTP Response %d while fetching releases for GitHub URL %s. Full HTTP response: %s", resp.StatusCode, repo.Url, respBody))
+	}
+
+	return resp, nil
+}
+
+// Write the body of the given HTTP response to disk at the given path
+func writeResonseToDisk(resp *http.Response, destPath string) *FetchError {
+	out, err := os.Create(destPath)
+	if err != nil  {
+		return wrapError(err)
+	}
+
+	defer out.Close()
+	defer resp.Body.Close()
+
+	_, err = io.Copy(out, resp.Body)
+	return wrapError(err)
+}