diff --git a/README.md b/README.md index 45098da..e242c2a 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ authentication. Fetch makes it possible to handle all of these cases with a one- - Download a binary asset from a specific release. - Verify the SHA256 or SHA512 checksum of a binary asset. - Download from public repos, or from private repos by specifying a [GitHub Personal Access Token](https://help.github.com/articles/creating-an-access-token-for-command-line-use/). +- Download from GitHub Enterprise. - When specifying a git tag, you can can specify either exactly the tag you want, or a [Tag Constraint Expression](#tag-constraint-expressions) to do things like "get the latest non-breaking version" of this repo. Note that fetch assumes git tags are specified according to [Semantic Versioning](http://semver.org/) principles. #### Quick examples @@ -75,6 +76,8 @@ The supported options are: downloading from private GitHub repos. **NOTE:** fetch will also look for this token using the `GITHUB_OAUTH_TOKEN` environment variable, which we recommend using instead of the command line option to ensure the token doesn't get saved in bash history. +- `--github-api-version` (**Optional**): Used when fetching an artifact from a GitHub Enterprise instance. + Defaults to `v3`. This is ignored when fetching from GitHub.com. The supported arguments are: @@ -152,6 +155,14 @@ Download the release asset `foo.exe` from a GitHub release where the tag is exac fetch --repo="https://github.com/foo/bar" --tag="0.1.5" --release-asset="foo.exe" /tmp ``` +#### Usage Example 7 + +Download the release asset `foo.exe` from a GitHub release hosted on a GitHub Enterprise instance running at `ghe.mycompany.com` where the tag is exactly `0.1.5`, and save it to `/tmp`: + +``` +fetch --repo="https://ghe.mycompany.com/foo/bar" --tag="0.1.5" --release-asset="foo.exe" /tmp +``` + ## License This code is released under the MIT License. See [LICENSE.txt](/LICENSE.txt). diff --git a/checksum_test.go b/checksum_test.go index aa792ea..36e1011 100644 --- a/checksum_test.go +++ b/checksum_test.go @@ -16,8 +16,12 @@ const SAMPLE_RELEASE_ASSET_CHECKSUM_SHA512="28d9e487c1001e3c28d915c9edd3ed37632f func TestVerifyReleaseAsset(t *testing.T) { tmpDir := mkTempDir(t) + testInst := GitHubInstance{ + BaseUrl: "github.com", + ApiUrl: "api.github.com", + } - githubRepo, err := ParseUrlIntoGitHubRepo(SAMPLE_RELEASE_ASSET_GITHUB_REPO_URL, "") + githubRepo, err := ParseUrlIntoGitHubRepo(SAMPLE_RELEASE_ASSET_GITHUB_REPO_URL, "", testInst) if err != nil { t.Fatalf("Failed to parse sample release asset GitHub URL into Fetch GitHubRepo struct: %s", err) } diff --git a/github.go b/github.go index 55e2de7..5e1bfc3 100644 --- a/github.go +++ b/github.go @@ -1,20 +1,28 @@ package main import ( - "net/http" - "fmt" "bytes" "encoding/json" - "regexp" - "os" + "fmt" "io" + "net/http" + "net/url" + "os" + "regexp" ) type GitHubRepo struct { - Url string // The URL of the GitHub repo - Owner string // The GitHub account name under which the repo exists - Name string // The GitHub repo name - Token string // The personal access token to access this repo (if it's a private repo) + Url string // The URL of the GitHub repo + BaseUrl string // The Base URL of the GitHub Instance + ApiUrl string // The API Url of the GitHub Instance + Owner string // The GitHub account name under which the repo exists + Name string // The GitHub repo name + Token string // The personal access token to access this repo (if it's a private repo) +} + +type GitHubInstance struct { + BaseUrl string + ApiUrl string } // Represents a specific git commit. @@ -62,11 +70,34 @@ type GitHubReleaseAsset struct { Name string } +func ParseUrlIntoGithubInstance(repoUrl string, apiv string) (GitHubInstance, *FetchError) { + var instance GitHubInstance + + u, err := url.Parse(repoUrl) + if err != nil { + return instance, newError(GITHUB_REPO_URL_MALFORMED_OR_NOT_PARSEABLE, fmt.Sprintf("GitHub Repo URL %s is malformed.", repoUrl)) + } + + baseUrl := u.Host + apiUrl := "api.github.com" + if baseUrl != "github.com" && baseUrl != "www.github.com" { + fmt.Printf("Assuming GitHub Enterprise since the provided url (%s) does not appear to be for GitHub.com\n", repoUrl) + apiUrl = baseUrl + "/api/" + apiv + } + + instance = GitHubInstance{ + BaseUrl: baseUrl, + ApiUrl: apiUrl, + } + + return instance, nil +} + // Fetch all tags from the given GitHub repo -func FetchTags(githubRepoUrl string, githubToken string) ([]string, *FetchError) { +func FetchTags(githubRepoUrl string, githubToken string, instance GitHubInstance) ([]string, *FetchError) { var tagsString []string - repo, err := ParseUrlIntoGitHubRepo(githubRepoUrl, githubToken) + repo, err := ParseUrlIntoGitHubRepo(githubRepoUrl, githubToken, instance) if err != nil { return tagsString, wrapError(err) } @@ -96,10 +127,10 @@ func FetchTags(githubRepoUrl string, githubToken string) ([]string, *FetchError) } // Convert a URL into a GitHubRepo struct -func ParseUrlIntoGitHubRepo(url string, token string) (GitHubRepo, *FetchError) { +func ParseUrlIntoGitHubRepo(url string, token string, instance GitHubInstance) (GitHubRepo, *FetchError) { var gitHubRepo GitHubRepo - regex, regexErr := regexp.Compile("https?://(?:www\\.)?github.com/(.+?)/(.+?)(?:$|\\?|#|/)") + regex, regexErr := regexp.Compile("https?://(?:www\\.)?" + instance.BaseUrl + "/(.+?)/(.+?)(?:$|\\?|#|/)") if regexErr != nil { return gitHubRepo, newError(GITHUB_REPO_URL_MALFORMED_OR_NOT_PARSEABLE, fmt.Sprintf("GitHub Repo URL %s is malformed.", url)) } @@ -110,10 +141,12 @@ func ParseUrlIntoGitHubRepo(url string, token string) (GitHubRepo, *FetchError) } gitHubRepo = GitHubRepo{ - Url: url, - Owner: matches[1], - Name: matches[2], - Token: token, + Url: url, + BaseUrl: instance.BaseUrl, + ApiUrl: instance.ApiUrl, + Owner: matches[1], + Name: matches[2], + Token: token, } return gitHubRepo, nil @@ -161,7 +194,7 @@ func createGitHubRepoUrlForPath(repo GitHubRepo, path string) string { func callGitHubApi(repo GitHubRepo, path string, customHeaders map[string]string) (*http.Response, *FetchError) { httpClient := &http.Client{} - request, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/%s", path), nil) + request, err := http.NewRequest("GET", fmt.Sprintf("https://"+repo.ApiUrl+"/%s", path), nil) if err != nil { return nil, wrapError(err) } diff --git a/github_test.go b/github_test.go index 6f33c9a..127d9c6 100644 --- a/github_test.go +++ b/github_test.go @@ -1,30 +1,35 @@ package main import ( - "testing" + "io/ioutil" "os" "reflect" - "io/ioutil" + "testing" ) func TestGetListOfReleasesFromGitHubRepo(t *testing.T) { t.Parallel() + testInst := GitHubInstance{ + BaseUrl: "github.com", + ApiUrl: "api.github.com", + } cases := []struct { repoUrl string firstReleaseTag string lastReleaseTag string gitHubOAuthToken string + testInst GitHubInstance }{ // Test on a public repo whose sole purpose is to be a test fixture for this tool - {"https://github.com/gruntwork-io/fetch-test-public", "v0.0.1", "v0.0.3", ""}, + {"https://github.com/gruntwork-io/fetch-test-public", "v0.0.1", "v0.0.3", "", testInst}, // Private repo equivalent - {"https://github.com/gruntwork-io/fetch-test-private", "v0.0.2", "v0.0.2", os.Getenv("GITHUB_OAUTH_TOKEN")}, + {"https://github.com/gruntwork-io/fetch-test-private", "v0.0.2", "v0.0.2", os.Getenv("GITHUB_OAUTH_TOKEN"), testInst}, } for _, tc := range cases { - releases, err := FetchTags(tc.repoUrl, tc.gitHubOAuthToken) + releases, err := FetchTags(tc.repoUrl, tc.gitHubOAuthToken, testInst) if err != nil { t.Fatalf("error fetching releases: %s", err) } @@ -47,27 +52,114 @@ func TestGetListOfReleasesFromGitHubRepo(t *testing.T) { } } +func TestParseUrlIntoGithubInstance(t *testing.T) { + t.Parallel() + + ghTestInst := GitHubInstance{ + BaseUrl: "github.com", + ApiUrl: "api.github.com", + } + wwwGhTestInst := GitHubInstance{ + BaseUrl: "www.github.com", + ApiUrl: "api.github.com", + } + gheTestInst := GitHubInstance{ + BaseUrl: "ghe.mycompany.com", + ApiUrl: "ghe.mycompany.com/api/v3", + } + wwwGheTestInst := GitHubInstance{ + BaseUrl: "www.ghe.mycompany.com", + ApiUrl: "www.ghe.mycompany.com/api/v3", + } + myCoTestInst := GitHubInstance{ + BaseUrl: "mycogithub.com", + ApiUrl: "mycogithub.com/api/v3", + } + wwwMyCoTestInst := GitHubInstance{ + BaseUrl: "www.mycogithub.com", + ApiUrl: "www.mycogithub.com/api/v3", + } + localTestInst := GitHubInstance{ + BaseUrl: "mycogithub.local", + ApiUrl: "mycogithub.local/api/v3", + } + netTestInst := GitHubInstance{ + BaseUrl: "mycogithub.net", + ApiUrl: "mycogithub.net/api/v3", + } + + cases := []struct { + repoUrl string + apiv string + expectedInst GitHubInstance + }{ + {"http://www.github.com/gruntwork-io/script-modules/", "", wwwGhTestInst}, + {"https://www.github.com/gruntwork-io/script-modules/", "", wwwGhTestInst}, + {"http://github.com/gruntwork-io/script-modules/", "", ghTestInst}, + {"http://www.ghe.mycompany.com/gruntwork-io/script-modules", "v3", wwwGheTestInst}, + {"https://www.ghe.mycompany.com/gruntwork-io/script-modules", "v3", wwwGheTestInst}, + {"http://ghe.mycompany.com/gruntwork-io/script-modules", "v3", gheTestInst}, + {"http://www.mycogithub.com/gruntwork-io/script-modules", "v3", wwwMyCoTestInst}, + {"https://www.mycogithub.com/gruntwork-io/script-modules", "v3", wwwMyCoTestInst}, + {"http://mycogithub.com/gruntwork-io/script-modules", "v3", myCoTestInst}, + {"http://mycogithub.local/gruntwork-io/script-modules", "v3", localTestInst}, + {"http://mycogithub.net/gruntwork-io/script-modules", "v3", netTestInst}, + } + + for _, tc := range cases { + inst, err := ParseUrlIntoGithubInstance(tc.repoUrl, tc.apiv) + if err != nil { + t.Fatalf("error extracting url %s into a GitHubRepo struct: %s", tc.repoUrl, err) + } + + if inst.BaseUrl != tc.expectedInst.BaseUrl { + t.Fatalf("while parsing %s, expected base url %s, received %s", tc.repoUrl, tc.expectedInst.BaseUrl, inst.BaseUrl) + } + + if inst.ApiUrl != tc.expectedInst.ApiUrl { + t.Fatalf("while parsing %s, expected api url %s, received %s", tc.repoUrl, tc.expectedInst.ApiUrl, inst.ApiUrl) + } + } +} + func TestParseUrlIntoGitHubRepo(t *testing.T) { t.Parallel() + ghTestInst := GitHubInstance{ + BaseUrl: "github.com", + ApiUrl: "api.github.com", + } + gheTestInst := GitHubInstance{ + BaseUrl: "ghe.mycompany.com", + ApiUrl: "ghe.mycompany.com/api/v3", + } + myCoTestInst := GitHubInstance{ + BaseUrl: "mycogithub.com", + ApiUrl: "mycogithub.com/api/v3", + } cases := []struct { - repoUrl string - owner string - name string - token string + repoUrl string + owner string + name string + token string + testInst GitHubInstance }{ - {"https://github.com/brikis98/ping-play", "brikis98", "ping-play", ""}, - {"http://github.com/brikis98/ping-play", "brikis98", "ping-play", ""}, - {"https://github.com/gruntwork-io/script-modules", "gruntwork-io", "script-modules", ""}, - {"http://github.com/gruntwork-io/script-modules", "gruntwork-io", "script-modules", ""}, - {"http://www.github.com/gruntwork-io/script-modules", "gruntwork-io", "script-modules", ""}, - {"http://www.github.com/gruntwork-io/script-modules/", "gruntwork-io", "script-modules", ""}, - {"http://www.github.com/gruntwork-io/script-modules?foo=bar", "gruntwork-io", "script-modules", "token"}, - {"http://www.github.com/gruntwork-io/script-modules?foo=bar&foo=baz", "gruntwork-io", "script-modules", "token"}, + {"https://github.com/brikis98/ping-play", "brikis98", "ping-play", "", ghTestInst}, + {"http://github.com/brikis98/ping-play", "brikis98", "ping-play", "", ghTestInst}, + {"https://github.com/gruntwork-io/script-modules", "gruntwork-io", "script-modules", "", ghTestInst}, + {"http://github.com/gruntwork-io/script-modules", "gruntwork-io", "script-modules", "", ghTestInst}, + {"http://www.github.com/gruntwork-io/script-modules", "gruntwork-io", "script-modules", "", ghTestInst}, + {"http://www.github.com/gruntwork-io/script-modules/", "gruntwork-io", "script-modules", "", ghTestInst}, + {"http://www.github.com/gruntwork-io/script-modules?foo=bar", "gruntwork-io", "script-modules", "token", ghTestInst}, + {"http://www.github.com/gruntwork-io/script-modules?foo=bar&foo=baz", "gruntwork-io", "script-modules", "token", ghTestInst}, + {"http://www.ghe.mycompany.com/gruntwork-io/script-modules?foo=bar&foo=baz", "gruntwork-io", "script-modules", "token", gheTestInst}, + {"https://www.ghe.mycompany.com/gruntwork-io/script-modules?foo=bar&foo=baz", "gruntwork-io", "script-modules", "token", gheTestInst}, + {"http://ghe.mycompany.com/gruntwork-io/script-modules?foo=bar&foo=baz", "gruntwork-io", "script-modules", "token", gheTestInst}, + {"http://mycogithub.com/gruntwork-io/script-modules?foo=bar&foo=baz", "gruntwork-io", "script-modules", "token", myCoTestInst}, } for _, tc := range cases { - repo, err := ParseUrlIntoGitHubRepo(tc.repoUrl, tc.token) + repo, err := ParseUrlIntoGitHubRepo(tc.repoUrl, tc.token, tc.testInst) if err != nil { t.Fatalf("error extracting url %s into a GitHubRepo struct: %s", tc.repoUrl, err) } @@ -92,6 +184,7 @@ func TestParseUrlIntoGitHubRepo(t *testing.T) { func TestParseUrlThrowsErrorOnMalformedUrl(t *testing.T) { t.Parallel() + testInst := GitHubInstance{} cases := []struct { repoUrl string @@ -102,7 +195,7 @@ func TestParseUrlThrowsErrorOnMalformedUrl(t *testing.T) { } for _, tc := range cases { - _, err := ParseUrlIntoGitHubRepo(tc.repoUrl, "") + _, err := ParseUrlIntoGitHubRepo(tc.repoUrl, "", testInst) if err == nil { t.Fatalf("Expected error on malformed url %s, but no error was received.", tc.repoUrl) } @@ -134,6 +227,11 @@ func TestGetGitHubReleaseInfo(t *testing.T) { Assets: []GitHubReleaseAsset{}, } + testInst := GitHubInstance{ + BaseUrl: "github.com", + ApiUrl: "api.github.com", + } + cases := []struct { repoUrl string repoToken string @@ -145,7 +243,7 @@ func TestGetGitHubReleaseInfo(t *testing.T) { } for _, tc := range cases { - repo, err := ParseUrlIntoGitHubRepo(tc.repoUrl, tc.repoToken) + repo, err := ParseUrlIntoGitHubRepo(tc.repoUrl, tc.repoToken, testInst) if err != nil { t.Fatalf("Failed to parse %s into GitHub URL due to error: %s", tc.repoUrl, err.Error()) } @@ -156,7 +254,7 @@ func TestGetGitHubReleaseInfo(t *testing.T) { } if !reflect.DeepEqual(tc.expected, resp) { - t.Fatalf("Expected GitHub release %s but got GitHub release %s", tc.expected, resp) + t.Fatalf("Expected GitHub release %v but got GitHub release %v", tc.expected, resp) } } } @@ -166,6 +264,11 @@ func TestDownloadReleaseAsset(t *testing.T) { token := os.Getenv("GITHUB_OAUTH_TOKEN") + testInst := GitHubInstance{ + BaseUrl: "github.com", + ApiUrl: "api.github.com", + } + cases := []struct { repoUrl string repoToken string @@ -177,7 +280,7 @@ func TestDownloadReleaseAsset(t *testing.T) { } for _, tc := range cases { - repo, err := ParseUrlIntoGitHubRepo(tc.repoUrl, tc.repoToken) + repo, err := ParseUrlIntoGitHubRepo(tc.repoUrl, tc.repoToken, testInst) if err != nil { t.Fatalf("Failed to parse %s into GitHub URL due to error: %s", tc.repoUrl, err.Error()) } @@ -188,13 +291,13 @@ func TestDownloadReleaseAsset(t *testing.T) { } if err := DownloadReleaseAsset(repo, tc.assetId, tmpFile.Name()); err != nil { - t.Fatalf("Failed to download asset %s to %s from GitHub URL %s due to error: %s", tc.assetId, tmpFile.Name(), tc.repoUrl, err.Error()) + t.Fatalf("Failed to download asset %d to %s from GitHub URL %s due to error: %s", tc.assetId, tmpFile.Name(), tc.repoUrl, err.Error()) } defer os.Remove(tmpFile.Name()) if !fileExists(tmpFile.Name()) { - t.Fatalf("Got no errors downloading asset %s to %s from GitHub URL %s, but %s does not exist!", tc.assetId, tmpFile.Name(), tc.repoUrl, tmpFile.Name()) + t.Fatalf("Got no errors downloading asset %d to %s from GitHub URL %s, but %s does not exist!", tc.assetId, tmpFile.Name(), tc.repoUrl, tmpFile.Name()) } } } diff --git a/main.go b/main.go index 4502e28..450310d 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ type FetchOptions struct { ReleaseAssetChecksum string ReleaseAssetChecksumAlgo string LocalDownloadPath string + GithubApiVersion string } const OPTION_REPO = "repo" @@ -34,6 +35,7 @@ const OPTION_SOURCE_PATH = "source-path" const OPTION_RELEASE_ASSET = "release-asset" const OPTION_RELEASE_ASSET_CHECKSUM = "release-asset-checksum" const OPTION_RELEASE_ASSET_CHECKSUM_ALGO = "release-asset-checksum-algo" +const OPTION_GITHUB_API_VERSION = "github-api-version" const ENV_VAR_GITHUB_TOKEN = "GITHUB_OAUTH_TOKEN" @@ -82,6 +84,11 @@ func main() { Name: OPTION_RELEASE_ASSET_CHECKSUM_ALGO, Usage: "The algorithm Fetch will use to compute a checksum of the release asset. Acceptable values\n\tare \"sha256\" and \"sha512\".", }, + cli.StringFlag{ + Name: OPTION_GITHUB_API_VERSION, + Value: "v3", + Usage: "The api version of the GitHub instance. If left blank, v3 will be used.\n\tThis will only be used if the repo url is not a github.com url.", + }, } app.Action = runFetchWrapper @@ -106,8 +113,13 @@ func runFetch(c *cli.Context) error { return err } + instance, fetchErr := ParseUrlIntoGithubInstance(options.RepoUrl, options.GithubApiVersion) + if fetchErr != nil { + return fetchErr + } + // Get the tags for the given repo - tags, fetchErr := FetchTags(options.RepoUrl, options.GithubToken) + tags, fetchErr := FetchTags(options.RepoUrl, options.GithubToken, instance) if fetchErr != nil { if fetchErr.errorCode == INVALID_GITHUB_TOKEN_OR_ACCESS_DENIED { return errors.New(getErrorMessage(INVALID_GITHUB_TOKEN_OR_ACCESS_DENIED, fetchErr.details)) @@ -133,7 +145,7 @@ func runFetch(c *cli.Context) error { } // Prepare the vars we'll need to download - repo, fetchErr := ParseUrlIntoGitHubRepo(options.RepoUrl, options.GithubToken) + repo, fetchErr := ParseUrlIntoGitHubRepo(options.RepoUrl, options.GithubToken, instance) if fetchErr != nil { return fmt.Errorf("Error occurred while parsing GitHub URL: %s", fetchErr) } @@ -188,6 +200,7 @@ func parseOptions(c *cli.Context) FetchOptions { ReleaseAssetChecksum: c.String(OPTION_RELEASE_ASSET_CHECKSUM), ReleaseAssetChecksumAlgo: c.String(OPTION_RELEASE_ASSET_CHECKSUM_ALGO), LocalDownloadPath: localDownloadPath, + GithubApiVersion: c.String(OPTION_GITHUB_API_VERSION), } }