From 90372b007cfa32e85e5fd18d686a8319944eff37 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Fri, 26 Jan 2024 09:29:35 +0100 Subject: [PATCH 1/6] feat: with github registry Signed-off-by: Kasper J. Hermansen --- cmd/cmd.go | 2 +- cmd/ext.go | 24 ++ internal/extensions/downloader.go | 6 + internal/extensions/extension.go | 3 +- internal/extensions/extension_source.go | 44 +++ internal/extensions/extensions.go | 20 +- internal/extensions/git_registry.go | 5 + internal/extensions/github_registry.go | 330 +++++++++++++++++++ internal/extensions/registry.go | 17 +- internal/extensions/remote_registry.go | 9 + internal/extensions/remote_registry_paths.go | 15 + 11 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 internal/extensions/extension_source.go create mode 100644 internal/extensions/github_registry.go create mode 100644 internal/extensions/remote_registry.go create mode 100644 internal/extensions/remote_registry_paths.go diff --git a/cmd/cmd.go b/cmd/cmd.go index d9bd9b1..577f5ac 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -161,7 +161,7 @@ func initializedRootFromArgs(stdout, stderr io.Writer, args []string) (*cobra.Co rootCmd.AddCommand(newExtCmd()) if err := addExtensions(rootCmd); err != nil { - return nil, nil, fmt.Errorf("failed to register extensions: %w", err) + uii.Verboseln("failed to register extensions: %s", err.Error()) } if isInRepoContext() { diff --git a/cmd/ext.go b/cmd/ext.go index 46ceb75..d266676 100644 --- a/cmd/ext.go +++ b/cmd/ext.go @@ -86,6 +86,7 @@ func newExtCmd() *cobra.Command { newExtInstallCmd(globalConfig), newExtUpdateCmd(globalConfig), newExtInitCmd(globalConfig), + newExtPublishCmd(globalConfig), ) cmd.PersistentFlags().StringVar(&globalConfig.registry, "registry", "", "the given registry, if not set will default to SHUTTLE_EXTENSIONS_REGISTRY") @@ -145,3 +146,26 @@ func newExtInitCmd(globalConfig *extGlobalConfig) *cobra.Command { return cmd } + +func newExtPublishCmd(globalConfig *extGlobalConfig) *cobra.Command { + var version string + + cmd := &cobra.Command{ + Use: "publish", + Short: "Publishes the current extension to a registry", + RunE: func(cmd *cobra.Command, args []string) error { + extManager := extensions.NewExtensionsManager(global.NewGlobalStore()) + + if err := extManager.Publish(cmd.Context(), version); err != nil { + return err + } + + return nil + }, + } + + cmd.Flags().StringVar(&version, "version", "", "the version to publish") + cmd.MarkFlagRequired("version") + + return cmd +} diff --git a/internal/extensions/downloader.go b/internal/extensions/downloader.go index 6f3156d..dcd24f5 100644 --- a/internal/extensions/downloader.go +++ b/internal/extensions/downloader.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log" "net/http" "os" "time" @@ -60,11 +61,16 @@ func (d *gitHubReleaseDownloader) Download(ctx context.Context, dest string) err } defer resp.Body.Close() + if err := os.RemoveAll(dest); err != nil { + log.Printf("failed to remove extension before downloading new: %s", err.Error()) + } + extensionBinary, err := os.Create(dest) if err != nil { return err } defer extensionBinary.Close() + extensionBinary.Chmod(0o755) if _, err := io.Copy(extensionBinary, resp.Body); err != nil { return err diff --git a/internal/extensions/extension.go b/internal/extensions/extension.go index 3335d34..aa104e1 100644 --- a/internal/extensions/extension.go +++ b/internal/extensions/extension.go @@ -35,7 +35,8 @@ func (e *Extension) Ensure(ctx context.Context) error { binaryPath := path.Join(extensionsCachePath, binaryName) if exists(binaryPath) { - return nil + // TODO: do a checksum chck + //return nil } downloadLink := e.getRemoteBinaryDownloadLink() diff --git a/internal/extensions/extension_source.go b/internal/extensions/extension_source.go new file mode 100644 index 0000000..d9727f8 --- /dev/null +++ b/internal/extensions/extension_source.go @@ -0,0 +1,44 @@ +package extensions + +import ( + "context" + "fmt" + "os" + + "gopkg.in/yaml.v2" +) + +type shuttleExtensionsRegistry struct { + GitHub *string `json:"github" yaml:"github"` +} + +type shuttleExtensionProviderGitHubRelease struct { + Owner string `json:"owner" yaml:"owner"` + Repo string `json:"repo" yaml:"repo"` +} + +type shuttleExtensionsProvider struct { + GitHubRelease *shuttleExtensionProviderGitHubRelease `json:"github-release" yaml:"github-release"` +} + +type shuttleExtensionsFile struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + + Provider shuttleExtensionsProvider `json:"provider" yaml:"provider"` + Registry shuttleExtensionsRegistry `json:"registry" yaml:"registry"` +} + +func getExtensionsFile(ctx context.Context) (*shuttleExtensionsFile, error) { + templateFileContent, err := os.ReadFile("shuttle.template.yaml") + if err != nil { + return nil, fmt.Errorf("failed to find shuttle.template.yaml: %w", err) + } + + var templateFile shuttleExtensionsFile + if err := yaml.Unmarshal(templateFileContent, &templateFile); err != nil { + return nil, fmt.Errorf("failed to parse shuttle.template.yaml: %w", err) + } + + return &templateFile, nil +} diff --git a/internal/extensions/extensions.go b/internal/extensions/extensions.go index ef8ae35..2825310 100644 --- a/internal/extensions/extensions.go +++ b/internal/extensions/extensions.go @@ -74,7 +74,7 @@ func (e *ExtensionsManager) Install(ctx context.Context) error { // Update will fetch the latest extensions from a registry and install them afterwards so that they're ready for use func (e *ExtensionsManager) Update(ctx context.Context, registry string) error { - reg, err := NewRegistry(registry, e.globalStore) + reg, err := NewRegistryFromCombined(registry, e.globalStore) if err != nil { return fmt.Errorf("failed to update extensions: %w", err) } @@ -89,3 +89,21 @@ func (e *ExtensionsManager) Update(ctx context.Context, registry string) error { return nil } + +func (e *ExtensionsManager) Publish(ctx context.Context, version string) error { + extensionsFile, err := getExtensionsFile(ctx) + if err != nil { + return err + } + + registry, err := NewRegistry("github", "", e.globalStore) + if err != nil { + return err + } + + if err := registry.Publish(ctx, extensionsFile, version); err != nil { + return err + } + + return nil +} diff --git a/internal/extensions/git_registry.go b/internal/extensions/git_registry.go index f4630af..41d5656 100644 --- a/internal/extensions/git_registry.go +++ b/internal/extensions/git_registry.go @@ -16,6 +16,11 @@ type gitRegistry struct { globalStore *global.GlobalStore } +// Publish isn't implemented yet for gitRegistry +func (*gitRegistry) Publish(ctx context.Context, extFile *shuttleExtensionsFile, version string) error { + panic("unimplemented") +} + func (*gitRegistry) Get(ctx context.Context) error { panic("unimplemented") } diff --git a/internal/extensions/github_registry.go b/internal/extensions/github_registry.go new file mode 100644 index 0000000..d1b23bb --- /dev/null +++ b/internal/extensions/github_registry.go @@ -0,0 +1,330 @@ +package extensions + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "strings" +) + +type gitHubRegistry struct { + client *githubClient +} + +func (g *gitHubRegistry) Publish(ctx context.Context, extFile *shuttleExtensionsFile, version string) error { + release, err := g.client.GetRelease(ctx, extFile, version) + if err != nil { + return err + } + + sha, err := g.client.GetFile(ctx, extFile) + if err != nil { + log.Printf("failed to find file: %s", err.Error()) + // Ignore file as it probably means that the file wasn't there + } + + if err := g.client.UpsertFile(ctx, extFile, release, version, sha); err != nil { + return err + } + + log.Println("done") + return nil +} + +// Get isn't implemented yet for GitHubRegistry +func (*gitHubRegistry) Get(ctx context.Context) error { + panic("unimplemented") +} + +// Update isn't implemented yet for GitHubRegistry +func (*gitHubRegistry) Update(ctx context.Context) error { + panic("unimplemented") +} + +func newGitHubRegistry() (Registry, error) { + client, err := newGitHubClient() + if err != nil { + return nil, err + } + + return &gitHubRegistry{ + client: client, + }, nil +} + +type githubClient struct { + accessToken string + httpClient *http.Client +} + +func newGitHubClient() (*githubClient, error) { + var token string + if accessToken := os.Getenv("SHUTTLE_EXTENSIONS_GITHUB_ACCESS_TOKEN"); accessToken != "" { + token = accessToken + } else if accessToken := os.Getenv("GITHUB_ACCESS_TOKEN"); accessToken != "" { + token = accessToken + } + + if token == "" { + return nil, errors.New("GITHUB_ACCESS_TOKEN was not set") + } + + return &githubClient{ + accessToken: token, + httpClient: http.DefaultClient, + }, nil +} + +func (gc *githubClient) GetFile(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile) (string, error) { + owner, repo, ok := strings.Cut(*shuttleExtensionsFile.Registry.GitHub, "/") + if !ok { + return "", fmt.Errorf("failed to find owner and repo in registry: %s", *shuttleExtensionsFile.Registry.GitHub) + } + + extensionsFile, err := githubClientDo[any, githubFileShaResp]( + ctx, + gc, + http.MethodGet, + fmt.Sprintf( + "/repos/%s/%s/contents/%s", + owner, + repo, + getRemoteRegistryExtensionPathFile(shuttleExtensionsFile.Name), + ), + nil, + ) + if err != nil { + return "", err + } + + return extensionsFile.Sha, nil +} + +func (gc *githubClient) UpsertFile(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile, releaseInformation *githubReleaseInformation, version string, sha string) error { + registryExtensionsReq := registryExtension{ + Name: shuttleExtensionsFile.Name, + Description: shuttleExtensionsFile.Description, + Version: version, + DownloadUrls: make([]registryExtensionDownloadLink, 0), + } + + for _, releaseAsset := range releaseInformation.Assets { + arch, os, err := releaseAsset.ParseDownloadLink(shuttleExtensionsFile.Name) + if err != nil { + log.Printf("file did not match an actual binary: %s, %s", releaseAsset.DownloadUrl, err.Error()) + continue + } + + downloadLink := registryExtensionDownloadLink{ + Architecture: arch, + Os: os, + Url: releaseAsset.DownloadUrl, + Provider: "github-release", + } + + registryExtensionsReq.DownloadUrls = append(registryExtensionsReq.DownloadUrls, downloadLink) + } + + upsertRequest, err := newGitHubUpsertRequest( + shuttleExtensionsFile.Name, + version, + registryExtensionsReq, + sha, + ) + if err != nil { + return err + } + + owner, repo, ok := strings.Cut(*shuttleExtensionsFile.Registry.GitHub, "/") + if !ok { + return fmt.Errorf("failed to find owner and repo in registry: %s", *shuttleExtensionsFile.Registry.GitHub) + } + + _, err = githubClientDo[githubUpsertFileRequest, any]( + ctx, + gc, + http.MethodPut, + fmt.Sprintf( + "/repos/%s/%s/contents/%s", + owner, + repo, + getRemoteRegistryExtensionPathFile(shuttleExtensionsFile.Name), + ), + upsertRequest, + ) + if err != nil { + return err + } + + return nil +} + +func (gc *githubClient) GetRelease(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile, version string) (*githubReleaseInformation, error) { + release, err := githubClientDo[any, githubReleaseInformation]( + ctx, + gc, + http.MethodGet, + fmt.Sprintf( + "/repos/%s/%s/releases/tags/%s", + shuttleExtensionsFile.Provider.GitHubRelease.Owner, + shuttleExtensionsFile.Provider.GitHubRelease.Repo, + version, + ), + nil, + ) + if err != nil { + return nil, err + } + + if len(release.Assets) == 0 { + return nil, errors.New("found no releases for github release") + } + + return release, nil +} + +type githubReleaseAsset struct { + DownloadUrl string `json:"browser_download_url"` +} + +func (gra *githubReleaseAsset) ParseDownloadLink(name string) (arch string, os string, err error) { + components := strings.Split(gra.DownloadUrl, "/") + if len(components) < 3 { + return "", "", errors.New("failed to find a proper github download link") + } + + file := components[len(components)-1] + + rest, ok := strings.CutPrefix(file, name) + if !ok { + return "", "", errors.New("file link did not contain extension name") + } + rest = strings.TrimPrefix(rest, "-") + + os, arch, ok = strings.Cut(rest, "-") + if !ok { + return "", "", errors.New("file did not match os-arch") + } + + return arch, os, nil +} + +type githubReleaseInformation struct { + Assets []githubReleaseAsset `json:"assets"` +} + +type githubFileShaResp struct { + Sha string `json:"sha"` +} + +type githubCommitter struct { + Name string `json:"name"` + Email string `json:"email"` +} + +type githubUpsertFileRequest struct { + Message string `json:"message"` + Committer githubCommitter `json:"committer"` + Content string `json:"content"` + Sha *string `json:"sha"` +} + +func newGitHubUpsertRequest(name string, version string, registryExtensionsReq registryExtension, sha string) (*githubUpsertFileRequest, error) { + committerName := os.Getenv("GITHUB_COMMITTER_NAME") + if committerName == "" { + return nil, errors.New("GITHUB_COMMITTER_NAME was not found") + } + + committerEmail := os.Getenv("GITHUB_COMMITTER_EMAIL") + if committerEmail == "" { + return nil, errors.New("GITHUB_COMMITTER_EMAIL was not found") + } + + content, err := json.MarshalIndent(registryExtensionsReq, "", " ") + if err != nil { + return nil, err + } + + contentB64 := base64.StdEncoding.EncodeToString(content) + + req := &githubUpsertFileRequest{ + Message: fmt.Sprintf("chore(extensions): updating %s to %s", name, version), + Committer: githubCommitter{ + Name: committerName, + Email: committerEmail, + }, + Content: contentB64, + } + + if sha != "" { + req.Sha = &sha + } + + return req, nil +} + +func githubClientDo[TReq any, TResp any](ctx context.Context, githubClient *githubClient, method string, path string, reqBody *TReq) (*TResp, error) { + var bodyReader io.Reader + if reqBody != nil { + contents, err := json.Marshal(reqBody) + if err != nil { + return nil, err + } + + bodyReader = bytes.NewReader(contents) + } + + url := fmt.Sprintf("https://api.github.com/%s", strings.TrimPrefix(path, "/")) + + req, err := http.NewRequestWithContext( + ctx, + method, + url, + bodyReader, + ) + if err != nil { + return nil, err + } + + req.Header.Add("Accept", "application/vnd.github+json") + req.Header.Add("X-GitHub-Api-Version", "2022-11-28") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", githubClient.accessToken)) + + resp, err := githubClient.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respContent, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode > 399 { + var message githubMessage + if err := json.Unmarshal(respContent, &message); err != nil { + return nil, fmt.Errorf("failed to unmarshal resp: %w", err) + } + + return nil, fmt.Errorf("failed github request with: %s", message.Message) + } + + var returnObject TResp + if err := json.Unmarshal(respContent, &returnObject); err != nil { + return nil, err + } + + return &returnObject, nil +} + +type githubMessage struct { + Message string `json:"message"` +} diff --git a/internal/extensions/registry.go b/internal/extensions/registry.go index 13fcf21..5edcd49 100644 --- a/internal/extensions/registry.go +++ b/internal/extensions/registry.go @@ -12,10 +12,11 @@ import ( type Registry interface { Get(ctx context.Context) error Update(ctx context.Context) error + Publish(ctx context.Context, extFile *shuttleExtensionsFile, version string) error } -// NewRegistry is a shim for concrete implementations of the registries, such as gitRegistry -func NewRegistry(registry string, globalStore *global.GlobalStore) (Registry, error) { +// NewRegistryFromCombined is a shim for concrete implementations of the registries, such as gitRegistry +func NewRegistryFromCombined(registry string, globalStore *global.GlobalStore) (Registry, error) { registryType, registryUrl, ok := strings.Cut(registry, "=") if !ok { return nil, fmt.Errorf("registry was not a valid url: %s", registry) @@ -28,3 +29,15 @@ func NewRegistry(registry string, globalStore *global.GlobalStore) (Registry, er return nil, fmt.Errorf("registry type was not valid: %s", registryType) } } + +// NewRegistry is a shim for concrete implementations of the registries, such as gitRegistry +func NewRegistry(registryType string, registryUrl string, globalStore *global.GlobalStore) (Registry, error) { + switch registryType { + case "git": + return newGitRegistry(registryUrl, globalStore), nil + case "github": + return newGitHubRegistry() + default: + return nil, fmt.Errorf("registry type was not valid: %s", registryType) + } +} diff --git a/internal/extensions/remote_registry.go b/internal/extensions/remote_registry.go new file mode 100644 index 0000000..febb62e --- /dev/null +++ b/internal/extensions/remote_registry.go @@ -0,0 +1,9 @@ +package extensions + +import "context" + +type RemoteRegistry interface { + Publish(ctx context.Context) error +} + +func NewRemoteRegistry(registry string) {} diff --git a/internal/extensions/remote_registry_paths.go b/internal/extensions/remote_registry_paths.go new file mode 100644 index 0000000..fbfc180 --- /dev/null +++ b/internal/extensions/remote_registry_paths.go @@ -0,0 +1,15 @@ +package extensions + +import "path" + +func getRemoteRegistryIndex() string { + return "index" +} + +func getRemoteRegistryExtensionPath(name string) string { + return path.Join(getRemoteRegistryIndex(), name) +} + +func getRemoteRegistryExtensionPathFile(name string) string { + return path.Join(getRemoteRegistryExtensionPath(name), "shuttle-extension.json") +} From 4a6d4852e4fe3617e8abc592952c081e9b49e7ee Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Fri, 15 Mar 2024 11:23:16 +0100 Subject: [PATCH 2/6] fix: shuttle extensions Signed-off-by: Kasper J. Hermansen --- internal/extensions/extensions.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/extensions/extensions.go b/internal/extensions/extensions.go index 2825310..02116ef 100644 --- a/internal/extensions/extensions.go +++ b/internal/extensions/extensions.go @@ -36,6 +36,8 @@ func (e *ExtensionsManager) GetAll(ctx context.Context) ([]Extension, error) { extensions := make([]Extension, 0) for _, registryExtension := range registryExtensions { + registryExtension := registryExtension + extension, err := newExtensionFromRegistry(e.globalStore, ®istryExtension) if err != nil { return nil, err From d9807b8f683f19138b528f35624db0258d9bab16 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Fri, 17 May 2024 15:48:27 +0200 Subject: [PATCH 3/6] feat: can download private files Signed-off-by: Kasper J. Hermansen --- internal/extensions/downloader.go | 23 ++++++++++++++++++++++- internal/extensions/github_registry.go | 3 ++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/extensions/downloader.go b/internal/extensions/downloader.go index dcd24f5..36e89c6 100644 --- a/internal/extensions/downloader.go +++ b/internal/extensions/downloader.go @@ -8,6 +8,8 @@ import ( "log" "net/http" "os" + "os/exec" + "strings" "time" ) @@ -48,13 +50,17 @@ func (d *gitHubReleaseDownloader) Download(ctx context.Context, dest string) err bearer = accessToken } else if accessToken := os.Getenv("GITHUB_ACCESS_TOKEN"); accessToken != "" { bearer = accessToken + } else if accessToken, ok := getToken(); ok { + bearer = accessToken } if bearer == "" { return errors.New("failed to find a valid authorization token for github. Please make sure you're logged into github-cli (gh), or have followed the setup documentation") } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", bearer)) + req.Header.Add("Authorization", fmt.Sprintf("token %s", bearer)) + req.Header.Add("Accept", "application/octet-stream") + resp, err := client.Do(req) if err != nil { return err @@ -78,3 +84,18 @@ func (d *gitHubReleaseDownloader) Download(ctx context.Context, dest string) err return nil } + +func getToken() (string, bool) { + tokenRaw, err := exec.Command("gh", "auth", "token").Output() + if err != nil { + return "", false + } + + token := string(tokenRaw) + + if token != "" { + return strings.TrimSpace(token), false + } + + return "", false +} diff --git a/internal/extensions/github_registry.go b/internal/extensions/github_registry.go index d1b23bb..fe73fa8 100644 --- a/internal/extensions/github_registry.go +++ b/internal/extensions/github_registry.go @@ -125,7 +125,7 @@ func (gc *githubClient) UpsertFile(ctx context.Context, shuttleExtensionsFile *s downloadLink := registryExtensionDownloadLink{ Architecture: arch, Os: os, - Url: releaseAsset.DownloadUrl, + Url: releaseAsset.Url, Provider: "github-release", } @@ -192,6 +192,7 @@ func (gc *githubClient) GetRelease(ctx context.Context, shuttleExtensionsFile *s type githubReleaseAsset struct { DownloadUrl string `json:"browser_download_url"` + Url string `json:"url"` } func (gra *githubReleaseAsset) ParseDownloadLink(name string) (arch string, os string, err error) { From dea7d012d283959cc7186347496a47796df21cb8 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 13 Jun 2024 12:57:27 +0200 Subject: [PATCH 4/6] feat: remove fluff Signed-off-by: Kasper J. Hermansen --- cmd/ext.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/ext.go b/cmd/ext.go index d266676..2422b38 100644 --- a/cmd/ext.go +++ b/cmd/ext.go @@ -150,6 +150,7 @@ func newExtInitCmd(globalConfig *extGlobalConfig) *cobra.Command { func newExtPublishCmd(globalConfig *extGlobalConfig) *cobra.Command { var version string + // Publish can either be called by a user to rollback an extension, or by CI to automatically publish an extension. cmd := &cobra.Command{ Use: "publish", Short: "Publishes the current extension to a registry", From 6f64c93e45db2b63f5f43ed188abb898bc26f6cb Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 13 Jun 2024 14:40:14 +0200 Subject: [PATCH 5/6] feat: fix review comments Signed-off-by: Kasper J. Hermansen --- internal/extensions/downloader.go | 34 +++----------------- internal/extensions/extension_source.go | 2 +- internal/extensions/extensions.go | 2 +- internal/extensions/github_registry.go | 29 ++++------------- internal/extensions/github_token.go | 42 +++++++++++++++++++++++++ internal/extensions/registry.go | 12 ------- internal/extensions/registry_index.go | 2 +- 7 files changed, 55 insertions(+), 68 deletions(-) create mode 100644 internal/extensions/github_token.go diff --git a/internal/extensions/downloader.go b/internal/extensions/downloader.go index 36e89c6..0e5250f 100644 --- a/internal/extensions/downloader.go +++ b/internal/extensions/downloader.go @@ -2,14 +2,11 @@ package extensions import ( "context" - "errors" "fmt" "io" "log" "net/http" "os" - "os/exec" - "strings" "time" ) @@ -45,17 +42,9 @@ func (d *gitHubReleaseDownloader) Download(ctx context.Context, dest string) err return err } - var bearer string - if accessToken := os.Getenv("SHUTTLE_EXTENSIONS_GITHUB_ACCESS_TOKEN"); accessToken != "" { - bearer = accessToken - } else if accessToken := os.Getenv("GITHUB_ACCESS_TOKEN"); accessToken != "" { - bearer = accessToken - } else if accessToken, ok := getToken(); ok { - bearer = accessToken - } - - if bearer == "" { - return errors.New("failed to find a valid authorization token for github. Please make sure you're logged into github-cli (gh), or have followed the setup documentation") + bearer, err := getGithubToken() + if err != nil { + return err } req.Header.Add("Authorization", fmt.Sprintf("token %s", bearer)) @@ -68,7 +57,7 @@ func (d *gitHubReleaseDownloader) Download(ctx context.Context, dest string) err defer resp.Body.Close() if err := os.RemoveAll(dest); err != nil { - log.Printf("failed to remove extension before downloading new: %s", err.Error()) + log.Printf("failed to remove extension before downloading new: %s, please try again", err.Error()) } extensionBinary, err := os.Create(dest) @@ -84,18 +73,3 @@ func (d *gitHubReleaseDownloader) Download(ctx context.Context, dest string) err return nil } - -func getToken() (string, bool) { - tokenRaw, err := exec.Command("gh", "auth", "token").Output() - if err != nil { - return "", false - } - - token := string(tokenRaw) - - if token != "" { - return strings.TrimSpace(token), false - } - - return "", false -} diff --git a/internal/extensions/extension_source.go b/internal/extensions/extension_source.go index d9727f8..414ec8f 100644 --- a/internal/extensions/extension_source.go +++ b/internal/extensions/extension_source.go @@ -29,7 +29,7 @@ type shuttleExtensionsFile struct { Registry shuttleExtensionsRegistry `json:"registry" yaml:"registry"` } -func getExtensionsFile(ctx context.Context) (*shuttleExtensionsFile, error) { +func getExtensionsFile(_ context.Context) (*shuttleExtensionsFile, error) { templateFileContent, err := os.ReadFile("shuttle.template.yaml") if err != nil { return nil, fmt.Errorf("failed to find shuttle.template.yaml: %w", err) diff --git a/internal/extensions/extensions.go b/internal/extensions/extensions.go index 02116ef..25decf8 100644 --- a/internal/extensions/extensions.go +++ b/internal/extensions/extensions.go @@ -98,7 +98,7 @@ func (e *ExtensionsManager) Publish(ctx context.Context, version string) error { return err } - registry, err := NewRegistry("github", "", e.globalStore) + registry, err := newGitHubRegistry() if err != nil { return err } diff --git a/internal/extensions/github_registry.go b/internal/extensions/github_registry.go index fe73fa8..d6058e7 100644 --- a/internal/extensions/github_registry.go +++ b/internal/extensions/github_registry.go @@ -24,9 +24,9 @@ func (g *gitHubRegistry) Publish(ctx context.Context, extFile *shuttleExtensions return err } - sha, err := g.client.GetFile(ctx, extFile) + sha, err := g.client.GetFileSHA(ctx, extFile) if err != nil { - log.Printf("failed to find file: %s", err.Error()) + // TODO: Send error as a debug to log file somewhere // Ignore file as it probably means that the file wasn't there } @@ -34,20 +34,9 @@ func (g *gitHubRegistry) Publish(ctx context.Context, extFile *shuttleExtensions return err } - log.Println("done") return nil } -// Get isn't implemented yet for GitHubRegistry -func (*gitHubRegistry) Get(ctx context.Context) error { - panic("unimplemented") -} - -// Update isn't implemented yet for GitHubRegistry -func (*gitHubRegistry) Update(ctx context.Context) error { - panic("unimplemented") -} - func newGitHubRegistry() (Registry, error) { client, err := newGitHubClient() if err != nil { @@ -65,15 +54,9 @@ type githubClient struct { } func newGitHubClient() (*githubClient, error) { - var token string - if accessToken := os.Getenv("SHUTTLE_EXTENSIONS_GITHUB_ACCESS_TOKEN"); accessToken != "" { - token = accessToken - } else if accessToken := os.Getenv("GITHUB_ACCESS_TOKEN"); accessToken != "" { - token = accessToken - } - - if token == "" { - return nil, errors.New("GITHUB_ACCESS_TOKEN was not set") + token, err := getGithubToken() + if err != nil { + return nil, err } return &githubClient{ @@ -82,7 +65,7 @@ func newGitHubClient() (*githubClient, error) { }, nil } -func (gc *githubClient) GetFile(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile) (string, error) { +func (gc *githubClient) GetFileSHA(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile) (string, error) { owner, repo, ok := strings.Cut(*shuttleExtensionsFile.Registry.GitHub, "/") if !ok { return "", fmt.Errorf("failed to find owner and repo in registry: %s", *shuttleExtensionsFile.Registry.GitHub) diff --git a/internal/extensions/github_token.go b/internal/extensions/github_token.go new file mode 100644 index 0000000..82243bd --- /dev/null +++ b/internal/extensions/github_token.go @@ -0,0 +1,42 @@ +package extensions + +import ( + "errors" + "os" + "os/exec" + "strings" +) + +func getGithubToken() (string, error) { + if accessToken := os.Getenv("SHUTTLE_EXTENSIONS_GITHUB_ACCESS_TOKEN"); accessToken != "" { + return accessToken, nil + } else if accessToken := os.Getenv("GITHUB_ACCESS_TOKEN"); accessToken != "" { + return accessToken, nil + } else { + accessToken, err := getToken() + if err != nil { + return "", err + } + + return accessToken, nil + } +} + +func getToken() (string, error) { + tokenRaw, err := exec.Command("gh", "auth", "token").Output() + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return "", errors.New("github-cli (gh) is not installed") + } + + return "", err + } + + token := string(tokenRaw) + + if token != "" { + return strings.TrimSpace(token), nil + } + + return "", errors.New("no github token available (please sign in `gh auth login`)") +} diff --git a/internal/extensions/registry.go b/internal/extensions/registry.go index 5edcd49..cab4f58 100644 --- a/internal/extensions/registry.go +++ b/internal/extensions/registry.go @@ -29,15 +29,3 @@ func NewRegistryFromCombined(registry string, globalStore *global.GlobalStore) ( return nil, fmt.Errorf("registry type was not valid: %s", registryType) } } - -// NewRegistry is a shim for concrete implementations of the registries, such as gitRegistry -func NewRegistry(registryType string, registryUrl string, globalStore *global.GlobalStore) (Registry, error) { - switch registryType { - case "git": - return newGitRegistry(registryUrl, globalStore), nil - case "github": - return newGitHubRegistry() - default: - return nil, fmt.Errorf("registry type was not valid: %s", registryType) - } -} diff --git a/internal/extensions/registry_index.go b/internal/extensions/registry_index.go index ea33de3..0cedc62 100644 --- a/internal/extensions/registry_index.go +++ b/internal/extensions/registry_index.go @@ -36,7 +36,7 @@ func newRegistryIndex(registryPath string) *registryIndex { } } -func (r *registryIndex) getExtensions(ctx context.Context) ([]registryExtension, error) { +func (r *registryIndex) getExtensions(_ context.Context) ([]registryExtension, error) { contents, err := os.ReadDir(r.getIndexPath()) if err != nil { return nil, fmt.Errorf("failed to list index in registry: %s, %w", r.getIndexPath(), err) From 6cc92ea01db6fb894c1f5bacb0db2819234919c2 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 13 Jun 2024 14:42:50 +0200 Subject: [PATCH 6/6] feat: it needs to implement the functions Signed-off-by: Kasper J. Hermansen --- internal/extensions/github_registry.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/extensions/github_registry.go b/internal/extensions/github_registry.go index d6058e7..1a775e3 100644 --- a/internal/extensions/github_registry.go +++ b/internal/extensions/github_registry.go @@ -37,6 +37,16 @@ func (g *gitHubRegistry) Publish(ctx context.Context, extFile *shuttleExtensions return nil } +// Get isn't implemented yet for GitHubRegistry +func (*gitHubRegistry) Get(ctx context.Context) error { + panic("unimplemented") +} + +// Update isn't implemented yet for GitHubRegistry +func (*gitHubRegistry) Update(ctx context.Context) error { + panic("unimplemented") +} + func newGitHubRegistry() (Registry, error) { client, err := newGitHubClient() if err != nil {