From 9b72a7a591a42b46af6fdae87b7b00b1272d596e Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 25 Jan 2024 09:22:03 +0100 Subject: [PATCH 1/4] feat: adds basic commands Signed-off-by: Kasper J. Hermansen --- cmd/cmd.go | 2 ++ cmd/ext.go | 56 +++++++++++++++++++++++++++++++ internal/extensions/extensions.go | 31 +++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 cmd/ext.go create mode 100644 internal/extensions/extensions.go diff --git a/cmd/cmd.go b/cmd/cmd.go index bb0868d..f7092bd 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -159,6 +159,8 @@ func initializedRootFromArgs(stdout, stderr io.Writer, args []string) (*cobra.Co // Run and LS will not get closured variables from contextProvider rootCmd.ParseFlags(args) + rootCmd.AddCommand(newExtCmd()) + if isInRepoContext() { runCmd, err := newRun(uii, ctxProvider) if err != nil { diff --git a/cmd/ext.go b/cmd/ext.go new file mode 100644 index 0000000..4206495 --- /dev/null +++ b/cmd/ext.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "github.com/lunarway/shuttle/internal/extensions" + "github.com/spf13/cobra" +) + +func newExtCmd() *cobra.Command { + extManager := extensions.NewExtensionsManager("some registry") + + cmd := &cobra.Command{ + Use: "ext", + Long: "helps you manage shuttle extensions", + } + + cmd.AddCommand( + newExtInstallCmd(extManager), + newExtUpdateCmd(extManager), + newExtInitCmd(extManager), + ) + + return cmd +} + +func newExtInstallCmd(extManager *extensions.ExtensionsManager) *cobra.Command { + cmd := &cobra.Command{ + Use: "install", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + return cmd +} + +func newExtUpdateCmd(extManager *extensions.ExtensionsManager) *cobra.Command { + cmd := &cobra.Command{ + Use: "update", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + return cmd +} + +func newExtInitCmd(extManager *extensions.ExtensionsManager) *cobra.Command { + cmd := &cobra.Command{ + Use: "init", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + return cmd +} diff --git a/internal/extensions/extensions.go b/internal/extensions/extensions.go new file mode 100644 index 0000000..c4872d5 --- /dev/null +++ b/internal/extensions/extensions.go @@ -0,0 +1,31 @@ +package extensions + +import "context" + +type ExtensionsManager struct { + registry string +} + +func NewExtensionsManager(registry string) *ExtensionsManager { + return &ExtensionsManager{ + registry: registry, + } +} + +func (e *ExtensionsManager) Init(ctx context.Context) error { + return nil +} + +func (e *ExtensionsManager) GetAll(ctx context.Context) ([]Extension, error) { + return nil, nil +} + +func (e *ExtensionsManager) Install(ctx context.Context) error { + return nil +} + +func (e *ExtensionsManager) Update(ctx context.Context) error { + return nil +} + +type Extension struct{} From c75a15aa410e400194974fa7bf42f0067f5e1720 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 25 Jan 2024 09:26:03 +0100 Subject: [PATCH 2/4] feat: add some more description Signed-off-by: Kasper J. Hermansen --- cmd/ext.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/ext.go b/cmd/ext.go index 4206495..df88562 100644 --- a/cmd/ext.go +++ b/cmd/ext.go @@ -24,7 +24,8 @@ func newExtCmd() *cobra.Command { func newExtInstallCmd(extManager *extensions.ExtensionsManager) *cobra.Command { cmd := &cobra.Command{ - Use: "install", + Use: "install", + Long: "Install ensures that extensions already known about are downloaded and available", RunE: func(cmd *cobra.Command, args []string) error { return nil }, @@ -35,7 +36,8 @@ func newExtInstallCmd(extManager *extensions.ExtensionsManager) *cobra.Command { func newExtUpdateCmd(extManager *extensions.ExtensionsManager) *cobra.Command { cmd := &cobra.Command{ - Use: "update", + Use: "update", + Long: "Update will fetch the latest version of the extensions from the given registry", RunE: func(cmd *cobra.Command, args []string) error { return nil }, @@ -46,7 +48,8 @@ func newExtUpdateCmd(extManager *extensions.ExtensionsManager) *cobra.Command { func newExtInitCmd(extManager *extensions.ExtensionsManager) *cobra.Command { cmd := &cobra.Command{ - Use: "init", + Use: "init", + Long: "init will create an initial extensions repository", RunE: func(cmd *cobra.Command, args []string) error { return nil }, From 4839a64b6e141ac995a2dfc099f51aed401d9e25 Mon Sep 17 00:00:00 2001 From: "Kasper J. Hermansen" Date: Thu, 13 Jun 2024 11:56:37 +0200 Subject: [PATCH 3/4] docs: reword Signed-off-by: Kasper J. Hermansen --- cmd/ext.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ext.go b/cmd/ext.go index df88562..75e04cc 100644 --- a/cmd/ext.go +++ b/cmd/ext.go @@ -25,7 +25,7 @@ func newExtCmd() *cobra.Command { func newExtInstallCmd(extManager *extensions.ExtensionsManager) *cobra.Command { cmd := &cobra.Command{ Use: "install", - Long: "Install ensures that extensions already known about are downloaded and available", + Long: "Install ensures that extensions are downloaded and available", RunE: func(cmd *cobra.Command, args []string) error { return nil }, From 6b2365f64a5d2899eac59d6ab678fa57c778523a Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Thu, 13 Jun 2024 14:57:49 +0200 Subject: [PATCH 4/4] feat: add git extension registry for shuttle extensions (#221) * feat: add registry and global storage Signed-off-by: Kasper J. Hermansen * feat: remove bullet Signed-off-by: Kasper J. Hermansen * feat: remember to initiate install Signed-off-by: Kasper J. Hermansen * feat: make sure to check permissions Signed-off-by: Kasper J. Hermansen * chore: make registry private Signed-off-by: Kasper J. Hermansen * feat(extensions): with git operations (#222) * feat(extensions): with git operations Quite simply implements fetch and clone to make sure we can pull a registry Signed-off-by: Kasper J. Hermansen * feat: add git registry Signed-off-by: Kasper J. Hermansen * feat(extensions): add install extensions from registry (#223) * "feat(extensions): add install extensions from registry This pr includes everything up to actually downloading the extension from the upstream registry Such as pulling an example registry entry ```json { "name": "SuperExtender", "description": "Enhances your experience with superpowers.", "version": "1.0.3", "downloadUrls": [ { "architecture": "amd64", "os": "linux", "url": "https://example.com/linux/amd64/superextender.tar.gz", "checksum": "efgh5678" }, { "architecture": "x86", "os": "windows", "url": "https://example.com/windows/x86/superextender.zip", "checksum": "ijkl91011" }, { "architecture": "arm64", "os": "darwin", "url": "https://example.com/windows/x86/superextender.zip", "checksum": "ijkl91011" } ] } ``` Which will match on the current os using $GOOS and $GOARCH set by golang itself. We may however have to depend on some other variables if go is not installed by I believe it should work anyway" * feat: print error Signed-off-by: Kasper J. Hermansen * fix: review comments, wording etc. Signed-off-by: Kasper J. Hermansen * feat(extensions): add github release downloader (#224) * feat(extensions): add github release downloader This specifically adds a downloader for github releases it uses either the SHUTTLE_EXTENSIONS_GITHUB_ACCESS_TOKEN or GITHUB_ACCESS_TOKEN for access to the releases The downloader expects a full url from the registry as seen in a previous pr Signed-off-by: Kasper J. Hermansen * chore: reword bearer Signed-off-by: Kasper J. Hermansen * feat(extensions): add execute command (#225) * feat(extensions): add execute command this adds Extensions to the global cmd.go resulting in another section showing up when doing a shuttle --help To execute an extension simply shuttle myExtension where myExtension is the name of the downloaded extension. All args are passed to the child as well Signed-off-by: Kasper J. Hermansen * feat: remember to add error Signed-off-by: Kasper J. Hermansen * feat: without empty line Signed-off-by: Kasper J. Hermansen * feat: adds github remote registry index (#226) * feat: with github registry Signed-off-by: Kasper J. Hermansen * fix: shuttle extensions Signed-off-by: Kasper J. Hermansen * feat: can download private files Signed-off-by: Kasper J. Hermansen * feat: remove fluff Signed-off-by: Kasper J. Hermansen * feat: fix review comments Signed-off-by: Kasper J. Hermansen * feat: it needs to implement the functions Signed-off-by: Kasper J. Hermansen --------- Signed-off-by: Kasper J. Hermansen --------- Signed-off-by: Kasper J. Hermansen --------- Signed-off-by: Kasper J. Hermansen --------- Signed-off-by: Kasper J. Hermansen --------- Signed-off-by: Kasper J. Hermansen --------- Signed-off-by: Kasper J. Hermansen --- cmd/cmd.go | 3 + cmd/ext.go | 135 +++++++- internal/extensions/downloader.go | 75 +++++ internal/extensions/extension.go | 88 +++++ internal/extensions/extension_source.go | 44 +++ internal/extensions/extensions.go | 94 +++++- internal/extensions/git_registry.go | 105 ++++++ internal/extensions/github_registry.go | 324 +++++++++++++++++++ internal/extensions/github_token.go | 42 +++ internal/extensions/global_store_paths.go | 39 +++ internal/extensions/registry.go | 31 ++ internal/extensions/registry_index.go | 72 +++++ internal/extensions/remote_registry.go | 9 + internal/extensions/remote_registry_paths.go | 15 + internal/global/global.go | 41 +++ 15 files changed, 1099 insertions(+), 18 deletions(-) create mode 100644 internal/extensions/downloader.go create mode 100644 internal/extensions/extension.go create mode 100644 internal/extensions/extension_source.go create mode 100644 internal/extensions/git_registry.go create mode 100644 internal/extensions/github_registry.go create mode 100644 internal/extensions/github_token.go create mode 100644 internal/extensions/global_store_paths.go create mode 100644 internal/extensions/registry.go create mode 100644 internal/extensions/registry_index.go create mode 100644 internal/extensions/remote_registry.go create mode 100644 internal/extensions/remote_registry_paths.go create mode 100644 internal/global/global.go diff --git a/cmd/cmd.go b/cmd/cmd.go index f7092bd..577f5ac 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -160,6 +160,9 @@ func initializedRootFromArgs(stdout, stderr io.Writer, args []string) (*cobra.Co rootCmd.ParseFlags(args) rootCmd.AddCommand(newExtCmd()) + if err := addExtensions(rootCmd); err != nil { + uii.Verboseln("failed to register extensions: %s", err.Error()) + } if isInRepoContext() { runCmd, err := newRun(uii, ctxProvider) diff --git a/cmd/ext.go b/cmd/ext.go index 75e04cc..2422b38 100644 --- a/cmd/ext.go +++ b/cmd/ext.go @@ -1,12 +1,81 @@ package cmd import ( + "errors" + "os" + "os/exec" + + stdcontext "context" + "github.com/lunarway/shuttle/internal/extensions" + "github.com/lunarway/shuttle/internal/global" "github.com/spf13/cobra" ) +type extGlobalConfig struct { + registry string +} + +func (c *extGlobalConfig) getRegistry() (string, bool) { + if c.registry != "" { + return c.registry, true + } + + if registryEnv := os.Getenv("SHUTTLE_EXTENSIONS_REGISTRY"); registryEnv != "" { + return registryEnv, true + } + + return "", false +} + +func addExtensions(rootCmd *cobra.Command) error { + extManager := extensions.NewExtensionsManager(global.NewGlobalStore()) + + extensions, err := extManager.GetAll(stdcontext.Background()) + if err != nil { + return err + } + grp := &cobra.Group{ + ID: "extensions", + Title: "Extensions", + } + rootCmd.AddGroup(grp) + for _, extension := range extensions { + extension := extension + + rootCmd.AddCommand( + &cobra.Command{ + Use: extension.Name(), + Short: extension.Description(), + Version: extension.Version(), + GroupID: "extensions", + DisableFlagParsing: true, + RunE: func(cmd *cobra.Command, args []string) error { + extCmd := exec.CommandContext(cmd.Context(), extension.FullPath(), args...) + + extCmd.Stdout = os.Stdout + extCmd.Stderr = os.Stderr + extCmd.Stdin = os.Stdin + + if err := extCmd.Start(); err != nil { + return err + } + + if err := extCmd.Wait(); err != nil { + return err + } + + return nil + }, + }, + ) + } + + return nil +} + func newExtCmd() *cobra.Command { - extManager := extensions.NewExtensionsManager("some registry") + globalConfig := &extGlobalConfig{} cmd := &cobra.Command{ Use: "ext", @@ -14,19 +83,51 @@ func newExtCmd() *cobra.Command { } cmd.AddCommand( - newExtInstallCmd(extManager), - newExtUpdateCmd(extManager), - newExtInitCmd(extManager), + newExtInstallCmd(globalConfig), + newExtUpdateCmd(globalConfig), + newExtInitCmd(globalConfig), + newExtPublishCmd(globalConfig), ) + cmd.PersistentFlags().StringVar(&globalConfig.registry, "registry", "", "the given registry, if not set will default to SHUTTLE_EXTENSIONS_REGISTRY") + return cmd } -func newExtInstallCmd(extManager *extensions.ExtensionsManager) *cobra.Command { +func newExtInstallCmd(globalConfig *extGlobalConfig) *cobra.Command { cmd := &cobra.Command{ Use: "install", Long: "Install ensures that extensions are downloaded and available", RunE: func(cmd *cobra.Command, args []string) error { + extManager := extensions.NewExtensionsManager(global.NewGlobalStore()) + + if err := extManager.Install(cmd.Context()); err != nil { + return err + } + + return nil + }, + } + + return cmd +} + +func newExtUpdateCmd(globalConfig *extGlobalConfig) *cobra.Command { + cmd := &cobra.Command{ + Use: "update", + Short: "Update will fetch the latest version of the extensions from the given registry", + RunE: func(cmd *cobra.Command, args []string) error { + extManager := extensions.NewExtensionsManager(global.NewGlobalStore()) + + registry, ok := globalConfig.getRegistry() + if !ok { + return errors.New("registry is not set") + } + + if err := extManager.Update(cmd.Context(), registry); err != nil { + return err + } + return nil }, } @@ -34,10 +135,10 @@ func newExtInstallCmd(extManager *extensions.ExtensionsManager) *cobra.Command { return cmd } -func newExtUpdateCmd(extManager *extensions.ExtensionsManager) *cobra.Command { +func newExtInitCmd(globalConfig *extGlobalConfig) *cobra.Command { cmd := &cobra.Command{ - Use: "update", - Long: "Update will fetch the latest version of the extensions from the given registry", + Use: "init", + Short: "init will create an initial extensions repository", RunE: func(cmd *cobra.Command, args []string) error { return nil }, @@ -46,14 +147,26 @@ func newExtUpdateCmd(extManager *extensions.ExtensionsManager) *cobra.Command { return cmd } -func newExtInitCmd(extManager *extensions.ExtensionsManager) *cobra.Command { +func newExtPublishCmd(globalConfig *extGlobalConfig) *cobra.Command { + var version string + + // Publish can either be called by a user to rollback an extension, or by CI to automatically publish an extension. cmd := &cobra.Command{ - Use: "init", - Long: "init will create an initial extensions repository", + Use: "publish", + Short: "Publishes the current extension to a registry", RunE: func(cmd *cobra.Command, args []string) error { + extManager := extensions.NewExtensionsManager(global.NewGlobalStore()) + + if err := extManager.Publish(cmd.Context(), version); err != nil { + return err + } + return nil }, } + cmd.Flags().StringVar(&version, "version", "", "the version to publish") + cmd.MarkFlagRequired("version") + return cmd } diff --git a/internal/extensions/downloader.go b/internal/extensions/downloader.go new file mode 100644 index 0000000..0e5250f --- /dev/null +++ b/internal/extensions/downloader.go @@ -0,0 +1,75 @@ +package extensions + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "os" + "time" +) + +type Downloader interface { + Download(ctx context.Context, dest string) error +} + +func NewDownloader(downloadLink *registryExtensionDownloadLink) (Downloader, error) { + switch downloadLink.Provider { + case "github-release": + return newGitHubReleaseDownloader(downloadLink), nil + default: + return nil, fmt.Errorf("invalid provider type: %s", downloadLink.Provider) + } +} + +type gitHubReleaseDownloader struct { + link *registryExtensionDownloadLink +} + +func newGitHubReleaseDownloader(downloadLink *registryExtensionDownloadLink) Downloader { + return &gitHubReleaseDownloader{ + link: downloadLink, + } +} + +func (d *gitHubReleaseDownloader) Download(ctx context.Context, dest string) error { + client := http.DefaultClient + client.Timeout = time.Second * 60 + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.link.Url, nil) + if err != nil { + return err + } + + bearer, err := getGithubToken() + if err != nil { + return err + } + + req.Header.Add("Authorization", fmt.Sprintf("token %s", bearer)) + req.Header.Add("Accept", "application/octet-stream") + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if err := os.RemoveAll(dest); err != nil { + log.Printf("failed to remove extension before downloading new: %s, please try again", err.Error()) + } + + extensionBinary, err := os.Create(dest) + if err != nil { + return err + } + defer extensionBinary.Close() + extensionBinary.Chmod(0o755) + + if _, err := io.Copy(extensionBinary, resp.Body); err != nil { + return err + } + + return nil +} diff --git a/internal/extensions/extension.go b/internal/extensions/extension.go new file mode 100644 index 0000000..aa104e1 --- /dev/null +++ b/internal/extensions/extension.go @@ -0,0 +1,88 @@ +package extensions + +import ( + "context" + "fmt" + "path" + "runtime" + + "github.com/lunarway/shuttle/internal/global" +) + +// Extension is the descriptor of a single extension, it is used to add description to the cli, as well as calling the specific extension in question +type Extension struct { + os string + arch string + globalStore *global.GlobalStore + remote *registryExtension +} + +func newExtensionFromRegistry(globalStore *global.GlobalStore, registryExtension *registryExtension) (*Extension, error) { + return &Extension{ + os: runtime.GOOS, + arch: runtime.GOARCH, + globalStore: globalStore, + remote: registryExtension, + }, nil +} + +func (e *Extension) Ensure(ctx context.Context) error { + extensionsCachePath := getExtensionsCachePath(e.globalStore) + binaryName := e.getExtensionBinaryName() + if err := ensureExists(extensionsCachePath); err != nil { + return fmt.Errorf("failed to create cache path: %w", err) + } + + binaryPath := path.Join(extensionsCachePath, binaryName) + if exists(binaryPath) { + // TODO: do a checksum chck + //return nil + } + + downloadLink := e.getRemoteBinaryDownloadLink() + if downloadLink == nil { + return fmt.Errorf("failed to find a valid extension matching your os and architecture") + } + + downloader, err := NewDownloader(downloadLink) + if err != nil { + return err + } + + if err := downloader.Download(ctx, binaryPath); err != nil { + return err + } + + return nil +} + +func (e *Extension) Name() string { + return e.remote.Name +} + +func (e *Extension) Version() string { + return e.remote.Version +} + +func (e *Extension) Description() string { + return e.remote.Description +} + +func (e *Extension) getExtensionBinaryName() string { + return e.remote.Name +} + +func (e *Extension) FullPath() string { + return path.Join(getExtensionsCachePath(e.globalStore), e.Name()) +} + +func (e *Extension) getRemoteBinaryDownloadLink() *registryExtensionDownloadLink { + for _, download := range e.remote.DownloadUrls { + if download.Os == e.os && + download.Architecture == e.arch { + return &download + } + } + + return nil +} diff --git a/internal/extensions/extension_source.go b/internal/extensions/extension_source.go new file mode 100644 index 0000000..414ec8f --- /dev/null +++ b/internal/extensions/extension_source.go @@ -0,0 +1,44 @@ +package extensions + +import ( + "context" + "fmt" + "os" + + "gopkg.in/yaml.v2" +) + +type shuttleExtensionsRegistry struct { + GitHub *string `json:"github" yaml:"github"` +} + +type shuttleExtensionProviderGitHubRelease struct { + Owner string `json:"owner" yaml:"owner"` + Repo string `json:"repo" yaml:"repo"` +} + +type shuttleExtensionsProvider struct { + GitHubRelease *shuttleExtensionProviderGitHubRelease `json:"github-release" yaml:"github-release"` +} + +type shuttleExtensionsFile struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description" yaml:"description"` + + Provider shuttleExtensionsProvider `json:"provider" yaml:"provider"` + Registry shuttleExtensionsRegistry `json:"registry" yaml:"registry"` +} + +func getExtensionsFile(_ context.Context) (*shuttleExtensionsFile, error) { + templateFileContent, err := os.ReadFile("shuttle.template.yaml") + if err != nil { + return nil, fmt.Errorf("failed to find shuttle.template.yaml: %w", err) + } + + var templateFile shuttleExtensionsFile + if err := yaml.Unmarshal(templateFileContent, &templateFile); err != nil { + return nil, fmt.Errorf("failed to parse shuttle.template.yaml: %w", err) + } + + return &templateFile, nil +} diff --git a/internal/extensions/extensions.go b/internal/extensions/extensions.go index c4872d5..25decf8 100644 --- a/internal/extensions/extensions.go +++ b/internal/extensions/extensions.go @@ -1,31 +1,111 @@ package extensions -import "context" +import ( + "context" + "fmt" + "github.com/lunarway/shuttle/internal/global" +) + +// ExtensionsManager is the entry into installing, updating and using extensions. It is the orchestrator of all the parts that consist of extensions type ExtensionsManager struct { - registry string + globalStore *global.GlobalStore } -func NewExtensionsManager(registry string) *ExtensionsManager { +func NewExtensionsManager(globalStore *global.GlobalStore) *ExtensionsManager { return &ExtensionsManager{ - registry: registry, + globalStore: globalStore, } } +// Init will initialize a repository with a sample extension package func (e *ExtensionsManager) Init(ctx context.Context) error { return nil } +// GetAll will return all known and installed extensions func (e *ExtensionsManager) GetAll(ctx context.Context) ([]Extension, error) { - return nil, nil + registry := getRegistryPath(e.globalStore) + + index := newRegistryIndex(registry) + + registryExtensions, err := index.getExtensions(ctx) + if err != nil { + return nil, fmt.Errorf("failed to install extensions, could not get extensions from index: %w", err) + } + + extensions := make([]Extension, 0) + for _, registryExtension := range registryExtensions { + registryExtension := registryExtension + + extension, err := newExtensionFromRegistry(e.globalStore, ®istryExtension) + if err != nil { + return nil, err + } + + if extension != nil { + extensions = append(extensions, *extension) + } + } + + return extensions, nil } +// Install will ensure that all known extensions are installed and ready for use func (e *ExtensionsManager) Install(ctx context.Context) error { + registry := getRegistryPath(e.globalStore) + index := newRegistryIndex(registry) + extensions, err := index.getExtensions(ctx) + if err != nil { + return fmt.Errorf("failed to install extensions, could not get extensions from index: %w", err) + } + + for _, registryExtension := range extensions { + extension, err := newExtensionFromRegistry(e.globalStore, ®istryExtension) + if err != nil { + return err + } + + if err := extension.Ensure(ctx); err != nil { + return err + } + } + return nil } -func (e *ExtensionsManager) Update(ctx context.Context) error { +// Update will fetch the latest extensions from a registry and install them afterwards so that they're ready for use +func (e *ExtensionsManager) Update(ctx context.Context, registry string) error { + reg, err := NewRegistryFromCombined(registry, e.globalStore) + if err != nil { + return fmt.Errorf("failed to update extensions: %w", err) + } + + if err := reg.Update(ctx); err != nil { + return err + } + + if err := e.Install(ctx); err != nil { + return err + } + return nil } -type Extension struct{} +func (e *ExtensionsManager) Publish(ctx context.Context, version string) error { + extensionsFile, err := getExtensionsFile(ctx) + if err != nil { + return err + } + + registry, err := newGitHubRegistry() + if err != nil { + return err + } + + if err := registry.Publish(ctx, extensionsFile, version); err != nil { + return err + } + + return nil +} diff --git a/internal/extensions/git_registry.go b/internal/extensions/git_registry.go new file mode 100644 index 0000000..41d5656 --- /dev/null +++ b/internal/extensions/git_registry.go @@ -0,0 +1,105 @@ +package extensions + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + + "github.com/lunarway/shuttle/internal/global" +) + +// gitRegistry represents a type of registry backed by a remote git registry, whether folder or url based. it is denoted by the variable git=github.com/lunarway/shuttle-extensions.git as an example +type gitRegistry struct { + url string + globalStore *global.GlobalStore +} + +// Publish isn't implemented yet for gitRegistry +func (*gitRegistry) Publish(ctx context.Context, extFile *shuttleExtensionsFile, version string) error { + panic("unimplemented") +} + +func (*gitRegistry) Get(ctx context.Context) error { + panic("unimplemented") +} + +func (g *gitRegistry) Update(ctx context.Context) error { + if g.registryClonedAlready() { + if err := g.fetchGitRepository(ctx); err != nil { + return fmt.Errorf("failed to update registry: %w", err) + } + } else { + registry := getRegistryPath(g.globalStore) + + if err := ensureExists(registry); err != nil { + return fmt.Errorf("failed to create registry path: %w", err) + } + + if err := g.cloneGitRepository(ctx); err != nil { + return fmt.Errorf("failed to clone registry: %w", err) + } + } + + return nil +} + +func newGitRegistry(url string, globalStore *global.GlobalStore) Registry { + return &gitRegistry{ + url: url, + globalStore: globalStore, + } +} + +func (g *gitRegistry) fetchGitRepository(ctx context.Context) error { + registry := getRegistryPath(g.globalStore) + + return g.executeGit(ctx, "git", gitOptions{ + args: []string{ + "pull", + }, + dir: registry, + }) +} + +func (g *gitRegistry) cloneGitRepository(ctx context.Context) error { + registry := getRegistryPath(g.globalStore) + + return g.executeGit(ctx, "git", gitOptions{ + args: []string{ + "clone", + g.url, + registry, + }, + }) + +} + +func (g *gitRegistry) registryClonedAlready() bool { + registry := getRegistryPath(g.globalStore) + + return exists(path.Join(registry, ".git")) +} + +type gitOptions struct { + args []string + dir string +} + +func (g *gitRegistry) executeGit(ctx context.Context, name string, gitOptions gitOptions) error { + cmd := exec.CommandContext(ctx, name, gitOptions.args...) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if gitOptions.dir != "" { + cmd.Dir = gitOptions.dir + } + + if err := cmd.Start(); err != nil { + return err + } + + return cmd.Wait() +} diff --git a/internal/extensions/github_registry.go b/internal/extensions/github_registry.go new file mode 100644 index 0000000..1a775e3 --- /dev/null +++ b/internal/extensions/github_registry.go @@ -0,0 +1,324 @@ +package extensions + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "strings" +) + +type gitHubRegistry struct { + client *githubClient +} + +func (g *gitHubRegistry) Publish(ctx context.Context, extFile *shuttleExtensionsFile, version string) error { + release, err := g.client.GetRelease(ctx, extFile, version) + if err != nil { + return err + } + + sha, err := g.client.GetFileSHA(ctx, extFile) + if err != nil { + // TODO: Send error as a debug to log file somewhere + // Ignore file as it probably means that the file wasn't there + } + + if err := g.client.UpsertFile(ctx, extFile, release, version, sha); err != nil { + return err + } + + return nil +} + +// Get isn't implemented yet for GitHubRegistry +func (*gitHubRegistry) Get(ctx context.Context) error { + panic("unimplemented") +} + +// Update isn't implemented yet for GitHubRegistry +func (*gitHubRegistry) Update(ctx context.Context) error { + panic("unimplemented") +} + +func newGitHubRegistry() (Registry, error) { + client, err := newGitHubClient() + if err != nil { + return nil, err + } + + return &gitHubRegistry{ + client: client, + }, nil +} + +type githubClient struct { + accessToken string + httpClient *http.Client +} + +func newGitHubClient() (*githubClient, error) { + token, err := getGithubToken() + if err != nil { + return nil, err + } + + return &githubClient{ + accessToken: token, + httpClient: http.DefaultClient, + }, nil +} + +func (gc *githubClient) GetFileSHA(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile) (string, error) { + owner, repo, ok := strings.Cut(*shuttleExtensionsFile.Registry.GitHub, "/") + if !ok { + return "", fmt.Errorf("failed to find owner and repo in registry: %s", *shuttleExtensionsFile.Registry.GitHub) + } + + extensionsFile, err := githubClientDo[any, githubFileShaResp]( + ctx, + gc, + http.MethodGet, + fmt.Sprintf( + "/repos/%s/%s/contents/%s", + owner, + repo, + getRemoteRegistryExtensionPathFile(shuttleExtensionsFile.Name), + ), + nil, + ) + if err != nil { + return "", err + } + + return extensionsFile.Sha, nil +} + +func (gc *githubClient) UpsertFile(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile, releaseInformation *githubReleaseInformation, version string, sha string) error { + registryExtensionsReq := registryExtension{ + Name: shuttleExtensionsFile.Name, + Description: shuttleExtensionsFile.Description, + Version: version, + DownloadUrls: make([]registryExtensionDownloadLink, 0), + } + + for _, releaseAsset := range releaseInformation.Assets { + arch, os, err := releaseAsset.ParseDownloadLink(shuttleExtensionsFile.Name) + if err != nil { + log.Printf("file did not match an actual binary: %s, %s", releaseAsset.DownloadUrl, err.Error()) + continue + } + + downloadLink := registryExtensionDownloadLink{ + Architecture: arch, + Os: os, + Url: releaseAsset.Url, + Provider: "github-release", + } + + registryExtensionsReq.DownloadUrls = append(registryExtensionsReq.DownloadUrls, downloadLink) + } + + upsertRequest, err := newGitHubUpsertRequest( + shuttleExtensionsFile.Name, + version, + registryExtensionsReq, + sha, + ) + if err != nil { + return err + } + + owner, repo, ok := strings.Cut(*shuttleExtensionsFile.Registry.GitHub, "/") + if !ok { + return fmt.Errorf("failed to find owner and repo in registry: %s", *shuttleExtensionsFile.Registry.GitHub) + } + + _, err = githubClientDo[githubUpsertFileRequest, any]( + ctx, + gc, + http.MethodPut, + fmt.Sprintf( + "/repos/%s/%s/contents/%s", + owner, + repo, + getRemoteRegistryExtensionPathFile(shuttleExtensionsFile.Name), + ), + upsertRequest, + ) + if err != nil { + return err + } + + return nil +} + +func (gc *githubClient) GetRelease(ctx context.Context, shuttleExtensionsFile *shuttleExtensionsFile, version string) (*githubReleaseInformation, error) { + release, err := githubClientDo[any, githubReleaseInformation]( + ctx, + gc, + http.MethodGet, + fmt.Sprintf( + "/repos/%s/%s/releases/tags/%s", + shuttleExtensionsFile.Provider.GitHubRelease.Owner, + shuttleExtensionsFile.Provider.GitHubRelease.Repo, + version, + ), + nil, + ) + if err != nil { + return nil, err + } + + if len(release.Assets) == 0 { + return nil, errors.New("found no releases for github release") + } + + return release, nil +} + +type githubReleaseAsset struct { + DownloadUrl string `json:"browser_download_url"` + Url string `json:"url"` +} + +func (gra *githubReleaseAsset) ParseDownloadLink(name string) (arch string, os string, err error) { + components := strings.Split(gra.DownloadUrl, "/") + if len(components) < 3 { + return "", "", errors.New("failed to find a proper github download link") + } + + file := components[len(components)-1] + + rest, ok := strings.CutPrefix(file, name) + if !ok { + return "", "", errors.New("file link did not contain extension name") + } + rest = strings.TrimPrefix(rest, "-") + + os, arch, ok = strings.Cut(rest, "-") + if !ok { + return "", "", errors.New("file did not match os-arch") + } + + return arch, os, nil +} + +type githubReleaseInformation struct { + Assets []githubReleaseAsset `json:"assets"` +} + +type githubFileShaResp struct { + Sha string `json:"sha"` +} + +type githubCommitter struct { + Name string `json:"name"` + Email string `json:"email"` +} + +type githubUpsertFileRequest struct { + Message string `json:"message"` + Committer githubCommitter `json:"committer"` + Content string `json:"content"` + Sha *string `json:"sha"` +} + +func newGitHubUpsertRequest(name string, version string, registryExtensionsReq registryExtension, sha string) (*githubUpsertFileRequest, error) { + committerName := os.Getenv("GITHUB_COMMITTER_NAME") + if committerName == "" { + return nil, errors.New("GITHUB_COMMITTER_NAME was not found") + } + + committerEmail := os.Getenv("GITHUB_COMMITTER_EMAIL") + if committerEmail == "" { + return nil, errors.New("GITHUB_COMMITTER_EMAIL was not found") + } + + content, err := json.MarshalIndent(registryExtensionsReq, "", " ") + if err != nil { + return nil, err + } + + contentB64 := base64.StdEncoding.EncodeToString(content) + + req := &githubUpsertFileRequest{ + Message: fmt.Sprintf("chore(extensions): updating %s to %s", name, version), + Committer: githubCommitter{ + Name: committerName, + Email: committerEmail, + }, + Content: contentB64, + } + + if sha != "" { + req.Sha = &sha + } + + return req, nil +} + +func githubClientDo[TReq any, TResp any](ctx context.Context, githubClient *githubClient, method string, path string, reqBody *TReq) (*TResp, error) { + var bodyReader io.Reader + if reqBody != nil { + contents, err := json.Marshal(reqBody) + if err != nil { + return nil, err + } + + bodyReader = bytes.NewReader(contents) + } + + url := fmt.Sprintf("https://api.github.com/%s", strings.TrimPrefix(path, "/")) + + req, err := http.NewRequestWithContext( + ctx, + method, + url, + bodyReader, + ) + if err != nil { + return nil, err + } + + req.Header.Add("Accept", "application/vnd.github+json") + req.Header.Add("X-GitHub-Api-Version", "2022-11-28") + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", githubClient.accessToken)) + + resp, err := githubClient.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + respContent, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode > 399 { + var message githubMessage + if err := json.Unmarshal(respContent, &message); err != nil { + return nil, fmt.Errorf("failed to unmarshal resp: %w", err) + } + + return nil, fmt.Errorf("failed github request with: %s", message.Message) + } + + var returnObject TResp + if err := json.Unmarshal(respContent, &returnObject); err != nil { + return nil, err + } + + return &returnObject, nil +} + +type githubMessage struct { + Message string `json:"message"` +} diff --git a/internal/extensions/github_token.go b/internal/extensions/github_token.go new file mode 100644 index 0000000..82243bd --- /dev/null +++ b/internal/extensions/github_token.go @@ -0,0 +1,42 @@ +package extensions + +import ( + "errors" + "os" + "os/exec" + "strings" +) + +func getGithubToken() (string, error) { + if accessToken := os.Getenv("SHUTTLE_EXTENSIONS_GITHUB_ACCESS_TOKEN"); accessToken != "" { + return accessToken, nil + } else if accessToken := os.Getenv("GITHUB_ACCESS_TOKEN"); accessToken != "" { + return accessToken, nil + } else { + accessToken, err := getToken() + if err != nil { + return "", err + } + + return accessToken, nil + } +} + +func getToken() (string, error) { + tokenRaw, err := exec.Command("gh", "auth", "token").Output() + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return "", errors.New("github-cli (gh) is not installed") + } + + return "", err + } + + token := string(tokenRaw) + + if token != "" { + return strings.TrimSpace(token), nil + } + + return "", errors.New("no github token available (please sign in `gh auth login`)") +} diff --git a/internal/extensions/global_store_paths.go b/internal/extensions/global_store_paths.go new file mode 100644 index 0000000..f555358 --- /dev/null +++ b/internal/extensions/global_store_paths.go @@ -0,0 +1,39 @@ +package extensions + +import ( + "errors" + "os" + "path" + + "github.com/lunarway/shuttle/internal/global" +) + +func getRegistryPath(globalStore *global.GlobalStore) string { + return path.Join(globalStore.Root(), "registry") +} + +func getExtensionsPath(globalStore *global.GlobalStore) string { + return path.Join(globalStore.Root(), "extensions") +} + +func getExtensionsCachePath(globalStore *global.GlobalStore) string { + return path.Join(getExtensionsPath(globalStore), "cache") +} + +func ensureExists(dirPath string) error { + return os.MkdirAll(dirPath, 0o755) +} + +func exists(dirPath string) bool { + _, err := os.Stat(dirPath) + + if errors.Is(err, os.ErrNotExist) { + return false + } + + if err != nil { + return false + } + + return true +} diff --git a/internal/extensions/registry.go b/internal/extensions/registry.go new file mode 100644 index 0000000..cab4f58 --- /dev/null +++ b/internal/extensions/registry.go @@ -0,0 +1,31 @@ +package extensions + +import ( + "context" + "fmt" + "strings" + + "github.com/lunarway/shuttle/internal/global" +) + +// Registry represents some kind of upstream registry where extension metadata lives, such as which ones should be downloaded, which versions they're on and how to download them +type Registry interface { + Get(ctx context.Context) error + Update(ctx context.Context) error + Publish(ctx context.Context, extFile *shuttleExtensionsFile, version string) error +} + +// NewRegistryFromCombined is a shim for concrete implementations of the registries, such as gitRegistry +func NewRegistryFromCombined(registry string, globalStore *global.GlobalStore) (Registry, error) { + registryType, registryUrl, ok := strings.Cut(registry, "=") + if !ok { + return nil, fmt.Errorf("registry was not a valid url: %s", registry) + } + + switch registryType { + case "git": + return newGitRegistry(registryUrl, globalStore), nil + default: + return nil, fmt.Errorf("registry type was not valid: %s", registryType) + } +} diff --git a/internal/extensions/registry_index.go b/internal/extensions/registry_index.go new file mode 100644 index 0000000..0cedc62 --- /dev/null +++ b/internal/extensions/registry_index.go @@ -0,0 +1,72 @@ +package extensions + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "path" +) + +type ( + registryIndex struct { + registryPath string + } + + registryExtensionDownloadLink struct { + Architecture string `json:"architecture"` + Os string `json:"os"` + Url string `json:"url"` + Checksum string `json:"checksum"` + Provider string `json:"provider"` + } + + registryExtension struct { + Name string `json:"name"` + Description string `json:"description"` + Version string `json:"version"` + DownloadUrls []registryExtensionDownloadLink `json:"downloadUrls"` + } +) + +func newRegistryIndex(registryPath string) *registryIndex { + return ®istryIndex{ + registryPath: registryPath, + } +} + +func (r *registryIndex) getExtensions(_ context.Context) ([]registryExtension, error) { + contents, err := os.ReadDir(r.getIndexPath()) + if err != nil { + return nil, fmt.Errorf("failed to list index in registry: %s, %w", r.getIndexPath(), err) + } + + extensions := make([]registryExtension, 0) + for _, dir := range contents { + if !dir.IsDir() { + continue + } + + extensionPath := path.Join(r.getIndexPath(), dir.Name(), "shuttle-extension.json") + + extensionContent, err := os.ReadFile(extensionPath) + if err != nil { + log.Printf("failed to get extension: %s, skipping extension, the extension might be invalid at: %s, please contact your admin", err.Error(), extensionPath) + continue + } + + var extension registryExtension + if err := json.Unmarshal(extensionContent, &extension); err != nil { + return nil, fmt.Errorf("failed unmarshal extension at path: %s, err: %w", extensionPath, err) + } + + extensions = append(extensions, extension) + } + + return extensions, nil +} + +func (r *registryIndex) getIndexPath() string { + return path.Join(r.registryPath, "index") +} diff --git a/internal/extensions/remote_registry.go b/internal/extensions/remote_registry.go new file mode 100644 index 0000000..febb62e --- /dev/null +++ b/internal/extensions/remote_registry.go @@ -0,0 +1,9 @@ +package extensions + +import "context" + +type RemoteRegistry interface { + Publish(ctx context.Context) error +} + +func NewRemoteRegistry(registry string) {} diff --git a/internal/extensions/remote_registry_paths.go b/internal/extensions/remote_registry_paths.go new file mode 100644 index 0000000..fbfc180 --- /dev/null +++ b/internal/extensions/remote_registry_paths.go @@ -0,0 +1,15 @@ +package extensions + +import "path" + +func getRemoteRegistryIndex() string { + return "index" +} + +func getRemoteRegistryExtensionPath(name string) string { + return path.Join(getRemoteRegistryIndex(), name) +} + +func getRemoteRegistryExtensionPathFile(name string) string { + return path.Join(getRemoteRegistryExtensionPath(name), "shuttle-extension.json") +} diff --git a/internal/global/global.go b/internal/global/global.go new file mode 100644 index 0000000..eae8ace --- /dev/null +++ b/internal/global/global.go @@ -0,0 +1,41 @@ +package global + +import "os" + +// GlobalStore represents the ~/.shuttle folder it acts as an abstraction for said folder and ensures operations against it are consistent and controlled +type GlobalStore struct { + options *GlobalStoreOptions +} + +type GlobalStoreOption func(options *GlobalStoreOptions) + +func WithShuttleConfig(shuttleConfig string) GlobalStoreOption { + return func(options *GlobalStoreOptions) { + options.ShuttleConfig = shuttleConfig + } +} + +type GlobalStoreOptions struct { + ShuttleConfig string +} + +func newDefaultGlobalStoreOptions() *GlobalStoreOptions { + return &GlobalStoreOptions{ + ShuttleConfig: "$HOME/.shuttle", + } +} + +func NewGlobalStore(options ...GlobalStoreOption) *GlobalStore { + defaultOptions := newDefaultGlobalStoreOptions() + for _, opt := range options { + opt(defaultOptions) + } + + return &GlobalStore{ + options: defaultOptions, + } +} + +func (gs *GlobalStore) Root() string { + return os.ExpandEnv(gs.options.ShuttleConfig) +}