From 6735957337d9a91f9aed17ee96ab82a00c8c46e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jer=C3=B3nimo=20Albi?= Date: Tue, 14 Nov 2023 21:41:21 +0100 Subject: [PATCH] feat: add or vendor proto packages from Go dependencies (#3724) * refactor: change `cosmosgen` proto discovery to add extra paths * feat(pkg/cosmosbuf): add Buf mod update support * feat: add third party deps to app's Buf config Third party deps are added to app's Buf config when the dependency is not present and the Go package protos have a Buf config with a `name` assigned. * chore: update changelog * feat: add vendoring support for Go dependencies with proto files * refactor(pkg/cosmosgen): use context as function argument * feat: add `--update-buf-module` flag to generate go commands The flag explicitly enables proto vendoring or Buf config dependency updates. * feat: change scaffolder to update Buf with third party dependencies This updates Buf dependencies and vendors proto files from third party Go dependencies by default for the scaffold commands * feat: add events support to generator * chore: remove invalid TODO comments * chore: add Buf config error variable Co-authored-by: Danilo Pantani * chore: change `--update-buf-module` flag to be persistent Co-authored-by: Danilo Pantani * refactor: enable Buf dep vendoring for generate ts-client command This is required because OpenAPI generation is done using Buf while TS client is being generated. * refactor: add generate proto vendor target All generate commands require support for proto vendoring and Buf dependencies updates. A new target was added to make proto vendoring optional and to avoid using an extra argument in all generate targets. * chore: improve variable usage Co-authored-by: Danilo Pantani * refactor: rename proto vendoring flag to `--enable-proto-vendor` --------- Co-authored-by: Danilo Pantani --- changelog.md | 1 + ignite/cmd/generate.go | 19 ++ ignite/cmd/generate_composables.go | 8 +- ignite/cmd/generate_go.go | 8 +- ignite/cmd/generate_hooks.go | 8 +- ignite/cmd/generate_openapi.go | 8 +- ignite/cmd/generate_pulsar.go | 8 +- ignite/cmd/generate_typescript_client.go | 7 +- ignite/cmd/generate_vuex.go | 8 +- ignite/pkg/cosmosbuf/buf.go | 21 ++ ignite/pkg/cosmosgen/cosmosgen.go | 51 ++- ignite/pkg/cosmosgen/generate.go | 340 ++++++++++++++++---- ignite/pkg/cosmosgen/generate_go.go | 23 +- ignite/pkg/cosmosgen/generate_openapi.go | 27 +- ignite/pkg/cosmosgen/generate_typescript.go | 19 +- ignite/services/chain/generate.go | 18 ++ ignite/services/scaffolder/scaffolder.go | 1 + 17 files changed, 449 insertions(+), 126 deletions(-) diff --git a/changelog.md b/changelog.md index 8085043ab1..20c7a3979d 100644 --- a/changelog.md +++ b/changelog.md @@ -23,6 +23,7 @@ - [#3614](https://github.com/ignite/cli/pull/3614) feat: use DefaultBaseappOptions for app.New method - [#3536](https://github.com/ignite/cli/pull/3536) Change app.go to v2 and add AppWiring feature - [#3670](https://github.com/ignite/cli/pull/3670) Remove nodetime binaries +- [#3724](https://github.com/ignite/cli/pull/3724) Add or vendor proto packages from Go dependencies - [#3715](https://github.com/ignite/cli/pull/3715) Add test suite for the cli tests ### Changes diff --git a/ignite/cmd/generate.go b/ignite/cmd/generate.go index 4b6bec9e0f..9e366c012a 100644 --- a/ignite/cmd/generate.go +++ b/ignite/cmd/generate.go @@ -2,6 +2,11 @@ package ignitecmd import ( "github.com/spf13/cobra" + flag "github.com/spf13/pflag" +) + +const ( + flagEnableProtoVendor = "enable-proto-vendor" ) // NewGenerate returns a command that groups code generation related sub commands. @@ -22,8 +27,11 @@ meant to be edited by hand. PersistentPreRunE: migrationPreRunHandler, } + c.PersistentFlags().AddFlagSet(flagSetEnableProtoVendor()) + flagSetPath(c) flagSetClearCache(c) + c.AddCommand(NewGenerateGo()) c.AddCommand(NewGeneratePulsar()) c.AddCommand(NewGenerateTSClient()) @@ -34,3 +42,14 @@ meant to be edited by hand. return c } + +func flagSetEnableProtoVendor() *flag.FlagSet { + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.Bool(flagEnableProtoVendor, false, "enable proto package vendor for missing Buf dependencies") + return fs +} + +func flagGetEnableProtoVendor(cmd *cobra.Command) bool { + skip, _ := cmd.Flags().GetBool(flagEnableProtoVendor) + return skip +} diff --git a/ignite/cmd/generate_composables.go b/ignite/cmd/generate_composables.go index 67de559cd7..258a9acc47 100644 --- a/ignite/cmd/generate_composables.go +++ b/ignite/cmd/generate_composables.go @@ -44,7 +44,13 @@ func generateComposablesHandler(cmd *cobra.Command, _ []string) error { return err } - if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateComposables(output)); err != nil { + var opts []chain.GenerateTarget + if flagGetEnableProtoVendor(cmd) { + opts = append(opts, chain.GenerateProtoVendor()) + } + + err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateComposables(output), opts...) + if err != nil { return err } diff --git a/ignite/cmd/generate_go.go b/ignite/cmd/generate_go.go index 677f202b8a..263ebd388b 100644 --- a/ignite/cmd/generate_go.go +++ b/ignite/cmd/generate_go.go @@ -39,7 +39,13 @@ func generateGoHandler(cmd *cobra.Command, _ []string) error { return err } - if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateGo()); err != nil { + var opts []chain.GenerateTarget + if flagGetEnableProtoVendor(cmd) { + opts = append(opts, chain.GenerateProtoVendor()) + } + + err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateGo(), opts...) + if err != nil { return err } diff --git a/ignite/cmd/generate_hooks.go b/ignite/cmd/generate_hooks.go index 0af0be9d78..76f125f498 100644 --- a/ignite/cmd/generate_hooks.go +++ b/ignite/cmd/generate_hooks.go @@ -44,7 +44,13 @@ func generateHooksHandler(cmd *cobra.Command, _ []string) error { return err } - if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateHooks(output)); err != nil { + var opts []chain.GenerateTarget + if flagGetEnableProtoVendor(cmd) { + opts = append(opts, chain.GenerateProtoVendor()) + } + + err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateHooks(output), opts...) + if err != nil { return err } diff --git a/ignite/cmd/generate_openapi.go b/ignite/cmd/generate_openapi.go index 6b4e1e00c1..e0795e0d06 100644 --- a/ignite/cmd/generate_openapi.go +++ b/ignite/cmd/generate_openapi.go @@ -39,7 +39,13 @@ func generateOpenAPIHandler(cmd *cobra.Command, _ []string) error { return err } - if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateOpenAPI()); err != nil { + var opts []chain.GenerateTarget + if flagGetEnableProtoVendor(cmd) { + opts = append(opts, chain.GenerateProtoVendor()) + } + + err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateOpenAPI(), opts...) + if err != nil { return err } diff --git a/ignite/cmd/generate_pulsar.go b/ignite/cmd/generate_pulsar.go index 2c05104954..080063d980 100644 --- a/ignite/cmd/generate_pulsar.go +++ b/ignite/cmd/generate_pulsar.go @@ -39,7 +39,13 @@ func generatePulsarHandler(cmd *cobra.Command, _ []string) error { return err } - if err := c.Generate(cmd.Context(), cacheStorage, chain.GeneratePulsar()); err != nil { + var opts []chain.GenerateTarget + if flagGetEnableProtoVendor(cmd) { + opts = append(opts, chain.GenerateProtoVendor()) + } + + err = c.Generate(cmd.Context(), cacheStorage, chain.GeneratePulsar(), opts...) + if err != nil { return err } diff --git a/ignite/cmd/generate_typescript_client.go b/ignite/cmd/generate_typescript_client.go index ed1bc5a1ae..3bb349ca73 100644 --- a/ignite/cmd/generate_typescript_client.go +++ b/ignite/cmd/generate_typescript_client.go @@ -73,7 +73,12 @@ func generateTSClientHandler(cmd *cobra.Command, _ []string) error { return err } - err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateTSClient(output, useCache)) + var opts []chain.GenerateTarget + if flagGetEnableProtoVendor(cmd) { + opts = append(opts, chain.GenerateProtoVendor()) + } + + err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateTSClient(output, useCache), opts...) if err != nil { return err } diff --git a/ignite/cmd/generate_vuex.go b/ignite/cmd/generate_vuex.go index a28ea1ee64..41880b4aa3 100644 --- a/ignite/cmd/generate_vuex.go +++ b/ignite/cmd/generate_vuex.go @@ -45,7 +45,13 @@ func generateVuexHandler(cmd *cobra.Command, _ []string) error { return err } - if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateVuex(output)); err != nil { + var opts []chain.GenerateTarget + if flagGetEnableProtoVendor(cmd) { + opts = append(opts, chain.GenerateProtoVendor()) + } + + err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateVuex(output), opts...) + if err != nil { return err } diff --git a/ignite/pkg/cosmosbuf/buf.go b/ignite/pkg/cosmosbuf/buf.go index 7c4a70500e..b1b5a2c7e4 100644 --- a/ignite/pkg/cosmosbuf/buf.go +++ b/ignite/pkg/cosmosbuf/buf.go @@ -35,17 +35,20 @@ const ( flagOutput = "output" flagErrorFormat = "error-format" flagLogFormat = "log-format" + flagOnly = "only" fmtJSON = "json" // CMDGenerate generate command. CMDGenerate Command = "generate" CMDExport Command = "export" + CMDMod Command = "mod" ) var ( commands = map[Command]struct{}{ CMDGenerate: {}, CMDExport: {}, + CMDMod: {}, } // ErrInvalidCommand indicates an invalid command name. @@ -72,6 +75,24 @@ func (c Command) String() string { return string(c) } +// Update updates module dependencies. +// By default updates all dependencies unless one or more dependencies are specified. +func (b Buf) Update(ctx context.Context, modDir string, dependencies ...string) error { + var flags map[string]string + if dependencies != nil { + flags = map[string]string{ + flagOnly: strings.Join(dependencies, ","), + } + } + + cmd, err := b.generateCommand(CMDMod, flags, "update", modDir) + if err != nil { + return err + } + + return b.runCommand(ctx, cmd...) +} + // Export runs the buf Export command for the files in the proto directory. func (b Buf) Export(ctx context.Context, protoDir, output string) error { // Check if the proto directory is the Cosmos SDK one diff --git a/ignite/pkg/cosmosgen/cosmosgen.go b/ignite/pkg/cosmosgen/cosmosgen.go index 57eb8417b3..1e08ad0006 100644 --- a/ignite/pkg/cosmosgen/cosmosgen.go +++ b/ignite/pkg/cosmosgen/cosmosgen.go @@ -12,12 +12,15 @@ import ( "github.com/ignite/cli/ignite/pkg/cache" "github.com/ignite/cli/ignite/pkg/cosmosanalysis/module" "github.com/ignite/cli/ignite/pkg/cosmosbuf" + "github.com/ignite/cli/ignite/pkg/events" ) // generateOptions used to configure code generation. type generateOptions struct { - includeDirs []string - useCache bool + includeDirs []string + useCache bool + updateBufModule bool + ev events.Bus isGoEnabled bool isPulsarEnabled bool @@ -103,9 +106,24 @@ func IncludeDirs(dirs []string) Option { } } +// UpdateBufModule enables Buf config proto dependencies update. +// This option updates app's Buf config when proto packages or +// Buf modules are found within the Go dependencies. +func UpdateBufModule() Option { + return func(o *generateOptions) { + o.updateBufModule = true + } +} + +// CollectEvents sets an event bus for sending generation feedback events. +func CollectEvents(ev events.Bus) Option { + return func(c *generateOptions) { + c.ev = ev + } +} + // generator generates code for sdk and sdk apps. type generator struct { - ctx context.Context buf cosmosbuf.Buf cacheStorage cache.Storage appPath string @@ -115,9 +133,9 @@ type generator struct { sdkImport string deps []gomodule.Version appModules []module.Module - appIncludes []string + appIncludes protoIncludes thirdModules map[string][]module.Module - thirdModuleIncludes map[string][]string + thirdModuleIncludes map[string]protoIncludes tmpDirs []string } @@ -139,14 +157,13 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir defer b.Cleanup() g := &generator{ - ctx: ctx, buf: b, appPath: appPath, protoDir: protoDir, gomodPath: gomodPath, opts: &generateOptions{}, thirdModules: make(map[string][]module.Module), - thirdModuleIncludes: make(map[string][]string), + thirdModuleIncludes: make(map[string]protoIncludes), cacheStorage: cacheStorage, } @@ -156,31 +173,41 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir apply(g.opts) } - if err := g.setup(); err != nil { + if err := g.setup(ctx); err != nil { return err } + // Update app's Buf config for third party discovered proto modules. + // Go dependency packages might contain proto files which could also + // optionally be using Buf, so for those cases the discovered proto + // files should be available before code generation. + if g.opts.updateBufModule { + if err := g.updateBufModule(ctx); err != nil { + return err + } + } + // Go generation must run first so the types are created before other // generated code that requires sdk.Msg implementations to be defined if g.opts.isGoEnabled { - if err := g.generateGo(); err != nil { + if err := g.generateGo(ctx); err != nil { return err } } if g.opts.isPulsarEnabled { - if err := g.generatePulsar(); err != nil { + if err := g.generatePulsar(ctx); err != nil { return err } } if g.opts.specOut != "" { - if err := g.generateOpenAPISpec(); err != nil { + if err := g.generateOpenAPISpec(ctx); err != nil { return err } } if g.opts.jsOut != nil { - if err := g.generateTS(); err != nil { + if err := g.generateTS(ctx); err != nil { return err } } diff --git a/ignite/pkg/cosmosgen/generate.go b/ignite/pkg/cosmosgen/generate.go index 05cc679089..9fcac8cceb 100644 --- a/ignite/pkg/cosmosgen/generate.go +++ b/ignite/pkg/cosmosgen/generate.go @@ -2,40 +2,77 @@ package cosmosgen import ( "bytes" + "context" + "fmt" "io/fs" "os" "path/filepath" + "slices" "strings" "github.com/pkg/errors" + "gopkg.in/yaml.v2" "github.com/ignite/cli/ignite/pkg/cache" + "github.com/ignite/cli/ignite/pkg/cliui/colors" + "github.com/ignite/cli/ignite/pkg/cliui/icons" "github.com/ignite/cli/ignite/pkg/cmdrunner" "github.com/ignite/cli/ignite/pkg/cmdrunner/step" "github.com/ignite/cli/ignite/pkg/cosmosanalysis/module" "github.com/ignite/cli/ignite/pkg/cosmosbuf" "github.com/ignite/cli/ignite/pkg/cosmosver" + "github.com/ignite/cli/ignite/pkg/events" "github.com/ignite/cli/ignite/pkg/gomodule" "github.com/ignite/cli/ignite/pkg/xfilepath" + "github.com/ignite/cli/ignite/pkg/xos" ) const ( moduleCacheNamespace = "generate.setup.module" includeProtoCacheNamespace = "generator.includes.proto" + workFilename = "buf.work.yaml" ) -var protocGlobalInclude = xfilepath.List( - xfilepath.JoinFromHome(xfilepath.Path("local/include")), - xfilepath.JoinFromHome(xfilepath.Path(".local/include")), +var ( + ErrBufConfig = errors.New("invalid Buf config") + + protocGlobalInclude = xfilepath.List( + xfilepath.JoinFromHome(xfilepath.Path("local/include")), + xfilepath.JoinFromHome(xfilepath.Path(".local/include")), + ) ) -type ModulesInPath struct { - Path string - Modules []module.Module - Includes []string +// protoIncludes contains proto include paths for a package. +type protoIncludes struct { + // Paths is a list of proto include paths. + Paths []string + + // BufPath is the path to the Buf config file when it exists. + BufPath string + + // ProtoPath contains the path to the package's proto directory. + ProtoPath string +} + +// protoAnalysis contains proto module analysis data for a Go package dependency. +type protoAnalysis struct { + // Path is the full path to the Go dependency + Path string + + // Modules contains the proto modules analysis data. + // The list is empty when the Go package has no proto files. + Modules []module.Module + + // Includes contain proto include paths. + // These paths should be used when generating code. + Includes protoIncludes } -func (g *generator) setup() (err error) { +func newBufConfigError(path string, cause error) error { + return fmt.Errorf("%w: %s: %w", ErrBufConfig, path, cause) +} + +func (g *generator) setup(ctx context.Context) (err error) { // Cosmos SDK hosts proto files of own x/ modules and some third party ones needed by itself and // blockchain apps. Generate should be aware of these and make them available to the blockchain // app that wants to generate code for its own proto. @@ -49,7 +86,7 @@ func (g *generator) setup() (err error) { New( cmdrunner.DefaultStderr(&errb), cmdrunner.DefaultWorkdir(g.appPath), - ).Run(g.ctx, step.New(step.Exec("go", "mod", "download"))); err != nil { + ).Run(ctx, step.New(step.Exec("go", "mod", "download"))); err != nil { return errors.Wrap(err, errb.String()) } @@ -76,31 +113,34 @@ func (g *generator) setup() (err error) { } // Discover any custom modules defined by the user's app - g.appModules, err = g.discoverModules(g.appPath, g.protoDir) + g.appModules, err = g.discoverModules(ctx, g.appPath, g.protoDir) if err != nil { return err } - g.appIncludes, _, err = g.resolveIncludes(g.appPath) + + g.appIncludes, _, err = g.resolveIncludes(ctx, g.appPath) if err != nil { return err } - // Go through the Go dependencies of the user's app within go.mod, some of them might be hosting Cosmos SDK modules - // that could be in use by user's blockchain. + + // Go through the Go dependencies of the user's app within go.mod, some of them + // might be hosting Cosmos SDK modules that could be in use by user's blockchain. // - // Cosmos SDK is a dependency of all blockchains, so it's absolute that we'll be discovering all modules of the - // SDK as well during this process. + // Cosmos SDK is a dependency of all blockchains, so it's absolute that we'll be + // discovering all modules of the SDK as well during this process. // - // Even if a dependency contains some SDK modules, not all of these modules could be used by user's blockchain. - // this is fine, we can still generate TS clients for those non modules, it is up to user to use (import in typescript) - // not use generated modules. + // Even if a dependency contains some SDK modules, not all of these modules could + // be used by user's blockchain. This is fine, we can still generate TS clients + // for those non modules, it is up to user to use (import in typescript) not use + // generated modules. // - // TODO: we can still implement some sort of smart filtering to detect non used modules by the user's blockchain - // at some point, it is a nice to have. - moduleCache := cache.New[ModulesInPath](g.cacheStorage, moduleCacheNamespace) + // TODO: we can still implement some sort of smart filtering to detect non used + // modules by the user's blockchain at some point, it is a nice to have. + moduleCache := cache.New[protoAnalysis](g.cacheStorage, moduleCacheNamespace) for _, dep := range g.deps { // Try to get the cached list of modules for the current dependency package cacheKey := cache.Key(dep.Path, dep.Version) - modulesInPath, err := moduleCache.Get(cacheKey) + depInfo, err := moduleCache.Get(cacheKey) if err != nil && !errors.Is(err, cache.ErrorNotFound) { return err } @@ -108,50 +148,44 @@ func (g *generator) setup() (err error) { // Discover the modules of the dependency package when they are not cached if errors.Is(err, cache.ErrorNotFound) { // Get the absolute path to the package's directory - path, err := gomodule.LocatePath(g.ctx, g.cacheStorage, g.appPath, dep) + path, err := gomodule.LocatePath(ctx, g.cacheStorage, g.appPath, dep) if err != nil { return err } // Discover any modules defined by the package - modules, err := g.discoverModules(path, "") + modules, err := g.discoverModules(ctx, path, "") if err != nil { return err } - var includes []string - cacheable := true + // Dependency/includes resolution per module is done to solve versioning issues + var ( + includes protoIncludes + cacheable = true + ) if len(modules) > 0 { - // For versioning issues, we do dependency/includes resolution per module - includes, cacheable, err = g.resolveIncludes(path) + includes, cacheable, err = g.resolveIncludes(ctx, path) if err != nil { return err } } - modulesInPath = ModulesInPath{ + depInfo = protoAnalysis{ Path: path, Modules: modules, Includes: includes, } + if cacheable { - if err := moduleCache.Put(cacheKey, modulesInPath); err != nil { + if err = moduleCache.Put(cacheKey, depInfo); err != nil { return err } } } - g.thirdModules[modulesInPath.Path] = append( - g.thirdModules[modulesInPath.Path], - modulesInPath.Modules..., - ) - - if modulesInPath.Includes != nil { - g.thirdModuleIncludes[modulesInPath.Path] = append( - g.thirdModuleIncludes[modulesInPath.Path], - modulesInPath.Includes..., - ) - } + g.thirdModules[depInfo.Path] = depInfo.Modules + g.thirdModuleIncludes[depInfo.Path] = depInfo.Includes } return nil @@ -172,7 +206,8 @@ func (g *generator) findBufPath(modpath string) (string, error) { if err != nil { return err } - if filepath.Base(path) == "buf.yaml" { + base := filepath.Base(path) + if base == "buf.yaml" || base == "buf.yml" { bufPath = path return filepath.SkipAll } @@ -184,7 +219,7 @@ func (g *generator) findBufPath(modpath string) (string, error) { return bufPath, nil } -func (g *generator) generateBufIncludeFolder(modpath string) (string, error) { +func (g *generator) generateBufIncludeFolder(ctx context.Context, modpath string) (string, error) { protoPath, err := os.MkdirTemp("", "includeFolder") if err != nil { return "", err @@ -192,61 +227,68 @@ func (g *generator) generateBufIncludeFolder(modpath string) (string, error) { g.tmpDirs = append(g.tmpDirs, protoPath) - err = g.buf.Export(g.ctx, modpath, protoPath) + err = g.buf.Export(ctx, modpath, protoPath) if err != nil { return "", err } return protoPath, nil } -func (g *generator) resolveIncludes(path string) (paths []string, cacheable bool, err error) { +func (g *generator) resolveIncludes(ctx context.Context, path string) (protoIncludes, bool, error) { // Init paths with the global include paths for protoc - paths, err = protocGlobalInclude() + paths, err := protocGlobalInclude() if err != nil { - return nil, false, err + return protoIncludes{}, false, err } - // Check that the app proto directory exists + includes := protoIncludes{Paths: paths} + + // Check that the app/package proto directory exists protoPath := filepath.Join(path, g.protoDir) fi, err := os.Stat(protoPath) if err != nil && !os.IsNotExist(err) { - return nil, false, err + return protoIncludes{}, false, err } else if !fi.IsDir() { // Just return the global includes when a proto directory doesn't exist - return paths, true, nil + return includes, true, nil } // Add app's proto path to the list of proto paths - paths = append(paths, protoPath) + includes.Paths = append(includes.Paths, protoPath) + includes.ProtoPath = protoPath // Check if a Buf config file is present - bufPath, err := g.findBufPath(protoPath) + includes.BufPath, err = g.findBufPath(protoPath) if err != nil { - return nil, false, err + return includes, false, err } - // When a Buf config exists export all protos needed - // to build the modules to a temporary include folder. - if bufPath != "" { - includePath, err := g.generateBufIncludeFolder(protoPath) + if includes.BufPath != "" { + // When a Buf config exists export all protos needed + // to build the modules to a temporary include folder. + bufProtoPath, err := g.generateBufIncludeFolder(ctx, protoPath) if err != nil && !errors.Is(err, cosmosbuf.ErrProtoFilesNotFound) { - return nil, false, err + return protoIncludes{}, false, err } // Use exported files only when the path contains ".proto" files - if includePath != "" { - return append(paths, includePath), false, nil + if bufProtoPath != "" { + includes.Paths = append(includes.Paths, bufProtoPath) + return includes, false, nil } } - // By default use the configured directories - return append(paths, g.getProtoIncludeFolders(path)...), true, nil + // When there is no Buf config add the configured directories + // instead to keep the legacy (non Buf) behavior. + includes.Paths = append(includes.Paths, g.getProtoIncludeFolders(path)...) + + return includes, true, nil } -func (g *generator) discoverModules(path, protoDir string) ([]module.Module, error) { +func (g *generator) discoverModules(ctx context.Context, path, protoDir string) ([]module.Module, error) { var filteredModules []module.Module - modules, err := module.Discover(g.ctx, g.appPath, path, protoDir) + modules, err := module.Discover(ctx, g.appPath, path, protoDir) if err != nil { return nil, err } @@ -263,3 +305,173 @@ func (g *generator) discoverModules(path, protoDir string) ([]module.Module, err return filteredModules, nil } + +func (g generator) updateBufModule(ctx context.Context) error { + for pkgPath, includes := range g.thirdModuleIncludes { + // Skip third party dependencies without proto files + if includes.ProtoPath == "" { + continue + } + + // Resolve the Go package and use the module name as the proto vendor directory name + modFile, err := gomodule.ParseAt(pkgPath) + if err != nil { + return err + } + + pkgName := modFile.Module.Mod.Path + + // When a Buf config with name is available add it to app's dependencies + // or otherwise export the proto files to a vendor directory. + if includes.BufPath != "" { + if err := g.resolveBufDependency(ctx, pkgName, includes.BufPath); err != nil { + return err + } + } else { + if err := g.vendorProtoPackage(pkgName, includes.ProtoPath); err != nil { + return err + } + } + } + return nil +} + +func (g generator) resolveBufDependency(ctx context.Context, pkgName, bufPath string) error { + // Open the dependency Buf config to find the BSR package name + f, err := os.Open(bufPath) + if err != nil { + return err + } + defer f.Close() + + cfg := struct { + Name string `yaml:"name"` + }{} + + if err := yaml.NewDecoder(f).Decode(&cfg); err != nil { + return newBufConfigError(bufPath, err) + } + + // When dependency package has a Buf config name try to add it to app's + // dependencies. Name is optional and defines the BSR package name. + if cfg.Name != "" { + return g.addBufDependency(ctx, cfg.Name) + } + // By default just vendor the proto package + return g.vendorProtoPackage(pkgName, filepath.Dir(bufPath)) +} + +func (g generator) addBufDependency(ctx context.Context, depName string) error { + // Read app's Buf config + path := g.appIncludes.BufPath + bz, err := os.ReadFile(path) + if err != nil { + return err + } + + // Check if the proto dependency is already present in app's Buf config + cfg := struct { + Deps []string `yaml:"deps"` + }{} + if err := yaml.Unmarshal(bz, &cfg); err != nil { + return newBufConfigError(path, err) + } + + if slices.Contains(cfg.Deps, depName) { + return nil + } + + // Add the new dependency and update app's Buf config + f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer f.Close() + + var rawCfg map[string]interface{} + if err := yaml.Unmarshal(bz, &rawCfg); err != nil { + return newBufConfigError(path, err) + } + + rawCfg["deps"] = append(cfg.Deps, depName) + + enc := yaml.NewEncoder(f) + defer enc.Close() + + if err := enc.Encode(rawCfg); err != nil { + return err + } + + g.opts.ev.Send( + fmt.Sprintf("New Buf dependency added: %s", colors.Name(depName)), + events.Icon(icons.OK), + ) + + // Update Buf lock so it contains the new dependency + return g.buf.Update(ctx, filepath.Dir(path), depName) +} + +func (g generator) vendorProtoPackage(pkgName, protoPath string) (err error) { + // Check that the dependency vendor directory doesn't exist + vendorRelPath := filepath.Join("proto_vendor", pkgName) + vendorPath := filepath.Join(g.appPath, vendorRelPath) + _, err = os.Stat(vendorPath) + if err != nil && !os.IsNotExist(err) { + return err + } + + // Skip vendoring when the dependency is already vendored + if !os.IsNotExist(err) { + return nil + } + + if err = os.MkdirAll(vendorPath, 0o777); err != nil { + return err + } + + // Make sure that the vendor folder is removed on error + defer func() { + if err != nil { + _ = os.RemoveAll(vendorPath) + } + }() + + if err = xos.CopyFolder(protoPath, vendorPath); err != nil { + return err + } + + path := filepath.Join(g.appPath, workFilename) + bz, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading Buf workspace file: %s: %w", path, err) + } + + ws := struct { + Version string `yaml:"version"` + Directories []string `yaml:"directories"` + }{} + if err := yaml.Unmarshal(bz, &ws); err != nil { + return err + } + + ws.Directories = append(ws.Directories, vendorRelPath) + + f, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer f.Close() + + enc := yaml.NewEncoder(f) + defer enc.Close() + if err = enc.Encode(ws); err != nil { + return err + } + + g.opts.ev.Send( + fmt.Sprintf("New Buf vendored dependency added: %s", colors.Name(vendorRelPath)), + events.Icon(icons.OK), + ) + + return nil +} diff --git a/ignite/pkg/cosmosgen/generate_go.go b/ignite/pkg/cosmosgen/generate_go.go index ae8dfc6ce5..8423c17fca 100644 --- a/ignite/pkg/cosmosgen/generate_go.go +++ b/ignite/pkg/cosmosgen/generate_go.go @@ -1,10 +1,12 @@ package cosmosgen import ( + "context" "os" "path/filepath" "github.com/otiai10/copy" + "github.com/pkg/errors" ) @@ -16,7 +18,7 @@ func (g *generator) pulsarTemplate() string { return filepath.Join(g.appPath, g.protoDir, "buf.gen.pulsar.yaml") } -func (g *generator) generateGo() error { +func (g *generator) generateGo(ctx context.Context) error { // create a temporary dir to locate generated code under which later only some of them will be moved to the // app's source code. this also prevents having leftover files in the app's source code or its parent dir - when // command executed directly there - in case of an interrupt. @@ -29,13 +31,8 @@ func (g *generator) generateGo() error { protoPath := filepath.Join(g.appPath, g.protoDir) // code generate for each module. - if err := g.buf.Generate( - g.ctx, - protoPath, - tmp, - g.gogoTemplate(), - "module.proto", - ); err != nil { + err = g.buf.Generate(ctx, protoPath, tmp, g.gogoTemplate(), "module.proto") + if err != nil { return err } @@ -55,7 +52,7 @@ func (g *generator) generateGo() error { return nil } -func (g *generator) generatePulsar() error { +func (g *generator) generatePulsar(ctx context.Context) error { // create a temporary dir to locate generated code under which later only some of them will be moved to the // app's source code. this also prevents having leftover files in the app's source code or its parent dir - when // command executed directly there - in case of an interrupt. @@ -68,12 +65,8 @@ func (g *generator) generatePulsar() error { protoPath := filepath.Join(g.appPath, g.protoDir) // code generate for each module. - if err := g.buf.Generate( - g.ctx, - protoPath, - tmp, - g.pulsarTemplate(), - ); err != nil { + err = g.buf.Generate(ctx, protoPath, tmp, g.pulsarTemplate()) + if err != nil { return err } diff --git a/ignite/pkg/cosmosgen/generate_openapi.go b/ignite/pkg/cosmosgen/generate_openapi.go index 84623e6b68..0100775b39 100644 --- a/ignite/pkg/cosmosgen/generate_openapi.go +++ b/ignite/pkg/cosmosgen/generate_openapi.go @@ -1,6 +1,7 @@ package cosmosgen import ( + "context" "errors" "fmt" "os" @@ -30,7 +31,7 @@ func (g *generator) openAPITemplateForSTA() string { return filepath.Join(g.appPath, g.protoDir, "buf.gen.sta.yaml") } -func (g *generator) generateOpenAPISpec() error { +func (g *generator) generateOpenAPISpec(ctx context.Context) error { var ( specDirs []string conf = swaggercombine.Config{ @@ -83,13 +84,8 @@ func (g *generator) generateOpenAPISpec() error { } hasAnySpecChanged = true - if err := g.buf.Generate( - g.ctx, - m.Pkg.Path, - dir, - g.openAPITemplate(), - "module.proto", - ); err != nil { + err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplate(), "module.proto") + if err != nil { return err } @@ -162,14 +158,14 @@ func (g *generator) generateOpenAPISpec() error { } // combine specs into one and save to out. - if err := swaggercombine.Combine(g.ctx, conf, command, out); err != nil { + if err := swaggercombine.Combine(ctx, conf, command, out); err != nil { return err } return dirchange.SaveDirChecksum(specCache, out, g.appPath, out) } -func (g *generator) generateModuleOpenAPISpec(m module.Module, out string) error { +func (g *generator) generateModuleOpenAPISpec(ctx context.Context, m module.Module, out string) error { var ( specDirs []string conf = swaggercombine.Config{ @@ -199,13 +195,8 @@ func (g *generator) generateModuleOpenAPISpec(m module.Module, out string) error return err } - if err := g.buf.Generate( - g.ctx, - m.Pkg.Path, - dir, - g.openAPITemplateForSTA(), - "module.proto", - ); err != nil { + err = g.buf.Generate(ctx, m.Pkg.Path, dir, g.openAPITemplateForSTA(), "module.proto") + if err != nil { return err } @@ -253,5 +244,5 @@ func (g *generator) generateModuleOpenAPISpec(m module.Module, out string) error return err } // combine specs into one and save to out. - return swaggercombine.Combine(g.ctx, conf, command, out) + return swaggercombine.Combine(ctx, conf, command, out) } diff --git a/ignite/pkg/cosmosgen/generate_typescript.go b/ignite/pkg/cosmosgen/generate_typescript.go index 05f988a75e..328eb69b51 100644 --- a/ignite/pkg/cosmosgen/generate_typescript.go +++ b/ignite/pkg/cosmosgen/generate_typescript.go @@ -37,7 +37,7 @@ func newTSGenerator(g *generator) *tsGenerator { return &tsGenerator{g} } -func (g *generator) generateTS() error { +func (g *generator) generateTS(ctx context.Context) error { chainPath, _, err := gomodulepath.Find(g.appPath) if err != nil { return err @@ -70,14 +70,14 @@ func (g *generator) generateTS() error { }) tsg := newTSGenerator(g) - if err := tsg.generateModuleTemplates(); err != nil { + if err := tsg.generateModuleTemplates(ctx); err != nil { return err } return tsg.generateRootTemplates(data) } -func (g *tsGenerator) generateModuleTemplates() error { +func (g *tsGenerator) generateModuleTemplates(ctx context.Context) error { protocCmd, cleanupProtoc, err := protoc.Command() if err != nil { return err @@ -122,7 +122,7 @@ func (g *tsGenerator) generateModuleTemplates() error { } } - err = g.generateModuleTemplate(g.g.ctx, protocCmd, staCmd, tsprotoPluginPath, sourcePath, m, includes) + err = g.generateModuleTemplate(ctx, protocCmd, staCmd, tsprotoPluginPath, sourcePath, m, includes) if err != nil { return err } @@ -132,7 +132,7 @@ func (g *tsGenerator) generateModuleTemplates() error { } } - add(g.g.appPath, g.g.appModules, g.g.appIncludes) + add(g.g.appPath, g.g.appModules, g.g.appIncludes.Paths) // Always generate third party modules; This is required because not generating them might // lead to issues with the module registration in the root template. The root template must @@ -140,8 +140,9 @@ func (g *tsGenerator) generateModuleTemplates() error { // is available and not generated it would lead to the registration of a new not generated // 3rd party module. for sourcePath, modules := range g.g.thirdModules { - includes := g.g.thirdModuleIncludes[sourcePath] - add(sourcePath, modules, append(g.g.appIncludes, includes...)) + // TODO: Skip modules without proto files? + thirdIncludes := g.g.thirdModuleIncludes[sourcePath] + add(sourcePath, modules, append(g.g.appIncludes.Paths, thirdIncludes.Paths...)) } return gg.Wait() @@ -181,9 +182,7 @@ func (g *tsGenerator) generateModuleTemplate( specPath := filepath.Join(out, "api.swagger.yml") - err = g.g.generateModuleOpenAPISpec(m, specPath) - - if err != nil { + if err = g.g.generateModuleOpenAPISpec(ctx, m, specPath); err != nil { return err } // generate the REST client from the OpenAPI spec diff --git a/ignite/services/chain/generate.go b/ignite/services/chain/generate.go index 2cfbf26efb..3ebcbfaedb 100644 --- a/ignite/services/chain/generate.go +++ b/ignite/services/chain/generate.go @@ -16,6 +16,7 @@ import ( type generateOptions struct { useCache bool + isProtoVendorEnabled bool isGoEnabled bool isPulsarEnabled bool isTSClientEnabled bool @@ -95,6 +96,18 @@ func GenerateOpenAPI() GenerateTarget { } } +// GenerateProtoVendor enables `proto_vendor` folder generation. +// Proto vendor is generated from Go dependencies that contain proto files that +// are not included in the app's Buf config. +// Enabling proto vendoring might update Buf config with missing dependencies +// if a Go dependency contains proto files and a Buf config with a name that is +// not listed in the Buf dependencies. +func GenerateProtoVendor() GenerateTarget { + return func(o *generateOptions) { + o.isProtoVendorEnabled = true + } +} + // generateFromConfig makes code generation from proto files from the given config. func (c *Chain) generateFromConfig(ctx context.Context, cacheStorage cache.Storage, generateClients bool) error { conf, err := c.Config() @@ -157,6 +170,7 @@ func (c *Chain) Generate( c.ev.Send("Building proto...", events.ProgressUpdate()) options := []cosmosgen.Option{ + cosmosgen.CollectEvents(c.ev), cosmosgen.IncludeDirs(conf.Build.Proto.ThirdPartyPaths), } @@ -168,6 +182,10 @@ func (c *Chain) Generate( options = append(options, cosmosgen.WithPulsarGeneration()) } + if targetOptions.isProtoVendorEnabled { + options = append(options, cosmosgen.UpdateBufModule()) + } + var ( openAPIPath, tsClientPath, vuexPath, composablesPath, hooksPath string updateConfig bool diff --git a/ignite/services/scaffolder/scaffolder.go b/ignite/services/scaffolder/scaffolder.go index 6caa944113..6052ddae13 100644 --- a/ignite/services/scaffolder/scaffolder.go +++ b/ignite/services/scaffolder/scaffolder.go @@ -92,6 +92,7 @@ func protoc(ctx context.Context, cacheStorage cache.Storage, projectPath, gomodP options := []cosmosgen.Option{ cosmosgen.WithGoGeneration(), cosmosgen.WithPulsarGeneration(), + cosmosgen.UpdateBufModule(), cosmosgen.IncludeDirs(conf.Build.Proto.ThirdPartyPaths), }