feat: add or vendor proto packages from Go dependencies (#3724)

* refactor: change `cosmosgen` proto discovery to add extra paths

* feat(pkg/cosmosbuf): add Buf mod update support

* feat: add third party deps to app's Buf config

Third party deps are added to app's Buf config when the dependency is
not present and the Go package protos have a Buf config with a `name`
assigned.

* chore: update changelog

* feat: add vendoring support for Go dependencies with proto files

* refactor(pkg/cosmosgen): use context as function argument

* feat: add `--update-buf-module` flag to generate go commands

The flag explicitly enables proto vendoring or Buf config dependency
updates.

* feat: change scaffolder to update Buf with third party dependencies

This updates Buf dependencies and vendors proto files from third party
Go dependencies by default for the scaffold commands

* feat: add events support to generator

* chore: remove invalid TODO comments

* chore: add Buf config error variable

Co-authored-by: Danilo Pantani <[email protected]>

* chore: change `--update-buf-module` flag to be persistent

Co-authored-by: Danilo Pantani <[email protected]>

* refactor: enable Buf dep vendoring for generate ts-client command

This is required because OpenAPI generation is done using Buf while TS
client is being generated.

* refactor: add generate proto vendor target

All generate commands require support for proto vendoring and Buf
dependencies updates.
A new target was added to make proto vendoring optional and to avoid
using an extra argument in all generate targets.

* chore: improve variable usage

Co-authored-by: Danilo Pantani <[email protected]>

* refactor: rename proto vendoring flag to `--enable-proto-vendor`

---------

Co-authored-by: Danilo Pantani <[email protected]>

changelog.md

@@ -23,6 +23,7 @@
 - [#3614](https://github.com/ignite/cli/pull/3614) feat: use DefaultBaseappOptions for app.New method
 - [#3536](https://github.com/ignite/cli/pull/3536) Change app.go to v2 and add AppWiring feature
 - [#3670](https://github.com/ignite/cli/pull/3670) Remove nodetime binaries
+- [#3724](https://github.com/ignite/cli/pull/3724) Add or vendor proto packages from Go dependencies
 - [#3715](https://github.com/ignite/cli/pull/3715) Add test suite for the cli tests
 
 ### Changes

ignite/cmd/generate.go

@@ -2,6 +2,11 @@ package ignitecmd
 
 import (
 	"github.com/spf13/cobra"
+	flag "github.com/spf13/pflag"
+)
+
+const (
+	flagEnableProtoVendor = "enable-proto-vendor"
 )
 
 // NewGenerate returns a command that groups code generation related sub commands.
@@ -22,8 +27,11 @@ meant to be edited by hand.
 		PersistentPreRunE: migrationPreRunHandler,
 	}
 
+	c.PersistentFlags().AddFlagSet(flagSetEnableProtoVendor())
+
 	flagSetPath(c)
 	flagSetClearCache(c)
+
 	c.AddCommand(NewGenerateGo())
 	c.AddCommand(NewGeneratePulsar())
 	c.AddCommand(NewGenerateTSClient())
@@ -34,3 +42,14 @@ meant to be edited by hand.
 
 	return c
 }
+
+func flagSetEnableProtoVendor() *flag.FlagSet {
+	fs := flag.NewFlagSet("", flag.ContinueOnError)
+	fs.Bool(flagEnableProtoVendor, false, "enable proto package vendor for missing Buf dependencies")
+	return fs
+}
+
+func flagGetEnableProtoVendor(cmd *cobra.Command) bool {
+	skip, _ := cmd.Flags().GetBool(flagEnableProtoVendor)
+	return skip
+}

ignite/cmd/generate_composables.go

@@ -44,7 +44,13 @@ func generateComposablesHandler(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateComposables(output)); err != nil {
+	var opts []chain.GenerateTarget
+	if flagGetEnableProtoVendor(cmd) {
+		opts = append(opts, chain.GenerateProtoVendor())
+	}
+
+	err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateComposables(output), opts...)
+	if err != nil {
 		return err
 	}
 

ignite/cmd/generate_go.go

@@ -39,7 +39,13 @@ func generateGoHandler(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateGo()); err != nil {
+	var opts []chain.GenerateTarget
+	if flagGetEnableProtoVendor(cmd) {
+		opts = append(opts, chain.GenerateProtoVendor())
+	}
+
+	err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateGo(), opts...)
+	if err != nil {
 		return err
 	}
 

ignite/cmd/generate_hooks.go

@@ -44,7 +44,13 @@ func generateHooksHandler(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateHooks(output)); err != nil {
+	var opts []chain.GenerateTarget
+	if flagGetEnableProtoVendor(cmd) {
+		opts = append(opts, chain.GenerateProtoVendor())
+	}
+
+	err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateHooks(output), opts...)
+	if err != nil {
 		return err
 	}
 

ignite/cmd/generate_openapi.go

@@ -39,7 +39,13 @@ func generateOpenAPIHandler(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateOpenAPI()); err != nil {
+	var opts []chain.GenerateTarget
+	if flagGetEnableProtoVendor(cmd) {
+		opts = append(opts, chain.GenerateProtoVendor())
+	}
+
+	err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateOpenAPI(), opts...)
+	if err != nil {
 		return err
 	}
 

ignite/cmd/generate_pulsar.go

@@ -39,7 +39,13 @@ func generatePulsarHandler(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	if err := c.Generate(cmd.Context(), cacheStorage, chain.GeneratePulsar()); err != nil {
+	var opts []chain.GenerateTarget
+	if flagGetEnableProtoVendor(cmd) {
+		opts = append(opts, chain.GenerateProtoVendor())
+	}
+
+	err = c.Generate(cmd.Context(), cacheStorage, chain.GeneratePulsar(), opts...)
+	if err != nil {
 		return err
 	}
 

ignite/cmd/generate_typescript_client.go

@@ -73,7 +73,12 @@ func generateTSClientHandler(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateTSClient(output, useCache))
+	var opts []chain.GenerateTarget
+	if flagGetEnableProtoVendor(cmd) {
+		opts = append(opts, chain.GenerateProtoVendor())
+	}
+
+	err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateTSClient(output, useCache), opts...)
 	if err != nil {
 		return err
 	}

ignite/cmd/generate_vuex.go

@@ -45,7 +45,13 @@ func generateVuexHandler(cmd *cobra.Command, _ []string) error {
 		return err
 	}
 
-	if err := c.Generate(cmd.Context(), cacheStorage, chain.GenerateVuex(output)); err != nil {
+	var opts []chain.GenerateTarget
+	if flagGetEnableProtoVendor(cmd) {
+		opts = append(opts, chain.GenerateProtoVendor())
+	}
+
+	err = c.Generate(cmd.Context(), cacheStorage, chain.GenerateVuex(output), opts...)
+	if err != nil {
 		return err
 	}
 

ignite/pkg/cosmosbuf/buf.go

@@ -35,17 +35,20 @@ const (
 	flagOutput      = "output"
 	flagErrorFormat = "error-format"
 	flagLogFormat   = "log-format"
+	flagOnly        = "only"
 	fmtJSON         = "json"
 
 	// CMDGenerate generate command.
 	CMDGenerate Command = "generate"
 	CMDExport   Command = "export"
+	CMDMod      Command = "mod"
 )
 
 var (
 	commands = map[Command]struct{}{
 		CMDGenerate: {},
 		CMDExport:   {},
+		CMDMod:      {},
 	}
 
 	// ErrInvalidCommand indicates an invalid command name.
@@ -72,6 +75,24 @@ func (c Command) String() string {
 	return string(c)
 }
 
+// Update updates module dependencies.
+// By default updates all dependencies unless one or more dependencies are specified.
+func (b Buf) Update(ctx context.Context, modDir string, dependencies ...string) error {
+	var flags map[string]string
+	if dependencies != nil {
+		flags = map[string]string{
+			flagOnly: strings.Join(dependencies, ","),
+		}
+	}
+
+	cmd, err := b.generateCommand(CMDMod, flags, "update", modDir)
+	if err != nil {
+		return err
+	}
+
+	return b.runCommand(ctx, cmd...)
+}
+
 // Export runs the buf Export command for the files in the proto directory.
 func (b Buf) Export(ctx context.Context, protoDir, output string) error {
 	// Check if the proto directory is the Cosmos SDK one

ignite/pkg/cosmosgen/cosmosgen.go

@@ -12,12 +12,15 @@ import (
 	"github.com/ignite/cli/ignite/pkg/cache"
 	"github.com/ignite/cli/ignite/pkg/cosmosanalysis/module"
 	"github.com/ignite/cli/ignite/pkg/cosmosbuf"
+	"github.com/ignite/cli/ignite/pkg/events"
 )
 
 // generateOptions used to configure code generation.
 type generateOptions struct {
-	includeDirs []string
-	useCache    bool
+	includeDirs     []string
+	useCache        bool
+	updateBufModule bool
+	ev              events.Bus
 
 	isGoEnabled     bool
 	isPulsarEnabled bool
@@ -103,9 +106,24 @@ func IncludeDirs(dirs []string) Option {
 	}
 }
 
+// UpdateBufModule enables Buf config proto dependencies update.
+// This option updates app's Buf config when proto packages or
+// Buf modules are found within the Go dependencies.
+func UpdateBufModule() Option {
+	return func(o *generateOptions) {
+		o.updateBufModule = true
+	}
+}
+
+// CollectEvents sets an event bus for sending generation feedback events.
+func CollectEvents(ev events.Bus) Option {
+	return func(c *generateOptions) {
+		c.ev = ev
+	}
+}
+
 // generator generates code for sdk and sdk apps.
 type generator struct {
-	ctx                 context.Context
 	buf                 cosmosbuf.Buf
 	cacheStorage        cache.Storage
 	appPath             string
@@ -115,9 +133,9 @@ type generator struct {
 	sdkImport           string
 	deps                []gomodule.Version
 	appModules          []module.Module
-	appIncludes         []string
+	appIncludes         protoIncludes
 	thirdModules        map[string][]module.Module
-	thirdModuleIncludes map[string][]string
+	thirdModuleIncludes map[string]protoIncludes
 	tmpDirs             []string
 }
 
@@ -139,14 +157,13 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir
 	defer b.Cleanup()
 
 	g := &generator{
-		ctx:                 ctx,
 		buf:                 b,
 		appPath:             appPath,
 		protoDir:            protoDir,
 		gomodPath:           gomodPath,
 		opts:                &generateOptions{},
 		thirdModules:        make(map[string][]module.Module),
-		thirdModuleIncludes: make(map[string][]string),
+		thirdModuleIncludes: make(map[string]protoIncludes),
 		cacheStorage:        cacheStorage,
 	}
 
@@ -156,31 +173,41 @@ func Generate(ctx context.Context, cacheStorage cache.Storage, appPath, protoDir
 		apply(g.opts)
 	}
 
-	if err := g.setup(); err != nil {
+	if err := g.setup(ctx); err != nil {
 		return err
 	}
 
+	// Update app's Buf config for third party discovered proto modules.
+	// Go dependency packages might contain proto files which could also
+	// optionally be using Buf, so for those cases the discovered proto
+	// files should be available before code generation.
+	if g.opts.updateBufModule {
+		if err := g.updateBufModule(ctx); err != nil {
+			return err
+		}
+	}
+
 	// Go generation must run first so the types are created before other
 	// generated code that requires sdk.Msg implementations to be defined
 	if g.opts.isGoEnabled {
-		if err := g.generateGo(); err != nil {
+		if err := g.generateGo(ctx); err != nil {
 			return err
 		}
 	}
 	if g.opts.isPulsarEnabled {
-		if err := g.generatePulsar(); err != nil {
+		if err := g.generatePulsar(ctx); err != nil {
 			return err
 		}
 	}
 
 	if g.opts.specOut != "" {
-		if err := g.generateOpenAPISpec(); err != nil {
+		if err := g.generateOpenAPISpec(ctx); err != nil {
 			return err
 		}
 	}
 
 	if g.opts.jsOut != nil {
-		if err := g.generateTS(); err != nil {
+		if err := g.generateTS(ctx); err != nil {
 			return err
 		}
 	}