Add buf generate configs and update other packages to use these configs

TODO.txt

@@ -20,6 +20,13 @@
 - document behavior of file searching, config override
 - fix tamper-proofing
 - go through all todos
+- Un-skip relevant tests in generate_test.go and generate_unix_test.go, when
+  - the behavior of --path and --exclude-path is updated to match what's on main
+- Debug the skipped test with proto file ref in generate_test.go
+- Use syserror when possible (especially in buf gen related code).
+- Find the right place (right way to pass a logger) to print a warning when a buf.gen.yaml with
+  non-empty managed mode but without `enabled: true`. The key is to decide where to pass the logger.
+- Fix the issue where reading a buf.lock with empty digest errors. Run `buf mod update` to reproduce.
 
 NOTE: We are not allowing cross-workspace finding for include_package_files=true
 

make/buf/all.mk

@@ -61,6 +61,7 @@ testbufnew: installbuf
 		./private/buf/bufcurl/... \
 		./private/buf/buffetch/... \
 		./private/buf/bufformat/... \
+		./private/buf/bufgen/... \
 		./private/buf/bufprint/... \
 		./private/buf/bufworkspace/... \
 		./private/buf/cmd/buf/command/alpha/package/... \
@@ -74,6 +75,7 @@ testbufnew: installbuf
 		./private/buf/cmd/buf/command/build/... \
 		./private/buf/cmd/buf/command/breaking/... \
 		./private/buf/cmd/buf/command/convert/... \
+		./private/buf/cmd/buf/command/generate/... \
 		./private/buf/cmd/buf/command/lint/... \
 		./private/buf/cmd/buf/command/lsfiles/... \
 		./private/buf/cmd/buf/command/mod/... \

private/buf/bufctl/controller.go

@@ -86,6 +86,11 @@ type Controller interface {
 		input string,
 		options ...FunctionOption,
 	) (bufimage.Image, error)
+	GetImageForInputConfig(
+		ctx context.Context,
+		inputConfig bufconfig.InputConfig,
+		options ...FunctionOption,
+	) (bufimage.Image, error)
 	GetImageForWorkspace(
 		ctx context.Context,
 		workspace bufworkspace.Workspace,
@@ -275,6 +280,18 @@ func (c *controller) GetImage(
 	return c.getImage(ctx, input, functionOptions)
 }
 
+func (c *controller) GetImageForInputConfig(
+	ctx context.Context,
+	inputConfig bufconfig.InputConfig,
+	options ...FunctionOption,
+) (bufimage.Image, error) {
+	functionOptions := newFunctionOptions()
+	for _, option := range options {
+		option(functionOptions)
+	}
+	return c.getImageForInputConfig(ctx, inputConfig, functionOptions)
+}
+
 func (c *controller) GetImageForWorkspace(
 	ctx context.Context,
 	workspace bufworkspace.Workspace,
@@ -630,6 +647,26 @@ func (c *controller) getImage(
 	if err != nil {
 		return nil, err
 	}
+	return c.getImageForRef(ctx, ref, functionOptions)
+}
+
+func (c *controller) getImageForInputConfig(
+	ctx context.Context,
+	inputConfig bufconfig.InputConfig,
+	functionOptions *functionOptions,
+) (bufimage.Image, error) {
+	ref, err := c.buffetchRefParser.GetRefForInputConfig(ctx, inputConfig)
+	if err != nil {
+		return nil, err
+	}
+	return c.getImageForRef(ctx, ref, functionOptions)
+}
+
+func (c *controller) getImageForRef(
+	ctx context.Context,
+	ref buffetch.Ref,
+	functionOptions *functionOptions,
+) (bufimage.Image, error) {
 	switch t := ref.(type) {
 	case buffetch.ProtoFileRef:
 		workspace, err := c.getWorkspaceForProtoFileRef(ctx, t, functionOptions)

private/buf/buffetch/buffetch.go

@@ -16,10 +16,12 @@ package buffetch
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 
 	"github.com/bufbuild/buf/private/buf/buffetch/internal"
+	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
 	"github.com/bufbuild/buf/private/pkg/app"
 	"github.com/bufbuild/buf/private/pkg/git"
@@ -132,18 +134,33 @@ type ProtoFileRef interface {
 type MessageRefParser interface {
 	// GetMessageRef gets the reference for the message file.
 	GetMessageRef(ctx context.Context, value string) (MessageRef, error)
+	// GetMessageRefForInputConfig gets the reference for the message file.
+	GetMessageRefForInputConfig(
+		ctx context.Context,
+		inputConfig bufconfig.InputConfig,
+	) (MessageRef, error)
 }
 
 // SourceRefParser is a source ref parser for Buf.
 type SourceRefParser interface {
 	// GetSourceRef gets the reference for the source file.
 	GetSourceRef(ctx context.Context, value string) (SourceRef, error)
+	// GetSourceRef gets the reference for the source file.
+	GetSourceRefForInputConfig(
+		ctx context.Context,
+		inputConfig bufconfig.InputConfig,
+	) (SourceRef, error)
 }
 
 // DirRefParser is a dif ref parser for Buf.
 type DirRefParser interface {
 	// GetDirRef gets the reference for the source file.
 	GetDirRef(ctx context.Context, value string) (DirRef, error)
+	// GetDirRefForInputConfig gets the reference for the source file.
+	GetDirRefForInputConfig(
+		ctx context.Context,
+		inputConfig bufconfig.InputConfig,
+	) (DirRef, error)
 }
 
 // ModuleRefParser is a source ref parser for Buf.
@@ -161,6 +178,11 @@ type SourceOrModuleRefParser interface {
 
 	// GetSourceOrModuleRef gets the reference for the message file or source bucket.
 	GetSourceOrModuleRef(ctx context.Context, value string) (SourceOrModuleRef, error)
+	// GetSourceOrModuleRefForInputConfig gets the reference for the message file or source bucket.
+	GetSourceOrModuleRefForInputConfig(
+		ctx context.Context,
+		inputConfig bufconfig.InputConfig,
+	) (SourceOrModuleRef, error)
 }
 
 // RefParser is a ref parser for Buf.
@@ -170,8 +192,11 @@ type RefParser interface {
 	DirRefParser
 	SourceOrModuleRefParser
 
+	// TODO: should this be renamed to GetRefForString?
 	// GetRef gets the reference for the message file, source bucket, or module.
 	GetRef(ctx context.Context, value string) (Ref, error)
+	// GetRefForInputConfig gets the reference for the message file, source bucket, or module.
+	GetRefForInputConfig(ctx context.Context, inputConfig bufconfig.InputConfig) (Ref, error)
 }
 
 // NewRefParser returns a new RefParser.
@@ -394,6 +419,42 @@ func NewWriter(
 	)
 }
 
+// GetInputConfigForString returns the input config for the input string.
+func GetInputConfigForString(
+	ctx context.Context,
+	refParser RefParser,
+	value string,
+) (bufconfig.InputConfig, error) {
+	ref, err := refParser.GetRef(ctx, value)
+	if err != nil {
+		return nil, err
+	}
+	switch t := ref.(type) {
+	case MessageRef:
+		switch t.MessageEncoding() {
+		case MessageEncodingBinpb:
+			return bufconfig.NewBinaryImageInputConfig(
+				t.Path(),
+				t.internalSingleRef().CompressionType().String(),
+			)
+		case MessageEncodingJSON:
+			return bufconfig.NewJSONImageInputConfig(
+				t.Path(),
+				t.internalSingleRef().CompressionType().String(),
+			)
+		case MessageEncodingTxtpb:
+			return bufconfig.NewBinaryImageInputConfig(
+				t.Path(),
+				t.internalSingleRef().CompressionType().String(),
+			)
+		default:
+			// TODO: handle refs with YAML type
+			return nil, fmt.Errorf("unknown encoding: %v", t.MessageEncoding())
+		}
+	}
+	return internal.GetInputConfigForRef(ref.internalRef(), value)
+}
+
 type getBucketOptions struct {
 	noSearch bool
 }

private/buf/buffetch/internal/internal.go

@@ -16,9 +16,12 @@ package internal
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 
+	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
 	"github.com/bufbuild/buf/private/pkg/app"
 	"github.com/bufbuild/buf/private/pkg/git"
@@ -83,6 +86,20 @@ type ArchiveType int
 // CompressionType is a compression type.
 type CompressionType int
 
+// String implements fmt.Stringer
+func (c CompressionType) String() string {
+	switch c {
+	case CompressionTypeNone:
+		return "none"
+	case CompressionTypeGzip:
+		return "gzip"
+	case CompressionTypeZstd:
+		return "zstd"
+	default:
+		return strconv.Itoa(int(c))
+	}
+}
+
 // Ref is a reference.
 type Ref interface {
 	ref()
@@ -360,6 +377,12 @@ type RefParser interface {
 	//
 	// The options should be used to validate that you are getting one of the correct formats.
 	GetParsedRef(ctx context.Context, value string, options ...GetParsedRefOption) (ParsedRef, error)
+	// GetParsedRefForInputConfig gets the ParsedRef for the input config.
+	//
+	// The returned ParsedRef will be either a ParsedSingleRef, ParsedArchiveRef, ParsedDirRef, ParsedGitRef, or ParsedModuleRef.
+	//
+	// The options should be used to validate that you are getting one of the correct formats.
+	GetParsedRefForInputConfig(ctx context.Context, inputConfig bufconfig.InputConfig, options ...GetParsedRefOption) (ParsedRef, error)
 }
 
 // NewRefParser returns a new RefParser.
@@ -809,3 +832,62 @@ func WithPutFileNoFileCompression() PutFileOption {
 
 // GetModuleOption is a GetModule option.
 type GetModuleOption func(*getModuleOptions)
+
+// GetInputConfigForRef returns the input config for the ref. A string is also
+// passed because if the ref is a git ref, it would only have a git.Name, instead
+// of a git branch, a git ref and a git tag. Therefore the original string is passed.
+func GetInputConfigForRef(ref Ref, value string) (bufconfig.InputConfig, error) {
+	_, options, err := getRawPathAndOptions(value)
+	if err != nil {
+		return nil, err
+	}
+	switch t := ref.(type) {
+	case ArchiveRef:
+		switch t.ArchiveType() {
+		case ArchiveTypeZip:
+			return bufconfig.NewZipArchiveInputConfig(
+				t.Path(),
+				t.SubDirPath(),
+				t.StripComponents(),
+			)
+		case ArchiveTypeTar:
+			return bufconfig.NewTarballInputConfig(
+				t.Path(),
+				t.SubDirPath(),
+				t.CompressionType().String(),
+				t.StripComponents(),
+			)
+		default:
+			return nil, fmt.Errorf("invalid archive type: %v", t.ArchiveType())
+		}
+	case DirRef:
+		return bufconfig.NewDirectoryInputConfig(
+			t.Path(),
+		)
+	case ModuleRef:
+		return bufconfig.NewModuleInputConfig(
+			t.ModuleRef().String(),
+		)
+	case ProtoFileRef:
+		return bufconfig.NewProtoFileInputConfig(
+			t.Path(),
+			t.IncludePackageFiles(),
+		)
+	case GitRef:
+		return bufconfig.NewGitRepoInputConfig(
+			t.Path(),
+			t.SubDirPath(),
+			options["branch"],
+			options["tag"],
+			options["ref"],
+			toPointer(t.Depth()),
+			t.RecurseSubmodules(),
+		)
+	default:
+		return nil, fmt.Errorf("unexpected Ref of type %T", ref)
+	}
+}
+
+func toPointer[T any](value T) *T {
+	return &value
+}