Add plugin update command

This adds a new command `buf plugin update` enabling the use of remote
plugin refs in `buf.yaml` files. Running the command resolves the refs
and updates the `buf.lock` file, adding the new section plugins with the
resolved digest. Plugins are cached under the `v3/plugins` directory.
This allows `buf lint` to invoke remote plugins locally enabling sharing
of custom plugins.

private/buf/bufcli/cache.go

@@ -26,32 +26,40 @@ import (
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulecache"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulestore"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufplugincache"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore"
 	"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule"
 	"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner"
+	"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
 	"github.com/bufbuild/buf/private/pkg/app/appext"
 	"github.com/bufbuild/buf/private/pkg/filelock"
 	"github.com/bufbuild/buf/private/pkg/normalpath"
 	"github.com/bufbuild/buf/private/pkg/storage/storageos"
 )
 
 var (
-	// AllCacheModuleRelDirPaths are all directory paths for all time concerning the module cache.
+	// AllCacheRelDirPaths are all directory paths for all time
+	// concerning the module and plugin caches.
 	//
 	// These are normalized.
 	// These are relative to container.CacheDirPath().
 	//
 	// This variable is used for clearing the cache.
-	AllCacheModuleRelDirPaths = []string{
-		v1beta1CacheModuleDataRelDirPath,
-		v1beta1CacheModuleLockRelDirPath,
+	AllCacheRelDirPaths = []string{
 		v1CacheModuleDataRelDirPath,
 		v1CacheModuleLockRelDirPath,
 		v1CacheModuleSumRelDirPath,
+		v1beta1CacheModuleDataRelDirPath,
+		v1beta1CacheModuleLockRelDirPath,
 		v2CacheModuleRelDirPath,
-		v3CacheModuleRelDirPath,
 		v3CacheCommitsRelDirPath,
-		v3CacheWKTRelDirPath,
 		v3CacheModuleLockRelDirPath,
+		v3CacheModuleRelDirPath,
+		v3CachePluginRelDirPath,
+		v3CacheWKTRelDirPath,
+		v3CacheWasmRuntimeRelDirPath,
 	}
 
 	// v1CacheModuleDataRelDirPath is the relative path to the cache directory where module data
@@ -103,6 +111,10 @@ var (
 	//
 	// Normalized.
 	v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks")
+	// v3CachePluginRelDirPath is the relative path to the files cache directory in its newest iteration.
+	//
+	// Normalized.
+	v3CachePluginRelDirPath = normalpath.Join("v3", "plugins")
 	// v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration.
 	// This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime.
 	//
@@ -146,6 +158,21 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er
 	)
 }
 
+// NewPluginDataProvider returns a new PluginDataProvider while creating the
+// required cache directories.
+func NewPluginDataProvider(container appext.Container) (bufplugin.PluginDataProvider, error) {
+	clientConfig, err := NewConnectClientConfig(container)
+	if err != nil {
+		return nil, err
+	}
+	return newPluginDataProvider(
+		container,
+		bufregistryapiplugin.NewClientProvider(
+			clientConfig,
+		),
+	)
+}
+
 // CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime.
 //
 // This is used by the Wasm runtime to cache compiled Wasm plugins. This is an
@@ -241,6 +268,33 @@ func newCommitProvider(
 	), nil
 }
 
+func newPluginDataProvider(
+	container appext.Container,
+	pluginClientProvider bufregistryapiplugin.ClientProvider,
+) (bufplugin.PluginDataProvider, error) {
+	if err := createCacheDir(container.CacheDirPath(), v3CachePluginRelDirPath); err != nil {
+		return nil, err
+	}
+	fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePluginRelDirPath)
+	storageosProvider := storageos.NewProvider() // No symlinks.
+	cacheBucket, err := storageosProvider.NewReadWriteBucket(fullCacheDirPath)
+	if err != nil {
+		return nil, err
+	}
+	delegateModuleDataProvider := bufpluginapi.NewPluingDataProvider(
+		container.Logger(),
+		pluginClientProvider,
+	)
+	return bufplugincache.NewPluginDataProvider(
+		container.Logger(),
+		delegateModuleDataProvider,
+		bufpluginstore.NewPluginDataStore(
+			container.Logger(),
+			cacheBucket,
+		),
+	), nil
+}
+
 func createCacheDir(baseCacheDirPath string, relDirPath string) error {
 	baseCacheDirPath = normalpath.Unnormalize(baseCacheDirPath)
 	relDirPath = normalpath.Unnormalize(relDirPath)

private/buf/bufcli/controller.go

@@ -17,8 +17,10 @@ package bufcli
 import (
 	"github.com/bufbuild/buf/private/buf/bufctl"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
 	"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule"
 	"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner"
+	"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
 	"github.com/bufbuild/buf/private/pkg/app/appext"
 )
 
@@ -39,6 +41,7 @@ func NewController(
 	}
 	moduleClientProvider := bufregistryapimodule.NewClientProvider(clientConfig)
 	ownerClientProvider := bufregistryapiowner.NewClientProvider(clientConfig)
+	pluginClientProvider := bufregistryapiplugin.NewClientProvider(clientConfig)
 	moduleDataProvider, err := newModuleDataProvider(container, moduleClientProvider, ownerClientProvider)
 	if err != nil {
 		return nil, err
@@ -47,6 +50,10 @@ func NewController(
 	if err != nil {
 		return nil, err
 	}
+	pluginDataProvider, err := newPluginDataProvider(container, pluginClientProvider)
+	if err != nil {
+		return nil, err
+	}
 	wktStore, err := NewWKTStore(container)
 	if err != nil {
 		return nil, err
@@ -58,6 +65,8 @@ func NewController(
 		bufmoduleapi.NewModuleKeyProvider(container.Logger(), moduleClientProvider),
 		moduleDataProvider,
 		commitProvider,
+		bufpluginapi.NewPluginKeyProvider(container.Logger(), pluginClientProvider),
+		pluginDataProvider,
 		wktStore,
 		// TODO FUTURE: Delete defaultHTTPClient and use the one from newConfig
 		defaultHTTPClient,

private/buf/bufcli/plugin_key_provider.go

@@ -0,0 +1,36 @@
+// Copyright 2020-2024 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bufcli
+
+import (
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi"
+	"github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin"
+	"github.com/bufbuild/buf/private/pkg/app/appext"
+)
+
+// NewPluginKeyProvider returns a new PluginKeyProvider.
+func NewPluginKeyProvider(container appext.Container) (bufplugin.PluginKeyProvider, error) {
+	clientConfig, err := NewConnectClientConfig(container)
+	if err != nil {
+		return nil, err
+	}
+	return bufpluginapi.NewPluginKeyProvider(
+		container.Logger(),
+		bufregistryapiplugin.NewClientProvider(
+			clientConfig,
+		),
+	), nil
+}

private/buf/bufctl/controller.go

@@ -34,6 +34,7 @@ import (
 	"github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
 	"github.com/bufbuild/buf/private/bufpkg/bufparse"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
 	"github.com/bufbuild/buf/private/bufpkg/bufreflect"
 	"github.com/bufbuild/buf/private/gen/data/datawkt"
 	imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1"
@@ -127,6 +128,8 @@ type Controller interface {
 		defaultMessageEncoding buffetch.MessageEncoding,
 		options ...FunctionOption,
 	) error
+	PluginKeyProvider() bufplugin.PluginKeyProvider
+	PluginDataProvider() bufplugin.PluginDataProvider
 }
 
 func NewController(
@@ -136,6 +139,8 @@ func NewController(
 	moduleKeyProvider bufmodule.ModuleKeyProvider,
 	moduleDataProvider bufmodule.ModuleDataProvider,
 	commitProvider bufmodule.CommitProvider,
+	pluginKeyProvider bufplugin.PluginKeyProvider,
+	pluginDataProvider bufplugin.PluginDataProvider,
 	wktStore bufwktstore.Store,
 	httpClient *http.Client,
 	httpauthAuthenticator httpauth.Authenticator,
@@ -149,6 +154,8 @@ func NewController(
 		moduleKeyProvider,
 		moduleDataProvider,
 		commitProvider,
+		pluginKeyProvider,
+		pluginDataProvider,
 		wktStore,
 		httpClient,
 		httpauthAuthenticator,
@@ -170,6 +177,8 @@ type controller struct {
 	moduleDataProvider bufmodule.ModuleDataProvider
 	graphProvider      bufmodule.GraphProvider
 	commitProvider     bufmodule.CommitProvider
+	pluginKeyProvider  bufplugin.PluginKeyProvider
+	pluginDataProvider bufplugin.PluginDataProvider
 	wktStore           bufwktstore.Store
 
 	disableSymlinks           bool
@@ -192,6 +201,8 @@ func newController(
 	moduleKeyProvider bufmodule.ModuleKeyProvider,
 	moduleDataProvider bufmodule.ModuleDataProvider,
 	commitProvider bufmodule.CommitProvider,
+	pluginKeyProvider bufplugin.PluginKeyProvider,
+	pluginDataProvider bufplugin.PluginDataProvider,
 	wktStore bufwktstore.Store,
 	httpClient *http.Client,
 	httpauthAuthenticator httpauth.Authenticator,
@@ -204,6 +215,8 @@ func newController(
 		graphProvider:      graphProvider,
 		moduleDataProvider: moduleDataProvider,
 		commitProvider:     commitProvider,
+		pluginKeyProvider:  pluginKeyProvider,
+		pluginDataProvider: pluginDataProvider,
 		wktStore:           wktStore,
 	}
 	for _, option := range options {
@@ -695,6 +708,14 @@ func (c *controller) PutMessage(
 	return errors.Join(err, writeCloser.Close())
 }
 
+func (c *controller) PluginKeyProvider() bufplugin.PluginKeyProvider {
+	return c.pluginKeyProvider
+}
+
+func (c *controller) PluginDataProvider() bufplugin.PluginDataProvider {
+	return c.pluginDataProvider
+}
+
 func (c *controller) getImage(
 	ctx context.Context,
 	input string,

private/buf/bufmigrate/migrator.go

@@ -29,6 +29,7 @@ import (
 	"github.com/bufbuild/buf/private/bufpkg/bufconfig"
 	"github.com/bufbuild/buf/private/bufpkg/bufmodule"
 	"github.com/bufbuild/buf/private/bufpkg/bufparse"
+	"github.com/bufbuild/buf/private/bufpkg/bufplugin"
 	"github.com/bufbuild/buf/private/pkg/normalpath"
 	"github.com/bufbuild/buf/private/pkg/slicesext"
 	"github.com/bufbuild/buf/private/pkg/storage"
@@ -400,6 +401,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles(
 			bufLock, err = bufconfig.NewBufLockFile(
 				bufconfig.FileVersionV2,
 				resolvedLockEntries,
+				nil,
 			)
 			if err != nil {
 				return nil, nil, err
@@ -444,6 +446,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles(
 		bufLock, err = bufconfig.NewBufLockFile(
 			bufconfig.FileVersionV2,
 			resolvedDepModuleKeys,
+			nil, // Plugins are not supported in v1.
 		)
 		if err != nil {
 			return nil, nil, err
@@ -693,7 +696,7 @@ func equivalentCheckConfigInV2(
 ) (bufconfig.CheckConfig, error) {
 	// No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1.
 	// TODO: If we ever need v3, then we will have to deal with this.
-	client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime))
+	client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime, bufplugin.NopPluginKeyProvider, bufplugin.NopPluginDataProvider))
 	if err != nil {
 		return nil, err
 	}