diff --git a/private/buf/bufcli/cache.go b/private/buf/bufcli/cache.go index cfe01a9579..99ecb9381b 100644 --- a/private/buf/bufcli/cache.go +++ b/private/buf/bufcli/cache.go @@ -26,8 +26,13 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulecache" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulestore" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufplugincache" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore" "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule" "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/filelock" "github.com/bufbuild/buf/private/pkg/normalpath" @@ -35,23 +40,26 @@ import ( ) var ( - // AllCacheModuleRelDirPaths are all directory paths for all time concerning the module cache. + // AllCacheRelDirPaths are all directory paths for all time + // concerning the module and plugin caches. // // These are normalized. // These are relative to container.CacheDirPath(). // // This variable is used for clearing the cache. - AllCacheModuleRelDirPaths = []string{ - v1beta1CacheModuleDataRelDirPath, - v1beta1CacheModuleLockRelDirPath, + AllCacheRelDirPaths = []string{ v1CacheModuleDataRelDirPath, v1CacheModuleLockRelDirPath, v1CacheModuleSumRelDirPath, + v1beta1CacheModuleDataRelDirPath, + v1beta1CacheModuleLockRelDirPath, v2CacheModuleRelDirPath, - v3CacheModuleRelDirPath, v3CacheCommitsRelDirPath, - v3CacheWKTRelDirPath, v3CacheModuleLockRelDirPath, + v3CacheModuleRelDirPath, + v3CachePluginRelDirPath, + v3CacheWKTRelDirPath, + v3CacheWasmRuntimeRelDirPath, } // v1CacheModuleDataRelDirPath is the relative path to the cache directory where module data @@ -103,6 +111,10 @@ var ( // // Normalized. v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks") + // v3CachePluginRelDirPath is the relative path to the files cache directory in its newest iteration. + // + // Normalized. + v3CachePluginRelDirPath = normalpath.Join("v3", "plugins") // v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration. // This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime. // @@ -146,6 +158,21 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er ) } +// NewPluginDataProvider returns a new PluginDataProvider while creating the +// required cache directories. +func NewPluginDataProvider(container appext.Container) (bufplugin.PluginDataProvider, error) { + clientConfig, err := NewConnectClientConfig(container) + if err != nil { + return nil, err + } + return newPluginDataProvider( + container, + bufregistryapiplugin.NewClientProvider( + clientConfig, + ), + ) +} + // CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime. // // This is used by the Wasm runtime to cache compiled Wasm plugins. This is an @@ -241,6 +268,33 @@ func newCommitProvider( ), nil } +func newPluginDataProvider( + container appext.Container, + pluginClientProvider bufregistryapiplugin.ClientProvider, +) (bufplugin.PluginDataProvider, error) { + if err := createCacheDir(container.CacheDirPath(), v3CachePluginRelDirPath); err != nil { + return nil, err + } + fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CachePluginRelDirPath) + storageosProvider := storageos.NewProvider() // No symlinks. + cacheBucket, err := storageosProvider.NewReadWriteBucket(fullCacheDirPath) + if err != nil { + return nil, err + } + delegateModuleDataProvider := bufpluginapi.NewPluingDataProvider( + container.Logger(), + pluginClientProvider, + ) + return bufplugincache.NewPluginDataProvider( + container.Logger(), + delegateModuleDataProvider, + bufpluginstore.NewPluginDataStore( + container.Logger(), + cacheBucket, + ), + ), nil +} + func createCacheDir(baseCacheDirPath string, relDirPath string) error { baseCacheDirPath = normalpath.Unnormalize(baseCacheDirPath) relDirPath = normalpath.Unnormalize(relDirPath) diff --git a/private/buf/bufcli/controller.go b/private/buf/bufcli/controller.go index 78058ff7ca..15abc9d04e 100644 --- a/private/buf/bufcli/controller.go +++ b/private/buf/bufcli/controller.go @@ -17,8 +17,10 @@ package bufcli import ( "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule" "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiowner" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" "github.com/bufbuild/buf/private/pkg/app/appext" ) @@ -39,6 +41,7 @@ func NewController( } moduleClientProvider := bufregistryapimodule.NewClientProvider(clientConfig) ownerClientProvider := bufregistryapiowner.NewClientProvider(clientConfig) + pluginClientProvider := bufregistryapiplugin.NewClientProvider(clientConfig) moduleDataProvider, err := newModuleDataProvider(container, moduleClientProvider, ownerClientProvider) if err != nil { return nil, err @@ -47,6 +50,10 @@ func NewController( if err != nil { return nil, err } + pluginDataProvider, err := newPluginDataProvider(container, pluginClientProvider) + if err != nil { + return nil, err + } wktStore, err := NewWKTStore(container) if err != nil { return nil, err @@ -58,6 +65,8 @@ func NewController( bufmoduleapi.NewModuleKeyProvider(container.Logger(), moduleClientProvider), moduleDataProvider, commitProvider, + bufpluginapi.NewPluginKeyProvider(container.Logger(), pluginClientProvider), + pluginDataProvider, wktStore, // TODO FUTURE: Delete defaultHTTPClient and use the one from newConfig defaultHTTPClient, diff --git a/private/buf/bufcli/plugin_key_provider.go b/private/buf/bufcli/plugin_key_provider.go new file mode 100644 index 0000000000..a6e11f695a --- /dev/null +++ b/private/buf/bufcli/plugin_key_provider.go @@ -0,0 +1,36 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufcli + +import ( + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" + "github.com/bufbuild/buf/private/pkg/app/appext" +) + +// NewPluginKeyProvider returns a new PluginKeyProvider. +func NewPluginKeyProvider(container appext.Container) (bufplugin.PluginKeyProvider, error) { + clientConfig, err := NewConnectClientConfig(container) + if err != nil { + return nil, err + } + return bufpluginapi.NewPluginKeyProvider( + container.Logger(), + bufregistryapiplugin.NewClientProvider( + clientConfig, + ), + ), nil +} diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index e22b501de1..c7f80d3e36 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -34,6 +34,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/bufpkg/bufreflect" "github.com/bufbuild/buf/private/gen/data/datawkt" imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1" @@ -127,6 +128,8 @@ type Controller interface { defaultMessageEncoding buffetch.MessageEncoding, options ...FunctionOption, ) error + PluginKeyProvider() bufplugin.PluginKeyProvider + PluginDataProvider() bufplugin.PluginDataProvider } func NewController( @@ -136,6 +139,8 @@ func NewController( moduleKeyProvider bufmodule.ModuleKeyProvider, moduleDataProvider bufmodule.ModuleDataProvider, commitProvider bufmodule.CommitProvider, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, wktStore bufwktstore.Store, httpClient *http.Client, httpauthAuthenticator httpauth.Authenticator, @@ -149,6 +154,8 @@ func NewController( moduleKeyProvider, moduleDataProvider, commitProvider, + pluginKeyProvider, + pluginDataProvider, wktStore, httpClient, httpauthAuthenticator, @@ -170,6 +177,8 @@ type controller struct { moduleDataProvider bufmodule.ModuleDataProvider graphProvider bufmodule.GraphProvider commitProvider bufmodule.CommitProvider + pluginKeyProvider bufplugin.PluginKeyProvider + pluginDataProvider bufplugin.PluginDataProvider wktStore bufwktstore.Store disableSymlinks bool @@ -192,6 +201,8 @@ func newController( moduleKeyProvider bufmodule.ModuleKeyProvider, moduleDataProvider bufmodule.ModuleDataProvider, commitProvider bufmodule.CommitProvider, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, wktStore bufwktstore.Store, httpClient *http.Client, httpauthAuthenticator httpauth.Authenticator, @@ -204,6 +215,8 @@ func newController( graphProvider: graphProvider, moduleDataProvider: moduleDataProvider, commitProvider: commitProvider, + pluginKeyProvider: pluginKeyProvider, + pluginDataProvider: pluginDataProvider, wktStore: wktStore, } for _, option := range options { @@ -695,6 +708,14 @@ func (c *controller) PutMessage( return errors.Join(err, writeCloser.Close()) } +func (c *controller) PluginKeyProvider() bufplugin.PluginKeyProvider { + return c.pluginKeyProvider +} + +func (c *controller) PluginDataProvider() bufplugin.PluginDataProvider { + return c.pluginDataProvider +} + func (c *controller) getImage( ctx context.Context, input string, diff --git a/private/buf/bufmigrate/migrator.go b/private/buf/bufmigrate/migrator.go index f36f21f37c..8843345695 100644 --- a/private/buf/bufmigrate/migrator.go +++ b/private/buf/bufmigrate/migrator.go @@ -29,6 +29,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/normalpath" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" @@ -400,6 +401,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles( bufLock, err = bufconfig.NewBufLockFile( bufconfig.FileVersionV2, resolvedLockEntries, + nil, ) if err != nil { return nil, nil, err @@ -444,6 +446,7 @@ func (m *migrator) buildBufYAMLAndBufLockFiles( bufLock, err = bufconfig.NewBufLockFile( bufconfig.FileVersionV2, resolvedDepModuleKeys, + nil, // Plugins are not supported in v1. ) if err != nil { return nil, nil, err @@ -693,7 +696,7 @@ func equivalentCheckConfigInV2( ) (bufconfig.CheckConfig, error) { // No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1. // TODO: If we ever need v3, then we will have to deal with this. - client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime)) + client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime, bufplugin.NopPluginKeyProvider, bufplugin.NopPluginDataProvider)) if err != nil { return nil, err } diff --git a/private/buf/bufworkspace/workspace_dep_manager.go b/private/buf/bufworkspace/workspace_dep_manager.go index 1b2f8f5239..d035a3c8cd 100644 --- a/private/buf/bufworkspace/workspace_dep_manager.go +++ b/private/buf/bufworkspace/workspace_dep_manager.go @@ -18,10 +18,13 @@ import ( "context" "errors" "io/fs" + "sort" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/syserror" ) @@ -39,11 +42,13 @@ type WorkspaceDepManager interface { BufLockFileDigestType() bufmodule.DigestType // ExisingBufLockFileDepModuleKeys returns the ModuleKeys from the buf.lock file. ExistingBufLockFileDepModuleKeys(ctx context.Context) ([]bufmodule.ModuleKey, error) + // ExistingBufLockFileRemotePluginKeys returns the PluginKeys from the buf.lock file. + ExistingBufLockFileRemotePluginKeys(ctx context.Context) ([]bufplugin.PluginKey, error) // UpdateBufLockFile updates the lock file that backs the Workspace to contain exactly - // the given ModuleKeys. + // the given ModuleKeys and PluginKeys. // // If a buf.lock does not exist, one will be created. - UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey) error + UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey, remotePluginKeys []bufplugin.PluginKey) error // ConfiguredDepModuleRefs returns the configured dependencies of the Workspace as ModuleRefs. // // These come from buf.yaml files. @@ -56,6 +61,16 @@ type WorkspaceDepManager interface { // // Sorted. ConfiguredDepModuleRefs(ctx context.Context) ([]bufparse.Ref, error) + // ConfiguredRemotePluginRefs returns the configured remote plugins of the Workspace as PluginRefs. + // + // These come from buf.yaml files. + // + // The PluginRefs in this list will be unique by FullName. If there are two PluginRefs + // in the buf.yaml with the same FullName but different Refs, an error will be given + // at workspace constructions. + // + // Sorted. + ConfiguredRemotePluginRefs(ctx context.Context) ([]bufparse.Ref, error) isWorkspaceDepManager() } @@ -110,7 +125,7 @@ func (w *workspaceDepManager) ConfiguredDepModuleRefs(ctx context.Context) ([]bu } case bufconfig.FileVersionV2: if !w.isV2 { - return nil, syserror.Newf("buf.yaml at %q did had version %v but expected v12", w.targetSubDirPath, fileVersion) + return nil, syserror.Newf("buf.yaml at %q did had version %v but expected v2", w.targetSubDirPath, fileVersion) } default: return nil, syserror.Newf("unknown FileVersion: %v", fileVersion) @@ -118,6 +133,50 @@ func (w *workspaceDepManager) ConfiguredDepModuleRefs(ctx context.Context) ([]bu return bufYAMLFile.ConfiguredDepModuleRefs(), nil } +func (w *workspaceDepManager) ConfiguredRemotePluginRefs(ctx context.Context) ([]bufparse.Ref, error) { + bufYAMLFile, err := bufconfig.GetBufYAMLFileForPrefix(ctx, w.bucket, w.targetSubDirPath) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, err + } + } + if bufYAMLFile == nil { + return nil, nil + } + switch fileVersion := bufYAMLFile.FileVersion(); fileVersion { + case bufconfig.FileVersionV1Beta1, bufconfig.FileVersionV1: + if w.isV2 { + return nil, syserror.Newf("buf.yaml at %q did had version %v but expected v1beta1, v1", w.targetSubDirPath, fileVersion) + } + // Plugins are not supported in versions less than v2. + return nil, nil + case bufconfig.FileVersionV2: + if !w.isV2 { + return nil, syserror.Newf("buf.yaml at %q did had version %v but expected v2", w.targetSubDirPath, fileVersion) + } + default: + return nil, syserror.Newf("unknown FileVersion: %v", fileVersion) + } + pluginRefs := slicesext.Filter( + slicesext.Map( + bufYAMLFile.PluginConfigs(), + func(value bufconfig.PluginConfig) bufparse.Ref { + return value.Ref() + }, + ), + func(value bufparse.Ref) bool { + return value != nil + }, + ) + sort.Slice( + pluginRefs, + func(i int, j int) bool { + return pluginRefs[i].FullName().String() < pluginRefs[j].FullName().String() + }, + ) + return pluginRefs, nil +} + func (w *workspaceDepManager) BufLockFileDigestType() bufmodule.DigestType { if w.isV2 { return bufmodule.DigestTypeB5 @@ -136,11 +195,22 @@ func (w *workspaceDepManager) ExistingBufLockFileDepModuleKeys(ctx context.Conte return bufLockFile.DepModuleKeys(), nil } -func (w *workspaceDepManager) UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey) error { +func (w *workspaceDepManager) ExistingBufLockFileRemotePluginKeys(ctx context.Context) ([]bufplugin.PluginKey, error) { + bufLockFile, err := bufconfig.GetBufLockFileForPrefix(ctx, w.bucket, w.targetSubDirPath) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, err + } + return bufLockFile.RemotePluginKeys(), nil +} + +func (w *workspaceDepManager) UpdateBufLockFile(ctx context.Context, depModuleKeys []bufmodule.ModuleKey, remotePluginKeys []bufplugin.PluginKey) error { var bufLockFile bufconfig.BufLockFile var err error if w.isV2 { - bufLockFile, err = bufconfig.NewBufLockFile(bufconfig.FileVersionV2, depModuleKeys) + bufLockFile, err = bufconfig.NewBufLockFile(bufconfig.FileVersionV2, depModuleKeys, remotePluginKeys) if err != nil { return err } @@ -154,7 +224,10 @@ func (w *workspaceDepManager) UpdateBufLockFile(ctx context.Context, depModuleKe } else { fileVersion = existingBufYAMLFile.FileVersion() } - bufLockFile, err = bufconfig.NewBufLockFile(fileVersion, depModuleKeys) + if len(remotePluginKeys) > 0 { + return syserror.Newf("remote plugins are not supported for v1 buf.yaml files") + } + bufLockFile, err = bufconfig.NewBufLockFile(fileVersion, depModuleKeys, nil) if err != nil { return err } diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index 57dcc15bfc..0bb6095b2d 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -62,6 +62,7 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlsbreakingrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlslintrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modopen" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/plugin/pluginupdate" "github.com/bufbuild/buf/private/buf/cmd/buf/command/push" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/module/modulecommit/modulecommitaddlabel" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/module/modulecommit/modulecommitinfo" @@ -171,6 +172,13 @@ func NewRootCommand(name string) *appcmd.Command { modlsbreakingrules.NewCommand("ls-breaking-rules", builder), }, }, + { + Use: "plugin", + Short: "Work with plugins", + SubCommands: []*appcmd.Command{ + pluginupdate.NewCommand("update", builder), + }, + }, { Use: "registry", Short: "Manage assets on the Buf Schema Registry", diff --git a/private/buf/cmd/buf/buf_test.go b/private/buf/cmd/buf/buf_test.go index 1bb1c22b16..5d4780ed95 100644 --- a/private/buf/cmd/buf/buf_test.go +++ b/private/buf/cmd/buf/buf_test.go @@ -35,6 +35,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" imagev1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/image/v1" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appcmd/appcmdtesting" @@ -1348,7 +1349,14 @@ func TestCheckLsBreakingRulesFromConfigExceptDeprecated(t *testing.T) { t.Run(version.String(), func(t *testing.T) { t.Parallel() // Do not need any custom lint/breaking plugins here. - client, err := bufcheck.NewClient(slogtestext.NewLogger(t), bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime)) + client, err := bufcheck.NewClient( + slogtestext.NewLogger(t), + bufcheck.NewRunnerProvider( + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), + ) require.NoError(t, err) allRules, err := client.AllRules(context.Background(), check.RuleTypeBreaking, version) require.NoError(t, err) diff --git a/private/buf/cmd/buf/command/beta/lsp/lsp.go b/private/buf/cmd/buf/command/beta/lsp/lsp.go index 24f4bec061..f63ca7d7ec 100644 --- a/private/buf/cmd/buf/command/beta/lsp/lsp.go +++ b/private/buf/cmd/buf/command/beta/lsp/lsp.go @@ -115,7 +115,11 @@ func run( }() checkClient, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(wasmRuntime), + bufcheck.NewRunnerProvider( + wasmRuntime, + controller.PluginKeyProvider(), + controller.PluginDataProvider(), + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/breaking/breaking.go b/private/buf/cmd/buf/command/breaking/breaking.go index 71b699c9a2..dde2bd7649 100644 --- a/private/buf/cmd/buf/command/breaking/breaking.go +++ b/private/buf/cmd/buf/command/breaking/breaking.go @@ -220,7 +220,11 @@ func run( for i, imageWithConfig := range imageWithConfigs { client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(wasmRuntime), + bufcheck.NewRunnerProvider( + wasmRuntime, + controller.PluginKeyProvider(), + controller.PluginDataProvider(), + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/config/internal/internal.go b/private/buf/cmd/buf/command/config/internal/internal.go index f1dfeb5524..0d772cedef 100644 --- a/private/buf/cmd/buf/command/config/internal/internal.go +++ b/private/buf/cmd/buf/command/config/internal/internal.go @@ -183,6 +183,10 @@ func lsRun( return err } } + controller, err := bufcli.NewController(container) + if err != nil { + return err + } wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) if err != nil { return err @@ -196,7 +200,11 @@ func lsRun( }() client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(wasmRuntime), + bufcheck.NewRunnerProvider( + wasmRuntime, + controller.PluginKeyProvider(), + controller.PluginDataProvider(), + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/dep/depupdate/depupdate.go b/private/buf/cmd/buf/command/dep/depupdate/depupdate.go index 4e3bef0749..facec6e958 100644 --- a/private/buf/cmd/buf/command/dep/depupdate/depupdate.go +++ b/private/buf/cmd/buf/command/dep/depupdate/depupdate.go @@ -45,8 +45,8 @@ func NewCommand( flags := newFlags() return &appcmd.Command{ Use: name + " ", - Short: "Update pinned dependencies in a buf.lock", - Long: `Fetch the latest digests for the specified references in buf.yaml, + Short: "Update pinned module dependencies in a buf.lock", + Long: `Fetch the latest digests for the specified module references in buf.yaml, and write them and their transitive dependencies to buf.lock. The first argument is the directory of the local module to update. @@ -139,6 +139,10 @@ func run( logger.Warn(fmt.Sprintf("No configured dependencies were found to update in %q.", dirPath)) return nil } + existingRemotePluginKeys, err := workspaceDepManager.ExistingBufLockFileRemotePluginKeys(ctx) + if err != nil { + return err + } // We're about to edit the buf.lock file on disk. If we have a subsequent error, // attempt to revert the buf.lock file. @@ -148,11 +152,11 @@ func run( // overlay the new buf.lock file in a union bucket. defer func() { if retErr != nil { - retErr = errors.Join(retErr, workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys)) + retErr = errors.Join(retErr, workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, existingRemotePluginKeys)) } }() // Edit the buf.lock file with the unpruned dependencies. - if err := workspaceDepManager.UpdateBufLockFile(ctx, configuredDepModuleKeys); err != nil { + if err := workspaceDepManager.UpdateBufLockFile(ctx, configuredDepModuleKeys, existingRemotePluginKeys); err != nil { return err } workspace, err := controller.GetWorkspace(ctx, dirPath, bufctl.WithIgnoreAndDisallowV1BufWorkYAMLs()) diff --git a/private/buf/cmd/buf/command/dep/internal/internal.go b/private/buf/cmd/buf/command/dep/internal/internal.go index d5d57cf584..04c94df4ab 100644 --- a/private/buf/cmd/buf/command/dep/internal/internal.go +++ b/private/buf/cmd/buf/command/dep/internal/internal.go @@ -106,7 +106,11 @@ func Prune( if err := validateModuleKeysContains(bufYAMLBasedDepModuleKeys, depModuleKeys); err != nil { return err } - return workspaceDepManager.UpdateBufLockFile(ctx, depModuleKeys) + existingRemotePluginKeys, err := workspaceDepManager.ExistingBufLockFileRemotePluginKeys(ctx) + if err != nil { + return err + } + return workspaceDepManager.UpdateBufLockFile(ctx, depModuleKeys, existingRemotePluginKeys) } // LogUnusedConfiugredDepsForWorkspace takes a workspace and logs the unused configured diff --git a/private/buf/cmd/buf/command/lint/lint.go b/private/buf/cmd/buf/command/lint/lint.go index 496a0d6638..10260a9190 100644 --- a/private/buf/cmd/buf/command/lint/lint.go +++ b/private/buf/cmd/buf/command/lint/lint.go @@ -145,7 +145,11 @@ func run( for _, imageWithConfig := range imageWithConfigs { client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(wasmRuntime), + bufcheck.NewRunnerProvider( + wasmRuntime, + controller.PluginKeyProvider(), + controller.PluginDataProvider(), + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/mod/internal/internal.go b/private/buf/cmd/buf/command/mod/internal/internal.go index c802e34e50..38218413fb 100644 --- a/private/buf/cmd/buf/command/mod/internal/internal.go +++ b/private/buf/cmd/buf/command/mod/internal/internal.go @@ -24,6 +24,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/slicesext" @@ -175,7 +176,11 @@ func lsRun( // BufYAMLFiles <=v1 never had plugins. client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime), + bufcheck.NewRunnerProvider( + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(container.Stderr()), ) if err != nil { diff --git a/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go b/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go new file mode 100644 index 0000000000..3965bb4cee --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go @@ -0,0 +1,151 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginupdate + +import ( + "context" + "errors" + "fmt" + + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/spf13/pflag" +) + +const ( + onlyFlagName = "only" +) + +// NewCommand returns a new Command. +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Update pinned remote plugins in a buf.lock", + Long: `Fetch the latest digests for the specified plugin references in buf.yaml. + +The first argument is the directory of the local module to update. +Defaults to "." if no argument is specified.`, + Args: appcmd.MaximumNArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + Only []string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + flagSet.StringSliceVar( + &f.Only, + onlyFlagName, + nil, + "The name of the plugin to update. When set, only this plugin is updated. May be provided multiple times", + ) + // TODO FUTURE: implement + _ = flagSet.MarkHidden(onlyFlagName) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) (retErr error) { + dirPath := "." + if container.NumArgs() > 0 { + dirPath = container.Arg(0) + } + if len(flags.Only) > 0 { + // TODO FUTURE: implement + return syserror.Newf("--%s is not implemented", onlyFlagName) + } + + logger := container.Logger() + controller, err := bufcli.NewController(container) + if err != nil { + return err + } + workspaceDepManager, err := controller.GetWorkspaceDepManager(ctx, dirPath) + if err != nil { + return err + } + configuredRemotePluginRefs, err := workspaceDepManager.ConfiguredRemotePluginRefs(ctx) + if err != nil { + return err + } + pluginKeyProvider, err := bufcli.NewPluginKeyProvider(container) + if err != nil { + return err + } + configuredRemotePluginKeys, err := pluginKeyProvider.GetPluginKeysForPluginRefs( + ctx, + configuredRemotePluginRefs, + bufplugin.DigestTypeP1, + ) + if err != nil { + return err + } + + // Store the existing buf.lock data. + existingRemotePluginKeys, err := workspaceDepManager.ExistingBufLockFileRemotePluginKeys(ctx) + if err != nil { + return err + } + if configuredRemotePluginKeys == nil && existingRemotePluginKeys == nil { + // No new configured remote plugins were found, and no existing buf.lock deps were found, so there + // is nothing to update, we can return here. + // This ensures we do not create an empty buf.lock when one did not exist in the first + // place and we do not need to go through the entire operation of updating non-existent + // deps and building the image for tamper-proofing. + logger.Warn(fmt.Sprintf("No configured remote plugins were found to update in %q.", dirPath)) + return nil + } + existingDepModuleKeys, err := workspaceDepManager.ExistingBufLockFileDepModuleKeys(ctx) + if err != nil { + return err + } + + // We're about to edit the buf.lock file on disk. If we have a subsequent error, + // attempt to revert the buf.lock file. + // + // TODO FUTURE: We should be able to update the buf.lock file in an in-memory bucket, then do the rebuild, + // and if the rebuild is successful, then actually write to disk. It shouldn't even be that much work - just + // overlay the new buf.lock file in a union bucket. + defer func() { + if retErr != nil { + retErr = errors.Join(retErr, workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, existingRemotePluginKeys)) + } + }() + // Edit the buf.lock file with the updated remote plugins. + if err := workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, configuredRemotePluginKeys); err != nil { + return err + } + return nil +} diff --git a/private/buf/cmd/buf/command/plugin/pluginupdate/usage.gen.go b/private/buf/cmd/buf/command/plugin/pluginupdate/usage.gen.go new file mode 100644 index 0000000000..23136ff2a4 --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginupdate/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package pluginupdate + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/buf/command/registry/registrycc/registrycc.go b/private/buf/cmd/buf/command/registry/registrycc/registrycc.go index 3eb7722b8f..ae8038bb4e 100644 --- a/private/buf/cmd/buf/command/registry/registrycc/registrycc.go +++ b/private/buf/cmd/buf/command/registry/registrycc/registrycc.go @@ -65,7 +65,7 @@ func run( container appext.Container, flags *flags, ) error { - for _, cacheModuleRelDirPath := range bufcli.AllCacheModuleRelDirPaths { + for _, cacheModuleRelDirPath := range bufcli.AllCacheRelDirPaths { dirPath := filepath.Join(container.CacheDirPath(), normalpath.Unnormalize(cacheModuleRelDirPath)) fileInfo, err := os.Stat(dirPath) if err != nil { diff --git a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go index 1de59b0149..b3b24fb487 100644 --- a/private/buf/cmd/protoc-gen-buf-breaking/breaking.go +++ b/private/buf/cmd/protoc-gen-buf-breaking/breaking.go @@ -28,6 +28,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/protodescriptor" "github.com/bufbuild/buf/private/pkg/protoencoding" @@ -125,7 +126,11 @@ func handle( // The protoc plugins do not support custom lint/breaking change plugins for now. client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime), + bufcheck.NewRunnerProvider( + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(pluginEnv.Stderr), ) if err != nil { diff --git a/private/buf/cmd/protoc-gen-buf-lint/lint.go b/private/buf/cmd/protoc-gen-buf-lint/lint.go index d7ed7b0a1a..0aec2e490c 100644 --- a/private/buf/cmd/protoc-gen-buf-lint/lint.go +++ b/private/buf/cmd/protoc-gen-buf-lint/lint.go @@ -27,6 +27,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/protodescriptor" "github.com/bufbuild/buf/private/pkg/protoencoding" @@ -100,7 +101,11 @@ func handle( // The protoc plugins do not support custom lint/breaking change plugins for now. client, err := bufcheck.NewClient( container.Logger(), - bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime), + bufcheck.NewRunnerProvider( + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), bufcheck.ClientWithStderr(pluginEnv.Stderr), ) if err != nil { diff --git a/private/bufpkg/bufcheck/breaking_test.go b/private/bufpkg/bufcheck/breaking_test.go index 2bab7cbf4d..8f4232ef95 100644 --- a/private/bufpkg/bufcheck/breaking_test.go +++ b/private/bufpkg/bufcheck/breaking_test.go @@ -30,6 +30,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/slogtestext" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/wasm" @@ -1344,7 +1345,11 @@ func testBreaking( require.NoError(t, err) breakingConfig := workspace.GetBreakingConfigForOpaqueID(opaqueID) require.NotNil(t, breakingConfig) - client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider(wasm.UnimplementedRuntime)) + client, err := bufcheck.NewClient(logger, bufcheck.NewRunnerProvider( + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + )) require.NoError(t, err) err = client.Breaking( ctx, diff --git a/private/bufpkg/bufcheck/bufcheck.go b/private/bufpkg/bufcheck/bufcheck.go index 91a6cda45f..5d02ddae0d 100644 --- a/private/bufpkg/bufcheck/bufcheck.go +++ b/private/bufpkg/bufcheck/bufcheck.go @@ -22,6 +22,7 @@ import ( "buf.build/go/bufplugin/check" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/bufbuild/buf/private/pkg/wasm" @@ -180,8 +181,16 @@ func (r RunnerProviderFunc) NewRunner(pluginConfig bufconfig.PluginConfig) (plug // - bufconfig.PluginConfigTypeLocalWasm // // If the PluginConfigType is not supported, an error is returned. -func NewRunnerProvider(wasmRuntime wasm.Runtime) RunnerProvider { - return newRunnerProvider(wasmRuntime) +func NewRunnerProvider( + wasmRuntime wasm.Runtime, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, +) RunnerProvider { + return newRunnerProvider( + wasmRuntime, + pluginKeyProvider, + pluginDataProvider, + ) } // NewClient returns a new Client. diff --git a/private/bufpkg/bufcheck/lint_test.go b/private/bufpkg/bufcheck/lint_test.go index 626f3738b2..56a19189d3 100644 --- a/private/bufpkg/bufcheck/lint_test.go +++ b/private/bufpkg/bufcheck/lint_test.go @@ -27,6 +27,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufcheck" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/slogtestext" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/wasm" @@ -1355,7 +1356,11 @@ func testLintWithOptions( }) client, err := bufcheck.NewClient( logger, - bufcheck.NewRunnerProvider(wasmRuntime), + bufcheck.NewRunnerProvider( + wasmRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), ) require.NoError(t, err) err = client.Lint( diff --git a/private/bufpkg/bufcheck/multi_client_test.go b/private/bufpkg/bufcheck/multi_client_test.go index 8fde74ddae..39c6dc8198 100644 --- a/private/bufpkg/bufcheck/multi_client_test.go +++ b/private/bufpkg/bufcheck/multi_client_test.go @@ -24,6 +24,7 @@ import ( "buf.build/go/bufplugin/check/checkutil" "buf.build/go/bufplugin/option" "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/slogtestext" "github.com/bufbuild/buf/private/pkg/stringutil" @@ -182,7 +183,11 @@ func TestMultiClientCannotHaveOverlappingRulesWithBuiltIn(t *testing.T) { client, err := newClient( slogtestext.NewLogger(t), - NewRunnerProvider(wasm.UnimplementedRuntime), + NewRunnerProvider( + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), ) require.NoError(t, err) duplicateBuiltInRulePluginConfig, err := bufconfig.NewLocalPluginConfig( @@ -275,7 +280,11 @@ func TestMultiClientCannotHaveOverlappingCategoriesWithBuiltIn(t *testing.T) { client, err := newClient( slogtestext.NewLogger(t), - NewRunnerProvider(wasm.UnimplementedRuntime), + NewRunnerProvider( + wasm.UnimplementedRuntime, + bufplugin.NopPluginKeyProvider, + bufplugin.NopPluginDataProvider, + ), ) require.NoError(t, err) duplicateBuiltInRulePluginConfig, err := bufconfig.NewLocalPluginConfig( diff --git a/private/bufpkg/bufcheck/runner_provider.go b/private/bufpkg/bufcheck/runner_provider.go index ca8a8ed626..4d4df349b0 100644 --- a/private/bufpkg/bufcheck/runner_provider.go +++ b/private/bufpkg/bufcheck/runner_provider.go @@ -15,7 +15,12 @@ package bufcheck import ( + "context" + "sync" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/pluginrpcutil" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/bufbuild/buf/private/pkg/wasm" @@ -23,12 +28,20 @@ import ( ) type runnerProvider struct { - wasmRuntime wasm.Runtime + wasmRuntime wasm.Runtime + pluginKeyProvider bufplugin.PluginKeyProvider + pluginDataProvider bufplugin.PluginDataProvider } -func newRunnerProvider(wasmRuntime wasm.Runtime) *runnerProvider { +func newRunnerProvider( + wasmRuntime wasm.Runtime, + pluginKeyProvider bufplugin.PluginKeyProvider, + pluginDataProvider bufplugin.PluginDataProvider, +) *runnerProvider { return &runnerProvider{ - wasmRuntime: wasmRuntime, + wasmRuntime: wasmRuntime, + pluginKeyProvider: pluginKeyProvider, + pluginDataProvider: pluginDataProvider, } } @@ -49,7 +62,64 @@ func (r *runnerProvider) NewRunner(pluginConfig bufconfig.PluginConfig) (pluginr path[0], path[1:]..., ), nil + case bufconfig.PluginConfigTypeRemote: + // We need to load the plugin from the remote provider, then run it. + var ( + once sync.Once + compiledPlugin wasm.CompiledModule + compiledPluginErr error + ) + return rpcRunnerFunc(func(ctx context.Context, env pluginrpc.Env) error { + once.Do(func() { + compiledPlugin, compiledPluginErr = r.loadRemotePlugin(ctx, pluginConfig) + }) + if compiledPluginErr != nil { + return compiledPluginErr + } + return compiledPlugin.Run(ctx, env) + }), nil default: return nil, syserror.Newf("unknown PluginConfigType: %v", pluginConfig.Type()) } } + +// loadRemotePlugin loads the remote plugin, the plugin must be of type bufconfig.PluginConfigTypeRemote. +func (r *runnerProvider) loadRemotePlugin(ctx context.Context, pluginConfig bufconfig.PluginConfig) (wasm.CompiledModule, error) { + pluginRef := pluginConfig.Ref() + if pluginRef == nil { + return nil, syserror.New("Ref is required for remote plugins") + } + pluginKeys, err := r.pluginKeyProvider.GetPluginKeysForPluginRefs( + ctx, + []bufparse.Ref{pluginRef}, + bufplugin.DigestTypeP1, + ) + if err != nil { + return nil, err + } + if len(pluginKeys) != 1 { + return nil, syserror.Newf("expected 1 plugin key for %s", pluginRef) + } + pluginDatas, err := r.pluginDataProvider.GetPluginDatasForPluginKeys( + ctx, + pluginKeys, + ) + if err != nil { + return nil, err + } + if len(pluginDatas) != 1 { + return nil, syserror.Newf("expected 1 plugin data for %s", pluginRef) + } + pluginData := pluginDatas[0] + data, err := pluginData.Data() + if err != nil { + return nil, err + } + return r.wasmRuntime.Compile(ctx, pluginConfig.Name(), data) +} + +type rpcRunnerFunc func(ctx context.Context, env pluginrpc.Env) error + +func (f rpcRunnerFunc) Run(ctx context.Context, env pluginrpc.Env) error { + return f(ctx, env) +} diff --git a/private/bufpkg/bufconfig/buf_lock_file.go b/private/bufpkg/bufconfig/buf_lock_file.go index 3eee9d2fb6..bb0490fd1e 100644 --- a/private/bufpkg/bufconfig/buf_lock_file.go +++ b/private/bufpkg/bufconfig/buf_lock_file.go @@ -25,6 +25,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" @@ -72,6 +73,14 @@ type BufLockFile interface { // Files with FileVersionV1Beta1 or FileVersionV1 will only have ModuleKeys with Digests of DigestTypeB4, // while Files with FileVersionV2 will only have ModuleKeys with Digests of DigestTypeB5. DepModuleKeys() []bufmodule.ModuleKey + // RemotePluginKeys returns the PluginKeys representing the remote plugins as specified in the buf.lock file. + // + // All PluginKeys will have unique FullNames. + // PluginKeys are sorted by FullName. + // + // Files with FileVersionV1Beta1 or FileVersionV1 will not have PluginKeys. + // Only files with FileVersionV2 will have PluginKeys with Digests of DigestTypeP1. + RemotePluginKeys() []bufplugin.PluginKey isBufLockFile() } @@ -80,8 +89,8 @@ type BufLockFile interface { // // Note that digests are lazily-loaded; if you need to ensure that all digests are valid, run // ValidateBufLockFileDigests(). -func NewBufLockFile(fileVersion FileVersion, depModuleKeys []bufmodule.ModuleKey) (BufLockFile, error) { - return newBufLockFile(fileVersion, nil, depModuleKeys) +func NewBufLockFile(fileVersion FileVersion, depModuleKeys []bufmodule.ModuleKey, pluginKeys []bufplugin.PluginKey) (BufLockFile, error) { + return newBufLockFile(fileVersion, nil, depModuleKeys, pluginKeys) } // GetBufLockFileForPrefix gets the buf.lock file at the given bucket prefix. @@ -181,15 +190,17 @@ func BufLockFileWithDigestResolver( // *** PRIVATE *** type bufLockFile struct { - fileVersion FileVersion - objectData ObjectData - depModuleKeys []bufmodule.ModuleKey + fileVersion FileVersion + objectData ObjectData + depModuleKeys []bufmodule.ModuleKey + remotePluginKeys []bufplugin.PluginKey } func newBufLockFile( fileVersion FileVersion, objectData ObjectData, depModuleKeys []bufmodule.ModuleKey, + remotePluginKeys []bufplugin.PluginKey, ) (*bufLockFile, error) { if err := validateNoDuplicateModuleKeysByFullName(depModuleKeys); err != nil { return nil, err @@ -214,10 +225,18 @@ func newBufLockFile( return depModuleKeys[i].FullName().String() < depModuleKeys[j].FullName().String() }, ) + remotePluginKeys = slicesext.Copy(remotePluginKeys) + sort.Slice( + remotePluginKeys, + func(i int, j int) bool { + return remotePluginKeys[i].FullName().String() < remotePluginKeys[j].FullName().String() + }, + ) bufLockFile := &bufLockFile{ - fileVersion: fileVersion, - objectData: objectData, - depModuleKeys: depModuleKeys, + fileVersion: fileVersion, + objectData: objectData, + depModuleKeys: depModuleKeys, + remotePluginKeys: remotePluginKeys, } if err := validateV1AndV1Beta1DepsHaveCommits(bufLockFile); err != nil { return nil, err @@ -241,6 +260,10 @@ func (l *bufLockFile) DepModuleKeys() []bufmodule.ModuleKey { return l.depModuleKeys } +func (l *bufLockFile) RemotePluginKeys() []bufplugin.PluginKey { + return l.remotePluginKeys +} + func (*bufLockFile) isBufLockFile() {} func (*bufLockFile) isFile() {} func (*bufLockFile) isFileInfo() {} @@ -315,7 +338,7 @@ func readBufLockFile( } depModuleKeys[i] = depModuleKey } - return newBufLockFile(fileVersion, objectData, depModuleKeys) + return newBufLockFile(fileVersion, objectData, depModuleKeys, nil /* remotePluginKeys */) case FileVersionV2: var externalBufLockFile externalBufLockFileV2 if err := getUnmarshalStrict(allowJSON)(data, &externalBufLockFile); err != nil { @@ -357,7 +380,39 @@ func readBufLockFile( } depModuleKeys[i] = depModuleKey } - return newBufLockFile(fileVersion, objectData, depModuleKeys) + remotePluginKeys := make([]bufplugin.PluginKey, len(externalBufLockFile.Plugins)) + for i, plugin := range externalBufLockFile.Plugins { + plugin := plugin + if plugin.Name == "" { + return nil, errors.New("no plugin name specified") + } + pluginFullName, err := bufparse.ParseFullName(plugin.Name) + if err != nil { + return nil, fmt.Errorf("invalid plugin name: %w", err) + } + if plugin.Commit == "" { + return nil, fmt.Errorf("no commit specified for plugin %s", pluginFullName.String()) + } + if plugin.Digest == "" { + return nil, fmt.Errorf("no digest specified for plugin %s", pluginFullName.String()) + } + commitID, err := uuidutil.FromDashless(plugin.Commit) + if err != nil { + return nil, err + } + pluginKey, err := bufplugin.NewPluginKey( + pluginFullName, + commitID, + func() (bufplugin.Digest, error) { + return bufplugin.ParseDigest(plugin.Digest) + }, + ) + if err != nil { + return nil, err + } + remotePluginKeys[i] = pluginKey + } + return newBufLockFile(fileVersion, objectData, depModuleKeys, remotePluginKeys) default: // This is a system error since we've already parsed. return nil, syserror.Newf("unknown FileVersion: %v", fileVersion) @@ -400,9 +455,11 @@ func writeBufLockFile( return err case FileVersionV2: depModuleKeys := bufLockFile.DepModuleKeys() + remotePluginKeys := bufLockFile.RemotePluginKeys() externalBufLockFile := externalBufLockFileV2{ Version: fileVersion.String(), Deps: make([]externalBufLockFileDepV2, len(depModuleKeys)), + Plugins: make([]externalBufLockFileDepV2, len(remotePluginKeys)), } for i, depModuleKey := range depModuleKeys { digest, err := depModuleKey.Digest() @@ -415,6 +472,17 @@ func writeBufLockFile( Digest: digest.String(), } } + for i, remotePluginKey := range remotePluginKeys { + digest, err := remotePluginKey.Digest() + if err != nil { + return err + } + externalBufLockFile.Plugins[i] = externalBufLockFileDepV2{ + Name: remotePluginKey.FullName().String(), + Commit: uuidutil.ToDashless(remotePluginKey.CommitID()), + Digest: digest.String(), + } + } // No need to sort - depModuleKeys is already sorted by FullName data, err := encoding.MarshalYAML(&externalBufLockFile) if err != nil { @@ -524,6 +592,7 @@ type externalBufLockFileDepV1Beta1V1 struct { type externalBufLockFileV2 struct { Version string `json:"version,omitempty" yaml:"version,omitempty"` Deps []externalBufLockFileDepV2 `json:"deps,omitempty" yaml:"deps,omitempty"` + Plugins []externalBufLockFileDepV2 `json:"plugins,omitempty" yaml:"plugins,omitempty"` } // externalBufLockFileDepV2 represents a single dep within a v2 buf.lock file. diff --git a/private/bufpkg/bufconfig/plugin_config.go b/private/bufpkg/bufconfig/plugin_config.go index 388bb15b50..c41a0dd4a3 100644 --- a/private/bufpkg/bufconfig/plugin_config.go +++ b/private/bufpkg/bufconfig/plugin_config.go @@ -20,6 +20,7 @@ import ( "path/filepath" "strings" + "github.com/bufbuild/buf/private/bufpkg/bufparse" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/syserror" ) @@ -29,6 +30,8 @@ const ( PluginConfigTypeLocal PluginConfigType = iota + 1 // PluginConfigTypeLocalWasm is the local Wasm plugin config type. PluginConfigTypeLocalWasm + // PluginConfigTypeRemote is the remote plugin config type. + PluginConfigTypeRemote ) // PluginConfigType is a generate plugin configuration type. @@ -49,6 +52,10 @@ type PluginConfig interface { // // This is not empty only when the plugin is local. Path() []string + // Ref returns the plugin reference. + // + // This is only non-nil when the plugin is remote. + Ref() bufparse.Ref isPluginConfig() } @@ -90,6 +97,7 @@ type pluginConfig struct { name string options map[string]any path []string + ref bufparse.Ref } func newPluginConfigForExternalV2( @@ -106,8 +114,7 @@ func newPluginConfigForExternalV2( } options[key] = value } - // TODO: differentiate between local and remote in the future - // Use the same heuristic that we do for dir vs module in buffetch + // Plugins are specified as a path, remote reference, or Wasm file. path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Plugin) if err != nil { return nil, err @@ -115,6 +122,13 @@ func newPluginConfigForExternalV2( if len(path) == 0 { return nil, errors.New("must specify a path to the plugin") } + // Remote plugins are specified as plugin references. + if pluginRef, err := bufparse.ParseRef(path[0]); err == nil { + return newRemotePluginConfig( + pluginRef, + options, + ) + } // Wasm plugins are suffixed with .wasm. Otherwise, it's a binary. if filepath.Ext(path[0]) == ".wasm" { return newLocalWasmPluginConfig( @@ -165,6 +179,18 @@ func newLocalWasmPluginConfig( }, nil } +func newRemotePluginConfig( + pluginRef bufparse.Ref, + options map[string]any, +) (*pluginConfig, error) { + return &pluginConfig{ + pluginConfigType: PluginConfigTypeRemote, + name: pluginRef.FullName().Name(), + options: options, + ref: pluginRef, + }, nil +} + func (p *pluginConfig) Type() PluginConfigType { return p.pluginConfigType } @@ -181,6 +207,10 @@ func (p *pluginConfig) Path() []string { return p.path } +func (p *pluginConfig) Ref() bufparse.Ref { + return p.ref +} + func (p *pluginConfig) isPluginConfig() {} func newExternalV2ForPluginConfig( diff --git a/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go new file mode 100644 index 0000000000..816d5770ad --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi diff --git a/private/bufpkg/bufplugin/bufpluginapi/convert.go b/private/bufpkg/bufplugin/bufpluginapi/convert.go new file mode 100644 index 0000000000..ae2130659c --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/convert.go @@ -0,0 +1,53 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "fmt" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" +) + +var ( + v1beta1ProtoDigestTypeToDigestType = map[pluginv1beta1.DigestType]bufplugin.DigestType{ + pluginv1beta1.DigestType_DIGEST_TYPE_P1: bufplugin.DigestTypeP1, + } +) + +// V1Beta1ProtoToDigest converts the given proto Digest to a Digest. +// +// Validation is performed to ensure the DigestType is known, and the value +// is a valid digest value for the given DigestType. +func V1Beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, error) { + digestType, err := v1beta1ProtoToDigestType(protoDigest.Type) + if err != nil { + return nil, err + } + bufcasDigest, err := bufcas.NewDigest(protoDigest.Value) + if err != nil { + return nil, err + } + return bufplugin.NewDigest(digestType, bufcasDigest) +} + +func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { + digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] + if !ok { + return 0, fmt.Errorf("unknown pluginv1beta1.DigestType: %v", protoDigestType) + } + return digestType, nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/plugin_data_provider.go b/private/bufpkg/bufplugin/bufpluginapi/plugin_data_provider.go new file mode 100644 index 0000000000..8a2061cfbd --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/plugin_data_provider.go @@ -0,0 +1,186 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "context" + "fmt" + "log/slog" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/google/uuid" + "github.com/klauspost/compress/zstd" +) + +// NewPluginDataProvider returns a new PluginDataProvider for the given API client. +// +// A warning is printed to the logger if a given Plugin is deprecated. +func NewPluingDataProvider( + logger *slog.Logger, + clientProvider interface { + bufregistryapiplugin.V1Beta1DownloadServiceClientProvider + }, +) bufplugin.PluginDataProvider { + return newPluginDataProvider(logger, clientProvider) +} + +// *** PRIVATE *** + +type pluginDataProvider struct { + logger *slog.Logger + clientProvider interface { + bufregistryapiplugin.V1Beta1DownloadServiceClientProvider + } +} + +func newPluginDataProvider( + logger *slog.Logger, + clientProvider interface { + bufregistryapiplugin.V1Beta1DownloadServiceClientProvider + }, +) *pluginDataProvider { + return &pluginDataProvider{ + logger: logger, + clientProvider: clientProvider, + } +} + +func (p *pluginDataProvider) GetPluginDatasForPluginKeys( + ctx context.Context, + pluginKeys []bufplugin.PluginKey, +) ([]bufplugin.PluginData, error) { + if len(pluginKeys) == 0 { + return nil, nil + } + digestType, err := bufplugin.UniqueDigestTypeForPluginKeys(pluginKeys) + if err != nil { + return nil, err + } + if digestType != bufplugin.DigestTypeP1 { + return nil, syserror.Newf("unsupported digest type: %v", digestType) + } + if _, err := bufparse.FullNameStringToUniqueValue(pluginKeys); err != nil { + return nil, err + } + + registryToIndexedPluginKeys := slicesext.ToIndexedValuesMap( + pluginKeys, + func(pluginKey bufplugin.PluginKey) string { + return pluginKey.FullName().Registry() + }, + ) + indexedPluginDatas := make([]slicesext.Indexed[bufplugin.PluginData], 0, len(pluginKeys)) + for registry, indexedPluginKeys := range registryToIndexedPluginKeys { + indexedRegistryPluginDatas, err := p.getIndexedPluginDatasForRegistryAndIndexedPluginKeys( + ctx, + registry, + indexedPluginKeys, + ) + if err != nil { + return nil, err + } + indexedPluginDatas = append(indexedPluginDatas, indexedRegistryPluginDatas...) + } + return slicesext.IndexedToSortedValues(indexedPluginDatas), nil +} + +func (p *pluginDataProvider) getIndexedPluginDatasForRegistryAndIndexedPluginKeys( + ctx context.Context, + registry string, + indexedPluginKeys []slicesext.Indexed[bufplugin.PluginKey], +) ([]slicesext.Indexed[bufplugin.PluginData], error) { + values := slicesext.Map(indexedPluginKeys, func(indexedPluginKey slicesext.Indexed[bufplugin.PluginKey]) *pluginv1beta1.DownloadRequest_Value { + return &pluginv1beta1.DownloadRequest_Value{ + ResourceRef: &pluginv1beta1.ResourceRef{ + Value: &pluginv1beta1.ResourceRef_Id{ + Id: uuidutil.ToDashless(indexedPluginKey.Value.CommitID()), + }, + }, + } + }) + + pluginResponse, err := p.clientProvider.V1Beta1DownloadServiceClient(registry).Download( + ctx, + connect.NewRequest(&pluginv1beta1.DownloadRequest{ + Values: values, + }), + ) + if err != nil { + return nil, err + } + pluginContents := pluginResponse.Msg.Contents + if len(pluginContents) != len(indexedPluginKeys) { + return nil, syserror.New("did not get the expected number of plugin datas") + } + + commitIDToIndexedPluginKeys, err := slicesext.ToUniqueValuesMapError( + indexedPluginKeys, + func(indexedPluginKey slicesext.Indexed[bufplugin.PluginKey]) (uuid.UUID, error) { + return indexedPluginKey.Value.CommitID(), nil + }, + ) + if err != nil { + return nil, err + } + + indexedPluginDatas := make([]slicesext.Indexed[bufplugin.PluginData], 0, len(indexedPluginKeys)) + for _, pluginContent := range pluginContents { + commitID, err := uuid.Parse(pluginContent.Commit.Id) + if err != nil { + return nil, err + } + indexedPluginKey, ok := commitIDToIndexedPluginKeys[commitID] + if !ok { + return nil, syserror.Newf("did not get plugin key from store with commitID %q", commitID) + } + var getData func() ([]byte, error) + switch compressionType := pluginContent.CompressionType; compressionType { + case pluginv1beta1.CompressionType_COMPRESSION_TYPE_NONE: + getData = func() ([]byte, error) { + return pluginContent.Content, nil + } + case pluginv1beta1.CompressionType_COMPRESSION_TYPE_ZSTD: + getData = func() ([]byte, error) { + zstdDecoder, err := zstd.NewReader(nil) + if err != nil { + return nil, err + } + defer zstdDecoder.Close() // Does not return an error. + return zstdDecoder.DecodeAll(pluginContent.Content, nil) + } + default: + return nil, fmt.Errorf("unknown CompressionType: %v", compressionType) + } + pluginData, err := bufplugin.NewPluginData(ctx, indexedPluginKey.Value, getData) + if err != nil { + return nil, err + } + indexedPluginDatas = append( + indexedPluginDatas, + slicesext.Indexed[bufplugin.PluginData]{ + Value: pluginData, + Index: indexedPluginKey.Index, + }, + ) + } + return indexedPluginDatas, nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/plugin_key_provider.go b/private/bufpkg/bufplugin/bufpluginapi/plugin_key_provider.go new file mode 100644 index 0000000000..8af99aaad7 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/plugin_key_provider.go @@ -0,0 +1,168 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "context" + "log/slog" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" +) + +// NewPluginKeyProvider returns a new PluginKeyProvider for the given API clients. +func NewPluginKeyProvider( + logger *slog.Logger, + clientProvider interface { + bufregistryapiplugin.V1Beta1CommitServiceClientProvider + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + }, +) bufplugin.PluginKeyProvider { + return newPluginKeyProvider(logger, clientProvider) +} + +// *** PRIVATE *** + +type pluginKeyProvider struct { + logger *slog.Logger + clientProvider interface { + bufregistryapiplugin.V1Beta1CommitServiceClientProvider + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + } +} + +func newPluginKeyProvider( + logger *slog.Logger, + clientProvider interface { + bufregistryapiplugin.V1Beta1CommitServiceClientProvider + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + }, +) *pluginKeyProvider { + return &pluginKeyProvider{ + logger: logger, + clientProvider: clientProvider, + } +} + +func (p *pluginKeyProvider) GetPluginKeysForPluginRefs( + ctx context.Context, + pluginRefs []bufparse.Ref, + digestType bufplugin.DigestType, +) ([]bufplugin.PluginKey, error) { + if len(pluginRefs) == 0 { + return nil, nil + } + // Check unique pluginRefs. + if _, err := slicesext.ToUniqueValuesMapError( + pluginRefs, + func(pluginRef bufparse.Ref) (string, error) { + return pluginRef.String(), nil + }, + ); err != nil { + return nil, err + } + registryToIndexedPluginRefs := slicesext.ToIndexedValuesMap( + pluginRefs, + func(pluginRef bufparse.Ref) string { + return pluginRef.FullName().Registry() + }, + ) + indexedPluginKeys := make([]slicesext.Indexed[bufplugin.PluginKey], 0, len(pluginRefs)) + for registry, indexedPluginRefs := range registryToIndexedPluginRefs { + indexedRegistryPluginKeys, err := p.getIndexedPluginKeysForRegistryAndIndexedPluginRefs( + ctx, + registry, + indexedPluginRefs, + digestType, + ) + if err != nil { + return nil, err + } + indexedPluginKeys = append(indexedPluginKeys, indexedRegistryPluginKeys...) + } + return slicesext.IndexedToSortedValues(indexedPluginKeys), nil +} + +func (p *pluginKeyProvider) getIndexedPluginKeysForRegistryAndIndexedPluginRefs( + ctx context.Context, + registry string, + indexedPluginRefs []slicesext.Indexed[bufparse.Ref], + digestType bufplugin.DigestType, +) ([]slicesext.Indexed[bufplugin.PluginKey], error) { + resourceRefs := slicesext.Map(indexedPluginRefs, func(indexedPluginRef slicesext.Indexed[bufparse.Ref]) *pluginv1beta1.ResourceRef { + resourceRefName := &pluginv1beta1.ResourceRef_Name{ + Owner: indexedPluginRef.Value.FullName().Owner(), + Plugin: indexedPluginRef.Value.FullName().Name(), + } + if ref := indexedPluginRef.Value.Ref(); ref != "" { + resourceRefName.Child = &pluginv1beta1.ResourceRef_Name_Ref{ + Ref: ref, + } + } + return &pluginv1beta1.ResourceRef{ + Value: &pluginv1beta1.ResourceRef_Name_{ + Name: resourceRefName, + }, + } + }) + + pluginResponse, err := p.clientProvider.V1Beta1CommitServiceClient(registry).GetCommits( + ctx, + connect.NewRequest(&pluginv1beta1.GetCommitsRequest{ + ResourceRefs: resourceRefs, + }), + ) + if err != nil { + return nil, err + } + commits := pluginResponse.Msg.Commits + if len(commits) != len(indexedPluginRefs) { + return nil, syserror.New("did not get the expected number of plugin datas") + } + + indexedPluginKeys := make([]slicesext.Indexed[bufplugin.PluginKey], len(commits)) + for i, commit := range commits { + commitID, err := uuidutil.FromDashless(commit.Id) + if err != nil { + return nil, err + } + digest, err := V1Beta1ProtoToDigest(commit.Digest) + if err != nil { + return nil, err + } + pluginKey, err := bufplugin.NewPluginKey( + // Note we don't have to resolve owner_name and plugin_name since we already have them. + indexedPluginRefs[i].Value.FullName(), + commitID, + func() (bufplugin.Digest, error) { + return digest, nil + }, + ) + if err != nil { + return nil, err + } + indexedPluginKeys[i] = slicesext.Indexed[bufplugin.PluginKey]{ + Value: pluginKey, + Index: indexedPluginRefs[i].Index, + } + } + return indexedPluginKeys, nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go b/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go new file mode 100644 index 0000000000..cba34bb462 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufpluginapi + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/bufplugincache/bufplugincache.go b/private/bufpkg/bufplugin/bufplugincache/bufplugincache.go new file mode 100644 index 0000000000..7c0fa4c721 --- /dev/null +++ b/private/bufpkg/bufplugin/bufplugincache/bufplugincache.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugincache diff --git a/private/bufpkg/bufplugin/bufplugincache/plugin_data_provider.go b/private/bufpkg/bufplugin/bufplugincache/plugin_data_provider.go new file mode 100644 index 0000000000..e252a2dca2 --- /dev/null +++ b/private/bufpkg/bufplugin/bufplugincache/plugin_data_provider.go @@ -0,0 +1,111 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugincache + +import ( + "context" + "log/slog" + "sync/atomic" + + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginstore" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/google/uuid" +) + +// NewPluginDataProvider returns a new PluginDataProvider that caches the results of the delegate. +// +// The PluginDataStore is used as a cache. +func NewPluginDataProvider( + logger *slog.Logger, + delegate bufplugin.PluginDataProvider, + store bufpluginstore.PluginDataStore, +) bufplugin.PluginDataProvider { + return newPluginDataProvider(logger, delegate, store) +} + +/// *** PRIVATE *** + +type pluginDataProvider struct { + logger *slog.Logger + delegate bufplugin.PluginDataProvider + store bufpluginstore.PluginDataStore + + keysRetrieved atomic.Int64 + keysHit atomic.Int64 +} + +func newPluginDataProvider( + logger *slog.Logger, + delegate bufplugin.PluginDataProvider, + store bufpluginstore.PluginDataStore, +) *pluginDataProvider { + return &pluginDataProvider{ + logger: logger, + delegate: delegate, + store: store, + } +} + +func (p *pluginDataProvider) GetPluginDatasForPluginKeys( + ctx context.Context, + pluginKeys []bufplugin.PluginKey, +) ([]bufplugin.PluginData, error) { + foundValues, notFoundKeys, err := p.store.GetPluginDatasForPluginKeys(ctx, pluginKeys) + if err != nil { + return nil, err + } + + delegateValues, err := p.delegate.GetPluginDatasForPluginKeys(ctx, notFoundKeys) + if err != nil { + return nil, err + } + if err := p.store.PutPluginDatas(ctx, delegateValues); err != nil { + return nil, err + } + + p.keysRetrieved.Add(int64(len(pluginKeys))) + p.keysHit.Add(int64(len(foundValues))) + + commitIDToIndexedKey, err := slicesext.ToUniqueIndexedValuesMap( + pluginKeys, + func(pluginKey bufplugin.PluginKey) uuid.UUID { + return pluginKey.CommitID() + }, + ) + if err != nil { + return nil, err + } + indexedValues, err := slicesext.MapError( + append(foundValues, delegateValues...), + func(value bufplugin.PluginData) (slicesext.Indexed[bufplugin.PluginData], error) { + commitID := value.PluginKey().CommitID() + indexedKey, ok := commitIDToIndexedKey[commitID] + if !ok { + return slicesext.Indexed[bufplugin.PluginData]{}, syserror.Newf("did not get value from store with commitID %q", uuidutil.ToDashless(commitID)) + } + return slicesext.Indexed[bufplugin.PluginData]{ + Value: value, + Index: indexedKey.Index, + }, nil + }, + ) + if err != nil { + return nil, err + } + return slicesext.IndexedToSortedValues(indexedValues), nil +} diff --git a/private/bufpkg/bufplugin/bufplugincache/usage.gen.go b/private/bufpkg/bufplugin/bufplugincache/usage.gen.go new file mode 100644 index 0000000000..4f54051370 --- /dev/null +++ b/private/bufpkg/bufplugin/bufplugincache/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufplugincache + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/bufpluginstore/bufpluginstore.go b/private/bufpkg/bufplugin/bufpluginstore/bufpluginstore.go new file mode 100644 index 0000000000..a8755d2de3 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginstore/bufpluginstore.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginstore diff --git a/private/bufpkg/bufplugin/bufpluginstore/module_data_store.go b/private/bufpkg/bufplugin/bufpluginstore/module_data_store.go new file mode 100644 index 0000000000..08cd8e1515 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginstore/module_data_store.go @@ -0,0 +1,165 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginstore + +import ( + "context" + "errors" + "io/fs" + "log/slog" + + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/normalpath" + "github.com/bufbuild/buf/private/pkg/storage" + "github.com/bufbuild/buf/private/pkg/uuidutil" +) + +// PluginStore reads and writes PluginsDatas. +type PluginDataStore interface { + // GetPluginDatasForPluginKey gets the PluginDatas from the store for the PluginKeys. + // + // Returns the found PluginDatas, and the input PluginKeys that were not found, each + // ordered by the order of the input PluginKeys. + GetPluginDatasForPluginKeys(context.Context, []bufplugin.PluginKey) ( + foundPluginDatas []bufplugin.PluginData, + notFoundPluginKeys []bufplugin.PluginKey, + err error, + ) + // Put puts the PluginDatas to the store. + PutPluginDatas(ctx context.Context, moduleDatas []bufplugin.PluginData) error +} + +// NewPluginDataStore returns a new PluginDataStore for the given bucket. +// +// It is assumed that the PluginDataStore has complete control of the bucket. +// +// This is typically used to interact with a cache directory. +func NewPluginDataStore( + logger *slog.Logger, + bucket storage.ReadWriteBucket, +) PluginDataStore { + return newPluginDataStore(logger, bucket) +} + +/// *** PRIVATE *** + +type pluginDataStore struct { + logger *slog.Logger + bucket storage.ReadWriteBucket +} + +func newPluginDataStore( + logger *slog.Logger, + bucket storage.ReadWriteBucket, +) *pluginDataStore { + return &pluginDataStore{ + logger: logger, + bucket: bucket, + } +} + +func (p *pluginDataStore) GetPluginDatasForPluginKeys( + ctx context.Context, + pluginKeys []bufplugin.PluginKey, +) ([]bufplugin.PluginData, []bufplugin.PluginKey, error) { + var foundPluginDatas []bufplugin.PluginData + var notFoundPluginKeys []bufplugin.PluginKey + for _, pluginKey := range pluginKeys { + pluginData, err := p.getPluginDataForPluginKey(ctx, pluginKey) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return nil, nil, err + } + notFoundPluginKeys = append(notFoundPluginKeys, pluginKey) + } else { + foundPluginDatas = append(foundPluginDatas, pluginData) + } + } + return foundPluginDatas, notFoundPluginKeys, nil +} + +func (p *pluginDataStore) PutPluginDatas( + ctx context.Context, + pluginDatas []bufplugin.PluginData, +) error { + for _, pluginData := range pluginDatas { + if err := p.putPluginData(ctx, pluginData); err != nil { + return err + } + } + return nil +} + +// getPluginDataForPluginKey reads the plugin data for the plugin key from the cache. +func (p *pluginDataStore) getPluginDataForPluginKey( + ctx context.Context, + pluginKey bufplugin.PluginKey, +) (bufplugin.PluginData, error) { + pluginDataStorePath, err := getPluginDataStorePath(pluginKey) + if err != nil { + return nil, err + } + if exists, err := storage.Exists(ctx, p.bucket, pluginDataStorePath); err != nil { + return nil, err + } else if !exists { + return nil, fs.ErrNotExist + } + return bufplugin.NewPluginData( + ctx, + pluginKey, + func() ([]byte, error) { + // Data is stored uncompressed. + return storage.ReadPath(ctx, p.bucket, pluginDataStorePath) + }, + ) +} + +// putPluginData puts the plugin data into the plugin cache. +func (p *pluginDataStore) putPluginData( + ctx context.Context, + pluginData bufplugin.PluginData, +) error { + pluginKey := pluginData.PluginKey() + pluginDataStorePath, err := getPluginDataStorePath(pluginKey) + if err != nil { + return err + } + data, err := pluginData.Data() + if err != nil { + return err + } + // Data is stored uncompressed. + return storage.PutPath(ctx, p.bucket, pluginDataStorePath, data) +} + +// getPluginDataStorePath returns the path for the plugin data store for the plugin key. +// +// This is "digestType/registry/owner/name/dashlessCommitID", e.g. the plugin +// "buf.build/acme/check-plugin" with commit "12345-abcde" and digest type "p1" +// will return "p1/buf.build/acme/check-plugin/12345abcde.wasm". +func getPluginDataStorePath(pluginKey bufplugin.PluginKey) (string, error) { + digest, err := pluginKey.Digest() + if err != nil { + return "", err + } + fullName := pluginKey.FullName() + return normalpath.Join( + digest.Type().String(), + fullName.Registry(), + fullName.Owner(), + fullName.Name(), + uuidutil.ToDashless(pluginKey.CommitID())+".wasm", + ), nil +} diff --git a/private/bufpkg/bufplugin/bufpluginstore/usage.gen.go b/private/bufpkg/bufplugin/bufpluginstore/usage.gen.go new file mode 100644 index 0000000000..3b402991cf --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginstore/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufpluginstore + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/plugin_key.go b/private/bufpkg/bufplugin/plugin_key.go index d78f140584..75970dcc8e 100644 --- a/private/bufpkg/bufplugin/plugin_key.go +++ b/private/bufpkg/bufplugin/plugin_key.go @@ -17,9 +17,12 @@ package bufplugin import ( "errors" "fmt" + "strings" "sync" "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" "github.com/bufbuild/buf/private/pkg/uuidutil" "github.com/google/uuid" ) @@ -71,6 +74,32 @@ func NewPluginKey( ) } +// UniqueDigestTypeForPluginKeys returns the unique DigestType for the given PluginKeys. +// +// If the PluginKeys have different DigestTypes, an error is returned. +// If the PluginKeys slice is empty, an error is returned. +func UniqueDigestTypeForPluginKeys(pluginKeys []PluginKey) (DigestType, error) { + if len(pluginKeys) == 0 { + return 0, syserror.New("empty pluginKeys passed to UniqueDigestTypeForPluginKeys") + } + digests, err := slicesext.MapError(pluginKeys, PluginKey.Digest) + if err != nil { + return 0, err + } + digestType := digests[0].Type() + for _, digest := range digests[1:] { + if digestType != digest.Type() { + return 0, fmt.Errorf( + "different digest types detected where the same digest type must be used: %v, %v\n%s", + digestType, + digest.Type(), + strings.Join(slicesext.Map(pluginKeys, PluginKey.String), "\n"), + ) + } + } + return digestType, nil +} + // ** PRIVATE ** type pluginKey struct {