diff --git a/CHANGELOG.md b/CHANGELOG.md index fcd8101d81..899ace6c4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Add `buf registry plugin {create,delete,info,update}` commands to manage BSR plugins. - Breaking analysis support for `buf beta lsp`. +- Add `buf plugin push` command to push a plugin to the Buf Schema Registry. + Only WebAssembly check plugins are supported at this time. ## [v1.47.2] - 2024-11-14 diff --git a/private/buf/bufcli/uploader.go b/private/buf/bufcli/uploader.go index df2d047a5e..4061c8114c 100644 --- a/private/buf/bufcli/uploader.go +++ b/private/buf/bufcli/uploader.go @@ -17,20 +17,32 @@ package bufcli import ( "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" "github.com/bufbuild/buf/private/pkg/app/appext" ) -// NewUploader returns a new Uploader. -func NewUploader(container appext.Container) (bufmodule.Uploader, error) { +// NewModuleUploader returns a new Uploader for ModuleSets. +func NewModuleUploader(container appext.Container) (bufmodule.Uploader, error) { clientConfig, err := NewConnectClientConfig(container) if err != nil { return nil, err } - return newUploader(container, bufregistryapimodule.NewClientProvider(clientConfig)), nil + return newModuleUploader(container, bufregistryapimodule.NewClientProvider(clientConfig)), nil } -func newUploader( +// NewPluginUploader returns a new Uploader for Plugins. +func NewPluginUploader(container appext.Container) (bufplugin.Uploader, error) { + clientConfig, err := NewConnectClientConfig(container) + if err != nil { + return nil, err + } + return newPluginUploader(container, bufregistryapiplugin.NewClientProvider(clientConfig)), nil +} + +func newModuleUploader( container appext.Container, clientProvider bufregistryapimodule.ClientProvider, ) bufmodule.Uploader { @@ -41,3 +53,13 @@ func newUploader( bufmoduleapi.UploaderWithPublicRegistry(container.Env(publicRegistryEnvKey)), ) } + +func newPluginUploader( + container appext.Container, + clientProvider bufregistryapiplugin.ClientProvider, +) bufplugin.Uploader { + return bufpluginapi.NewUploader( + container.Logger(), + clientProvider, + ) +} diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index 776b611c06..501769239a 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -62,6 +62,7 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlsbreakingrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlslintrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modopen" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/plugin/pluginpush" "github.com/bufbuild/buf/private/buf/cmd/buf/command/push" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/module/modulecommit/modulecommitaddlabel" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/module/modulecommit/modulecommitinfo" @@ -175,6 +176,13 @@ func NewRootCommand(name string) *appcmd.Command { modlsbreakingrules.NewCommand("ls-breaking-rules", builder), }, }, + { + Use: "plugin", + Short: "Work with plugins", + SubCommands: []*appcmd.Command{ + pluginpush.NewCommand("push", builder), + }, + }, { Use: "registry", Short: "Manage assets on the Buf Schema Registry", diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go new file mode 100644 index 0000000000..5a68efbe48 --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go @@ -0,0 +1,263 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginpush + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/stringutil" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/spf13/pflag" +) + +const ( + labelFlagName = "label" + binaryFlagName = "binary" + createFlagName = "create" + createVisibilityFlagName = "create-visibility" + createTypeFlagName = "create-type" + sourceControlURLFlagName = "source-control-url" +) + +// NewCommand returns a new Command. +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Push a plugin to a registry", + Long: `The first argument is the plugin full name in the format .`, + Args: appcmd.MaximumNArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + Labels []string + Binary string + Create bool + CreateVisibility string + CreateType string + SourceControlURL string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + bufcli.BindCreateVisibility(flagSet, &f.CreateVisibility, createVisibilityFlagName, createFlagName) + flagSet.StringSliceVar( + &f.Labels, + labelFlagName, + nil, + "Associate the label with the plugins pushed. Can be used multiple times.", + ) + flagSet.StringVar( + &f.Binary, + binaryFlagName, + "", + "The path to the Wasm binary file to push.", + ) + flagSet.BoolVar( + &f.Create, + createFlagName, + false, + fmt.Sprintf( + "Create the plugin if it does not exist. Defaults to creating a private repository if --%s is not set. Must be used with --%s.", + createVisibilityFlagName, + createTypeFlagName, + ), + ) + flagSet.StringVar( + &f.CreateType, + createTypeFlagName, + "", + fmt.Sprintf( + "The plugin's type setting, if created. Can only be set with --%s. Must be one of %s", + createTypeFlagName, + stringutil.SliceToString(bufplugin.AllPluginTypeStrings), + ), + ) + flagSet.StringVar( + &f.SourceControlURL, + sourceControlURLFlagName, + "", + "The URL for viewing the source code of the pushed plugins (e.g. the specific commit in source control).", + ) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) (retErr error) { + if err := validateFlags(flags); err != nil { + return err + } + // We parse the plugin full name from the user-provided argument. + pluginFullName, err := bufparse.ParseFullName(container.Arg(0)) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + commit, err := upload(ctx, container, flags, pluginFullName) + if err != nil { + return err + } + // Only one commit is returned. + if _, err := fmt.Fprintf(container.Stdout(), "%s\n", commit.PluginKey().String()); err != nil { + return syserror.Wrap(err) + } + return nil +} + +func upload( + ctx context.Context, + container appext.Container, + flags *flags, + pluginFullName bufparse.FullName, +) (_ bufplugin.Commit, retErr error) { + var plugin bufplugin.Plugin + switch { + case flags.Binary != "": + var err error + plugin, err = bufplugin.NewLocalWasmPlugin( + pluginFullName, + func() ([]byte, error) { + wasmBinary, err := os.ReadFile(flags.Binary) + if err != nil { + return nil, fmt.Errorf("could not read Wasm binary %q: %w", flags.Binary, err) + } + return wasmBinary, nil + }, + ) + if err != nil { + return nil, err + } + default: + // This should never happen because the flags are validated. + return nil, syserror.Newf("--%s must be set", binaryFlagName) + } + uploader, err := bufcli.NewPluginUploader(container) + if err != nil { + return nil, err + } + var options []bufplugin.UploadOption + if flags.Create { + createPluginVisibility, err := bufplugin.ParsePluginVisibility(flags.CreateVisibility) + if err != nil { + return nil, err + } + createPluginType, err := bufplugin.ParsePluginType(flags.CreateType) + if err != nil { + return nil, err + } + options = append(options, bufplugin.UploadWithCreateIfNotExist( + createPluginVisibility, + createPluginType, + )) + } + commits, err := uploader.Upload(ctx, []bufplugin.Plugin{plugin}, options...) + if err != nil { + return nil, err + } + if len(commits) != 1 { + return nil, syserror.Newf("unexpected number of commits returned from server: %d", len(commits)) + } + return commits[0], nil +} + +func validateFlags(flags *flags) error { + if err := validateLabelFlags(flags); err != nil { + return err + } + if err := validateTypeFlags(flags); err != nil { + return err + } + if err := validateCreateFlags(flags); err != nil { + return err + } + return nil +} + +func validateLabelFlags(flags *flags) error { + return validateLabelFlagValues(flags) +} + +func validateTypeFlags(flags *flags) error { + var typeFlags []string + if flags.Binary != "" { + typeFlags = append(typeFlags, binaryFlagName) + } + if len(typeFlags) > 1 { + usedFlagsErrStr := strings.Join( + slicesext.Map( + typeFlags, + func(flag string) string { return fmt.Sprintf("--%s", flag) }, + ), + ", ", + ) + return appcmd.NewInvalidArgumentErrorf("These flags cannot be used in combination with one another: %s", usedFlagsErrStr) + } + if len(typeFlags) == 0 { + return appcmd.NewInvalidArgumentErrorf("--%s must be set", binaryFlagName) + } + return nil +} + +func validateLabelFlagValues(flags *flags) error { + for _, label := range flags.Labels { + if label == "" { + return appcmd.NewInvalidArgumentErrorf("--%s requires a non-empty string", labelFlagName) + } + } + return nil +} + +func validateCreateFlags(flags *flags) error { + if flags.Create { + if flags.CreateVisibility == "" { + return appcmd.NewInvalidArgumentErrorf("--%s must be set if --%s is set", createVisibilityFlagName, createFlagName) + } + if _, err := bufplugin.ParsePluginVisibility(flags.CreateVisibility); err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + } + if flags.Create { + if flags.CreateType == "" { + return appcmd.NewInvalidArgumentErrorf("--%s must be set if --%s is set", createTypeFlagName, createFlagName) + } + if _, err := bufplugin.ParsePluginType(flags.CreateType); err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + } + return nil +} diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go b/private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go new file mode 100644 index 0000000000..3184fae49e --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package pluginpush + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/buf/command/push/push.go b/private/buf/cmd/buf/command/push/push.go index 213db7689e..53e3545a86 100644 --- a/private/buf/cmd/buf/command/push/push.go +++ b/private/buf/cmd/buf/command/push/push.go @@ -126,7 +126,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { createFlagName, false, fmt.Sprintf( - "Create the repository if it does not exist. Defaults to creating a private repository if --%s is not set.", + "Create the module if it does not exist. Defaults to creating a private module if --%s is not set.", createVisibilityFlagName, ), ) @@ -134,7 +134,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { &f.CreateDefaultLabel, createDefaultLabelFlagName, "", - `The repository's default label setting, if created. If this is not set, then the repository will be created with the default label "main".`, + `The module's default label setting, if created. If this is not set, then the module will be created with the default label "main".`, ) flagSet.StringVar( &f.SourceControlURL, @@ -218,7 +218,7 @@ func run( return err } - uploader, err := bufcli.NewUploader(container) + uploader, err := bufcli.NewModuleUploader(container) if err != nil { return err } diff --git a/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go new file mode 100644 index 0000000000..816d5770ad --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi diff --git a/private/bufpkg/bufplugin/bufpluginapi/convert.go b/private/bufpkg/bufplugin/bufpluginapi/convert.go new file mode 100644 index 0000000000..8b15660e68 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/convert.go @@ -0,0 +1,75 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "fmt" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" +) + +var ( + v1beta1ProtoDigestTypeToDigestType = map[pluginv1beta1.DigestType]bufplugin.DigestType{ + pluginv1beta1.DigestType_DIGEST_TYPE_P1: bufplugin.DigestTypeP1, + } +) + +// V1Beta1ProtoToDigest converts the given proto Digest to a Digest. +// +// Validation is performed to ensure the DigestType is known, and the value +// is a valid digest value for the given DigestType. +func V1Beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, error) { + digestType, err := v1beta1ProtoToDigestType(protoDigest.Type) + if err != nil { + return nil, err + } + bufcasDigest, err := bufcas.NewDigest(protoDigest.Value) + if err != nil { + return nil, err + } + return bufplugin.NewDigest(digestType, bufcasDigest) +} + +// *** PRIVATE *** + +func pluginVisibilityToV1Beta1Proto(pluginVisibility bufplugin.PluginVisibility) (pluginv1beta1.PluginVisibility, error) { + switch pluginVisibility { + case bufplugin.PluginVisibilityPublic: + return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PUBLIC, nil + case bufplugin.PluginVisibilityPrivate: + return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PRIVATE, nil + default: + return 0, fmt.Errorf("unknown PluginVisibility: %v", pluginVisibility) + } +} + +func pluginTypeToV1Beta1Proto(pluginType bufplugin.PluginType) (pluginv1beta1.PluginType, error) { + switch pluginType { + case bufplugin.PluginTypeCheck: + return pluginv1beta1.PluginType_PLUGIN_TYPE_CHECK, nil + default: + return 0, fmt.Errorf("unknown PluginType: %v", pluginType) + } +} + +func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { + digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] + if !ok { + return 0, fmt.Errorf("unknown pluginv1beta1.DigestType: %v", protoDigestType) + } + return digestType, nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/uploader.go b/private/bufpkg/bufplugin/bufpluginapi/uploader.go new file mode 100644 index 0000000000..f0900152e2 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/uploader.go @@ -0,0 +1,315 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "context" + "fmt" + "log/slog" + "time" + + ownerv1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/owner/v1" + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/klauspost/compress/zstd" +) + +// NewUploader returns a new Uploader for the given API client. +func NewUploader( + logger *slog.Logger, + pluginClientProvider interface { + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + }, + options ...UploaderOption, +) bufplugin.Uploader { + return newUploader(logger, pluginClientProvider, options...) +} + +// UploaderOption is an option for a new Uploader. +type UploaderOption func(*uploader) + +// *** PRIVATE *** + +type uploader struct { + logger *slog.Logger + pluginClientProvider interface { + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + } +} + +func newUploader( + logger *slog.Logger, + pluginClientProvider interface { + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + }, + options ...UploaderOption, +) *uploader { + uploader := &uploader{ + logger: logger, + pluginClientProvider: pluginClientProvider, + } + for _, option := range options { + option(uploader) + } + return uploader +} + +func (u *uploader) Upload( + ctx context.Context, + plugins []bufplugin.Plugin, + options ...bufplugin.UploadOption, +) ([]bufplugin.Commit, error) { + uploadOptions, err := bufplugin.NewUploadOptions(options) + if err != nil { + return nil, err + } + registryToIndexedPluginKeys := slicesext.ToIndexedValuesMap( + plugins, + func(plugin bufplugin.Plugin) string { + return plugin.FullName().Registry() + }, + ) + indexedCommits := make([]slicesext.Indexed[bufplugin.Commit], 0, len(plugins)) + for registry, indexedPluginKeys := range registryToIndexedPluginKeys { + indexedRegistryPluginDatas, err := u.uploadIndexedPluginsForRegistry( + ctx, + registry, + indexedPluginKeys, + uploadOptions, + ) + if err != nil { + return nil, err + } + indexedCommits = append(indexedCommits, indexedRegistryPluginDatas...) + } + return slicesext.IndexedToSortedValues(indexedCommits), nil +} + +func (u *uploader) uploadIndexedPluginsForRegistry( + ctx context.Context, + registry string, + indexedPlugins []slicesext.Indexed[bufplugin.Plugin], + uploadOptions bufplugin.UploadOptions, +) ([]slicesext.Indexed[bufplugin.Commit], error) { + if uploadOptions.CreateIfNotExist() { + // We must attempt to create each Plugin one at a time, since CreatePlugins will return + // an `AlreadyExists` if any of the Plugins we are attempting to create already exists, + // and no new Plugins will be created. + for _, indexedPlugin := range indexedPlugins { + plugin := indexedPlugin.Value + if _, err := u.createPluginIfNotExist( + ctx, + registry, + plugin, + uploadOptions.CreatePluginVisibility(), + uploadOptions.CreatePluginType(), + ); err != nil { + return nil, err + } + } + } + contents, err := slicesext.MapError(indexedPlugins, func(indexedPlugin slicesext.Indexed[bufplugin.Plugin]) (*pluginv1beta1.UploadRequest_Content, error) { + plugin := indexedPlugin.Value + if !plugin.IsLocal() { + return nil, syserror.New("expected local Plugin in uploadIndexedPluginsForRegistry") + } + if plugin.FullName() == nil { + return nil, syserror.Newf("expected Plugin name for local Plugin: %s", plugin.Description()) + } + data, err := plugin.Data() + if err != nil { + return nil, err + } + compressedWasmBinary, err := zstdCompress(data) + if err != nil { + return nil, fmt.Errorf("could not compress Plugin data %q: %w", plugin.OpaqueID(), err) + } + return &pluginv1beta1.UploadRequest_Content{ + PluginRef: &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: plugin.FullName().Owner(), + Plugin: plugin.FullName().Name(), + }, + }, + }, + CompressionType: pluginv1beta1.CompressionType_COMPRESSION_TYPE_ZSTD, + Content: compressedWasmBinary, + ScopedLabelRefs: slicesext.Map(uploadOptions.Labels(), func(label string) *pluginv1beta1.ScopedLabelRef { + return &pluginv1beta1.ScopedLabelRef{ + Value: &pluginv1beta1.ScopedLabelRef_Name{ + Name: label, + }, + } + }), + SourceControlUrl: uploadOptions.SourceControlURL(), + }, nil + }) + if err != nil { + return nil, err + } + + uploadResponse, err := u.pluginClientProvider.V1Beta1UploadServiceClient(registry).Upload( + ctx, + connect.NewRequest(&pluginv1beta1.UploadRequest{ + Contents: contents, + })) + if err != nil { + return nil, err + } + pluginCommits := uploadResponse.Msg.Commits + if len(pluginCommits) != len(indexedPlugins) { + return nil, syserror.Newf("expected %d Commits, found %d", len(indexedPlugins), len(pluginCommits)) + } + + indexedCommits := make([]slicesext.Indexed[bufplugin.Commit], 0, len(indexedPlugins)) + for i, pluginCommit := range pluginCommits { + pluginFullName := indexedPlugins[i].Value.FullName() + commitID, err := uuidutil.FromDashless(pluginCommit.Id) + if err != nil { + return nil, err + } + pluginKey, err := bufplugin.NewPluginKey( + pluginFullName, + commitID, + func() (bufplugin.Digest, error) { + return V1Beta1ProtoToDigest(pluginCommit.Digest) + }, + ) + if err != nil { + return nil, err + } + commit := bufplugin.NewCommit( + pluginKey, + func() (time.Time, error) { + return pluginCommit.CreateTime.AsTime(), nil + }, + ) + indexedCommits = append( + indexedCommits, + slicesext.Indexed[bufplugin.Commit]{ + Value: commit, + Index: i, + }, + ) + } + return indexedCommits, nil +} + +func (u *uploader) createPluginIfNotExist( + ctx context.Context, + primaryRegistry string, + plugin bufplugin.Plugin, + createPluginVisibility bufplugin.PluginVisibility, + createPluginType bufplugin.PluginType, +) (*pluginv1beta1.Plugin, error) { + v1Beta1ProtoCreatePluginVisibility, err := pluginVisibilityToV1Beta1Proto(createPluginVisibility) + if err != nil { + return nil, err + } + v1Beta1ProtoCreatePluginType, err := pluginTypeToV1Beta1Proto(createPluginType) + if err != nil { + return nil, err + } + response, err := u.pluginClientProvider.V1Beta1PluginServiceClient(primaryRegistry).CreatePlugins( + ctx, + connect.NewRequest( + &pluginv1beta1.CreatePluginsRequest{ + Values: []*pluginv1beta1.CreatePluginsRequest_Value{ + { + OwnerRef: &ownerv1.OwnerRef{ + Value: &ownerv1.OwnerRef_Name{ + Name: plugin.FullName().Owner(), + }, + }, + Name: plugin.FullName().Name(), + Visibility: v1Beta1ProtoCreatePluginVisibility, + Type: v1Beta1ProtoCreatePluginType, + }, + }, + }, + ), + ) + if err != nil { + if connect.CodeOf(err) == connect.CodeAlreadyExists { + // If a plugin already existed, then we check validate its contents. + plugins, err := u.validatePluginsExist(ctx, primaryRegistry, []bufplugin.Plugin{plugin}) + if err != nil { + return nil, err + } + if len(plugins) != 1 { + return nil, syserror.Newf("expected 1 Plugin, found %d", len(plugins)) + } + return plugins[0], nil + } + return nil, err + } + if len(response.Msg.Plugins) != 1 { + return nil, syserror.Newf("expected 1 Plugin, found %d", len(response.Msg.Plugins)) + } + // Otherwise we return the plugin we created. + return response.Msg.Plugins[0], nil +} + +func (u *uploader) validatePluginsExist( + ctx context.Context, + primaryRegistry string, + plugins []bufplugin.Plugin, +) ([]*pluginv1beta1.Plugin, error) { + response, err := u.pluginClientProvider.V1Beta1PluginServiceClient(primaryRegistry).GetPlugins( + ctx, + connect.NewRequest( + &pluginv1beta1.GetPluginsRequest{ + PluginRefs: slicesext.Map( + plugins, + func(plugin bufplugin.Plugin) *pluginv1beta1.PluginRef { + return &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: plugin.FullName().Owner(), + Plugin: plugin.FullName().Name(), + }, + }, + } + }, + ), + }, + ), + ) + if err != nil { + return nil, err + } + return response.Msg.Plugins, nil +} + +func zstdCompress(data []byte) ([]byte, error) { + encoder, err := zstd.NewWriter(nil) + if err != nil { + return nil, err + } + defer encoder.Close() + return encoder.EncodeAll(data, nil), nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go b/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go new file mode 100644 index 0000000000..cba34bb462 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufpluginapi + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/plugin.go b/private/bufpkg/bufplugin/plugin.go new file mode 100644 index 0000000000..438f52a60f --- /dev/null +++ b/private/bufpkg/bufplugin/plugin.go @@ -0,0 +1,244 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "fmt" + "strings" + "sync" + + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/google/uuid" +) + +// Plugin presents a BSR plugin. +type Plugin interface { + // OpaqueID returns an unstructured ID that can uniquely identify a Plugin + // relative to the Workspace. + // + // An OpaqueID's structure should not be relied upon, and is not a + // globally-unique identifier. It's uniqueness property only applies to + // the lifetime of the Plugin, and only within Plugin commonly built + // from the Workspace root. + // + // If two Plugins have the same FullName, they will have the same OpaqueID. + OpaqueID() string + // Path returns the path, including arguments, to invoke the binary plugin. + // + // This is not empty only when the plugin is local. + Path() []string + // FullName returns the FullName of the Plugin. + // + // This is nil + FullName() bufparse.FullName + // CommitID returns the BSR ID of the Commit. + // + // It is up to the caller to convert this to a dashless ID when necessary. + // + // May be empty, that is CommitID() == uuid.Nil may be true. + // Callers should not rely on this value being present. + // + // If FullName is nil, this will always be empty. + CommitID() uuid.UUID + // Description returns a human-readable description of the Plugin. + // + // This is used to construct descriptive error messages pointing to configured plugins. + // + // This will never be empty. If a description was not explicitly set, this falls back to + // OpaqueID. + Description() string + // Digest returns the Plugin digest for the given DigestType. + // + // Note this is *not* a bufcas.Digest - this is a Digest. + // bufcas.Digests are a lower-level type that just deal in terms of + // files and content. A Digest is a specific algorithm applied to the + // content of a Plugin. + // + // Will return an error if the Plugin is not a Wasm Plugin. + Digest(DigestType) (Digest, error) + // Data returns the bytes of the Plugin as a Wasm module. + // + // This is the raw bytes of the Wasm module in an uncompressed form. + // + // Will return an error if the Plugin is not a Wasm Plugin. + Data() ([]byte, error) + // IsWasm returns true if the Plugin is a Wasm Plugin. + // + // Plugins are either Wasm or not Wasm. + // + // A Wasm Plugin is a Plugin that is a Wasm module. Wasm Plugins are invoked + // with the wasm.Runtime. The Plugin will have Data and will be able to + // calculate Digests. + // + // Wasm Plugins will always have Data. + IsWasm() bool + // IsLocal returns true if the Plugin is a local Plugin. + // + // Plugins are either local or remote. + // + // A local Plugin is one that is built from sources from the "local context", + // such as a Workspace. Local Plugins are important for understanding what Plugins + // to push. + // + // Remote Plugins will always have FullNames. + IsLocal() bool + + isPlugin() +} + +// NewLocalWasmPlugin returns a new Plugin for a local Wasm plugin. +func NewLocalWasmPlugin( + pluginFullName bufparse.FullName, + getData func() ([]byte, error), +) (Plugin, error) { + return newPlugin( + "", // description + pluginFullName, + nil, // path + uuid.Nil, // commitID + true, // isWasm + true, // isLocal + getData, + ) +} + +// *** PRIVATE *** + +type plugin struct { + description string + pluginFullName bufparse.FullName + path []string + commitID uuid.UUID + isWasm bool + isLocal bool + getData func() ([]byte, error) + + digestTypeToGetDigest map[DigestType]func() (Digest, error) +} + +func newPlugin( + description string, + pluginFullName bufparse.FullName, + path []string, + commitID uuid.UUID, + isWasm bool, + isLocal bool, + getData func() ([]byte, error), +) (*plugin, error) { + if isWasm && getData == nil { + return nil, syserror.Newf("getData not present when constructing a Wasm Plugin") + } + if !isWasm && len(path) == 0 { + return nil, syserror.New("path not present when constructing a non-Wasm Plugin") + } + if !isLocal && pluginFullName == nil { + return nil, syserror.New("pluginFullName not present when constructing a remote Plugin") + } + if !isLocal && !isWasm { + return nil, syserror.New("non-Wasm remote Plugins are not supported") + } + if isLocal && commitID != uuid.Nil { + return nil, syserror.New("commitID present when constructing a local Plugin") + } + if pluginFullName == nil && commitID != uuid.Nil { + return nil, syserror.New("pluginFullName not present and commitID present when constructing a remote Plugin") + } + plugin := &plugin{ + description: description, + pluginFullName: pluginFullName, + path: path, + commitID: commitID, + isWasm: isWasm, + isLocal: isLocal, + getData: sync.OnceValues(getData), + } + plugin.digestTypeToGetDigest = newSyncOnceValueDigestTypeToGetDigestFuncForPlugin(plugin) + return plugin, nil +} + +func (p *plugin) OpaqueID() string { + if p.pluginFullName != nil { + return p.pluginFullName.String() + } + return strings.Join(p.path, " ") +} + +func (p *plugin) Path() []string { + return p.path +} + +func (p *plugin) FullName() bufparse.FullName { + return p.pluginFullName +} + +func (p *plugin) CommitID() uuid.UUID { + return p.commitID +} + +func (p *plugin) Description() string { + if p.description != "" { + return p.description + } + return p.OpaqueID() +} + +func (p *plugin) Data() ([]byte, error) { + if !p.isWasm { + return nil, fmt.Errorf("Plugin is not a Wasm Plugin") + } + return p.getData() +} + +func (p *plugin) Digest(digestType DigestType) (Digest, error) { + getDigest, ok := p.digestTypeToGetDigest[digestType] + if !ok { + return nil, syserror.Newf("DigestType %v was not in plugin.digestTypeToGetDigest", digestType) + } + return getDigest() +} + +func (p *plugin) IsWasm() bool { + return p.isWasm +} + +func (p *plugin) IsLocal() bool { + return p.isLocal +} + +func (p *plugin) isPlugin() {} + +func newSyncOnceValueDigestTypeToGetDigestFuncForPlugin(plugin *plugin) map[DigestType]func() (Digest, error) { + m := make(map[DigestType]func() (Digest, error)) + for digestType := range digestTypeToString { + m[digestType] = sync.OnceValues(newGetDigestFuncForPluginAndDigestType(plugin, digestType)) + } + return m +} + +func newGetDigestFuncForPluginAndDigestType(plugin *plugin, digestType DigestType) func() (Digest, error) { + return func() (Digest, error) { + data, err := plugin.getData() + if err != nil { + return nil, err + } + bufcasDigest, err := bufcas.NewDigest(data) + if err != nil { + return nil, err + } + return NewDigest(digestType, bufcasDigest) + } +} diff --git a/private/bufpkg/bufplugin/plugin_type.go b/private/bufpkg/bufplugin/plugin_type.go new file mode 100644 index 0000000000..2d06454ac4 --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_type.go @@ -0,0 +1,46 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "fmt" +) + +const ( + // PluginTypeCheck says the Plugin is a check plugin. + PluginTypeCheck = iota + 1 +) + +var ( + // AllPluginTypeStrings is all format strings without aliases. + // + // Sorted in the order we want to display them. + AllPluginTypeStrings = []string{ + "check", + } +) + +// PluginType is the type of a Plugin. +type PluginType int + +// ParsePluginType parses the PluginType from the string. +func ParsePluginType(s string) (PluginType, error) { + switch s { + case "check": + return PluginVisibilityPublic, nil + default: + return 0, fmt.Errorf("unknown PluginType: %q", s) + } +} diff --git a/private/bufpkg/bufplugin/plugin_visibility.go b/private/bufpkg/bufplugin/plugin_visibility.go new file mode 100644 index 0000000000..d9523989c9 --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_visibility.go @@ -0,0 +1,43 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "fmt" +) + +const ( + // PluginVisibilityPublic says the Plugin is public on the registry. + PluginVisibilityPublic = iota + 1 + // PluginVisibilityPublic says the Plugin is private on the registry. + PluginVisibilityPrivate +) + +// PluginVisibility is the visibility of a Plugin on a registry. +// +// Only used for Upload for now. +type PluginVisibility int + +// ParsePluginVisibility parses the PluginVisibility from the string. +func ParsePluginVisibility(s string) (PluginVisibility, error) { + switch s { + case "public": + return PluginVisibilityPublic, nil + case "private": + return PluginVisibilityPrivate, nil + default: + return 0, fmt.Errorf("unknown PluginVisibility: %q", s) + } +} diff --git a/private/bufpkg/bufplugin/uploader.go b/private/bufpkg/bufplugin/uploader.go new file mode 100644 index 0000000000..c5d2d09b44 --- /dev/null +++ b/private/bufpkg/bufplugin/uploader.go @@ -0,0 +1,156 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/bufbuild/buf/private/pkg/slicesext" +) + +var ( + // NopUploader is a no-op Uploader. + NopUploader Uploader = nopUploader{} +) + +// Uploader uploads Plugins. +type Uploader interface { + // Upload uploads the given Plugins. + Upload(ctx context.Context, plugins []Plugin, options ...UploadOption) ([]Commit, error) +} + +// UploadOption is an option for an Upload. +type UploadOption func(*uploadOptions) + +// UploadWithLabels returns a new UploadOption that adds the given labels. +// +// This can be called multiple times. The unique result set of labels will be used. +func UploadWithLabels(labels ...string) UploadOption { + return func(uploadOptions *uploadOptions) { + uploadOptions.labels = append(uploadOptions.labels, labels...) + } +} + +// UploadWithCreateIfNotExist returns a new UploadOption that will result in the +// Plugins being created on the registry with the given visibility if they do not exist. +func UploadWithCreateIfNotExist(createPluginVisibility PluginVisibility, createPluginType PluginType) UploadOption { + return func(uploadOptions *uploadOptions) { + uploadOptions.createIfNotExist = true + uploadOptions.createPluginVisibility = createPluginVisibility + uploadOptions.createPluginType = createPluginType + } +} + +// UploadWithSourceControlURL returns a new UploadOption that will set the source control +// url for the module contents uploaded. +func UploadWithSourceControlURL(sourceControlURL string) UploadOption { + return func(uploadOptions *uploadOptions) { + uploadOptions.sourceControlURL = sourceControlURL + } +} + +// UploadOptions are the possible options for upload. +// +// This is used by Uploader implementations. +type UploadOptions interface { + // Labels returns the unique and sorted set of labels to add. Labels + // are set using the `--label` flag when calling `buf plugin upload` + // and represent the labels that are set when uploading plugin data. + Labels() []string + // CreateIfNotExist says to create Plugins if they do not exist on the registry. + CreateIfNotExist() bool + // CreatePluginVisibility returns the visibility to create Plugins with. + // + // Will always be present if CreateIfNotExist() is true. + CreatePluginVisibility() PluginVisibility + // CreatePluginType returns the type to create Plugins with. + // + // Will always be present if CreateIfNotExist() is true. + CreatePluginType() PluginType + // SourceControlURL returns the source control URL set by the user for the module + // contents uploaded. We set the same source control URL for all module contents. + SourceControlURL() string + + isUploadOptions() +} + +// NewUploadOptions returns a new UploadOptions. +func NewUploadOptions(options []UploadOption) (UploadOptions, error) { + uploadOptions := newUploadOptions() + for _, option := range options { + option(uploadOptions) + } + if err := uploadOptions.validate(); err != nil { + return nil, err + } + return uploadOptions, nil +} + +// *** PRIVATE *** + +type nopUploader struct{} + +func (nopUploader) Upload(context.Context, []Plugin, ...UploadOption) ([]Commit, error) { + return nil, errors.New("unimplemented: no-op Uploader called") +} + +type uploadOptions struct { + labels []string + createIfNotExist bool + createPluginVisibility PluginVisibility + createPluginType PluginType + sourceControlURL string +} + +func newUploadOptions() *uploadOptions { + return &uploadOptions{} +} + +func (u *uploadOptions) Labels() []string { + return slicesext.ToUniqueSorted(u.labels) +} + +func (u *uploadOptions) CreateIfNotExist() bool { + return u.createIfNotExist +} + +func (u *uploadOptions) CreatePluginVisibility() PluginVisibility { + return u.createPluginVisibility +} + +func (u *uploadOptions) CreatePluginType() PluginType { + return u.createPluginType +} + +func (u *uploadOptions) SourceControlURL() string { + return u.sourceControlURL +} + +func (u *uploadOptions) isUploadOptions() {} + +func (u *uploadOptions) validate() error { + if u.createIfNotExist && u.createPluginVisibility == 0 { + return errors.New("must set a valid PluginVisibility if CreateIfNotExist was specified") + } + if u.sourceControlURL != "" { + if _, err := url.Parse(u.sourceControlURL); err != nil { + return fmt.Errorf("must set a valid url for the source control url: %w", err) + } + } + return nil +}