From 4b7a1e8f1c235d7b177194f2cf6b735f1bc93229 Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Thu, 14 Nov 2024 17:37:18 -0500 Subject: [PATCH 1/4] Add plugin push command This adds the `buf plugin push` command to upload Buf plugins to the BSR. Only Wasm binary check plugins are supported for now. Plugins must implement the PluginRPC framework. --- CHANGELOG.md | 3 +- private/buf/cmd/buf/buf.go | 16 +- .../buf/command/plugin/pluginpush/convert.go | 53 ++++ .../command/plugin/pluginpush/pluginpush.go | 280 ++++++++++++++++++ .../command/plugin/pluginpush/usage.gen.go | 19 ++ 5 files changed, 366 insertions(+), 5 deletions(-) create mode 100644 private/buf/cmd/buf/command/plugin/pluginpush/convert.go create mode 100644 private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go create mode 100644 private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cc2a049253..decc8d7096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## [Unreleased] -- No changes yet. +- Add `buf plugin push` command to push a plugin to the Buf Schema Registry. + Only WebAssembly check plugins are supported at this time. ## [v1.47.2] - 2024-11-14 diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index 57dcc15bfc..3a8f842ee9 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -35,8 +35,8 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/bufpluginv2" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/lsp" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/price" - "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/plugindelete" - "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/pluginpush" + betaplugindelete "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/plugindelete" + betapluginpush "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/plugin/pluginpush" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/webhook/webhookcreate" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/webhook/webhookdelete" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/webhook/webhooklist" @@ -62,6 +62,7 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlsbreakingrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlslintrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modopen" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/plugin/pluginpush" "github.com/bufbuild/buf/private/buf/cmd/buf/command/push" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/module/modulecommit/modulecommitaddlabel" "github.com/bufbuild/buf/private/buf/cmd/buf/command/registry/module/modulecommit/modulecommitinfo" @@ -171,6 +172,13 @@ func NewRootCommand(name string) *appcmd.Command { modlsbreakingrules.NewCommand("ls-breaking-rules", builder), }, }, + { + Use: "plugin", + Short: "Work with plugins", + SubCommands: []*appcmd.Command{ + pluginpush.NewCommand("push", builder), + }, + }, { Use: "registry", Short: "Manage assets on the Buf Schema Registry", @@ -282,8 +290,8 @@ func NewRootCommand(name string) *appcmd.Command { Use: "plugin", Short: "Manage plugins on the Buf Schema Registry", SubCommands: []*appcmd.Command{ - pluginpush.NewCommand("push", builder), - plugindelete.NewCommand("delete", builder), + betapluginpush.NewCommand("push", builder), + betaplugindelete.NewCommand("delete", builder), }, }, }, diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/convert.go b/private/buf/cmd/buf/command/plugin/pluginpush/convert.go new file mode 100644 index 0000000000..b62df9a658 --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginpush/convert.go @@ -0,0 +1,53 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginpush + +import ( + "fmt" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" +) + +var ( + v1beta1ProtoDigestTypeToDigestType = map[pluginv1beta1.DigestType]bufplugin.DigestType{ + pluginv1beta1.DigestType_DIGEST_TYPE_P1: bufplugin.DigestTypeP1, + } +) + +func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { + digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] + if !ok { + return 0, fmt.Errorf("unknown pluginv1beta1.DigestType: %v", protoDigestType) + } + return digestType, nil +} + +// v1beta1ProtoToDigest converts the given proto Digest to a Digest. +// +// Validation is performed to ensure the DigestType is known, and the value +// is a valid digest value for the given DigestType. +func v1beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, error) { + digestType, err := v1beta1ProtoToDigestType(protoDigest.Type) + if err != nil { + return nil, err + } + bufcasDigest, err := bufcas.NewDigest(protoDigest.Value) + if err != nil { + return nil, err + } + return bufplugin.NewDigest(digestType, bufcasDigest) +} diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go new file mode 100644 index 0000000000..282477377e --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go @@ -0,0 +1,280 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginpush + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/connectclient" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/bufbuild/buf/private/pkg/wasm" + "github.com/klauspost/compress/zstd" + "github.com/spf13/pflag" +) + +const ( + labelFlagName = "label" + binaryFlagName = "binary" + sourceControlURLFlagName = "source-control-url" +) + +// NewCommand returns a new Command. +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Push a plugin to a registry", + Long: `The first argument is the plugin full name in the format .`, + Args: appcmd.MaximumNArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container, flags) + }, + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + Labels []string + Binary string + SourceControlURL string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + flagSet.StringSliceVar( + &f.Labels, + labelFlagName, + nil, + "Associate the label with the plugins pushed. Can be used multiple times.", + ) + flagSet.StringVar( + &f.Binary, + binaryFlagName, + "", + "The path to the Wasm binary file to push.", + ) + flagSet.StringVar( + &f.SourceControlURL, + sourceControlURLFlagName, + "", + "The URL for viewing the source code of the pushed plugins (e.g. the specific commit in source control).", + ) +} + +func run( + ctx context.Context, + container appext.Container, + flags *flags, +) (retErr error) { + if err := validateFlags(flags); err != nil { + return err + } + // We parse the plugin full name from the user-provided argument. + pluginFullName, err := bufparse.ParseFullName(container.Arg(0)) + if err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + + clientConfig, err := bufcli.NewConnectClientConfig(container) + if err != nil { + return err + } + pluginKey, err := upload(ctx, container, flags, clientConfig, pluginFullName) + if err != nil { + return err + } + // Only one plugin key is returned. + if _, err := fmt.Fprintf(container.Stdout(), "%s\n", pluginKey.String()); err != nil { + return syserror.Wrap(err) + } + return nil +} + +func upload( + ctx context.Context, + container appext.Container, + flags *flags, + clientConfig *connectclient.Config, + pluginFullName bufparse.FullName, +) (_ bufplugin.PluginKey, retErr error) { + switch { + case flags.Binary != "": + return uploadBinary(ctx, container, flags, clientConfig, pluginFullName) + default: + // This should never happen because the flags are validated. + return nil, syserror.Newf("--%s must be set", binaryFlagName) + } +} + +func uploadBinary( + ctx context.Context, + container appext.Container, + flags *flags, + clientConfig *connectclient.Config, + pluginFullName bufparse.FullName, +) (pluginKey bufplugin.PluginKey, retErr error) { + uploadServiceClient := bufregistryapiplugin.NewClientProvider(clientConfig). + V1Beta1UploadServiceClient(pluginFullName.Registry()) + + wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + if err != nil { + return nil, err + } + wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + if err != nil { + return nil, err + } + defer func() { + retErr = errors.Join(retErr, wasmRuntime.Close(ctx)) + }() + // Load the binary from the `--binary` flag. + wasmBinary, err := os.ReadFile(flags.Binary) + if err != nil { + return nil, fmt.Errorf("could not read binary %q: %w", flags.Binary, err) + } + compressionType := pluginv1beta1.CompressionType_COMPRESSION_TYPE_ZSTD + compressedWasmBinary, err := zstdCompress(wasmBinary) + if err != nil { + return nil, fmt.Errorf("could not compress binary %q: %w", flags.Binary, err) + } + + // Defer validation of the plugin binary to the server, but compile the + // binary locally to catch any errors early. + _, err = wasmRuntime.Compile(ctx, pluginFullName.Name(), wasmBinary) + if err != nil { + return nil, fmt.Errorf("could not compile binary %q: %w", flags.Binary, err) + } + // Upload the binary to the registry. + content := &pluginv1beta1.UploadRequest_Content{ + PluginRef: &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: pluginFullName.Owner(), + Plugin: pluginFullName.Name(), + }, + }, + }, + CompressionType: compressionType, + Content: compressedWasmBinary, + ScopedLabelRefs: slicesext.Map(flags.Labels, func(label string) *pluginv1beta1.ScopedLabelRef { + return &pluginv1beta1.ScopedLabelRef{ + Value: &pluginv1beta1.ScopedLabelRef_Name{ + Name: label, + }, + } + }), + SourceControlUrl: flags.SourceControlURL, + } + uploadResponse, err := uploadServiceClient.Upload(ctx, connect.NewRequest(&pluginv1beta1.UploadRequest{ + Contents: []*pluginv1beta1.UploadRequest_Content{content}, + })) + if err != nil { + return nil, err + } + if len(uploadResponse.Msg.Commits) != 1 { + return nil, syserror.Newf("unexpected number of commits returned from server: %d", len(uploadResponse.Msg.Commits)) + } + protoCommit := uploadResponse.Msg.Commits[0] + commitID, err := uuidutil.FromDashless(protoCommit.Id) + if err != nil { + return nil, err + } + pluginKey, err = bufplugin.NewPluginKey( + pluginFullName, + commitID, + func() (bufplugin.Digest, error) { + return v1beta1ProtoToDigest(protoCommit.Digest) + }, + ) + if err != nil { + return nil, err + } + return pluginKey, nil +} + +func zstdCompress(data []byte) ([]byte, error) { + encoder, err := zstd.NewWriter(nil) + if err != nil { + return nil, fmt.Errorf("failed to create zstd encoder: %w", err) + } + defer encoder.Close() + return encoder.EncodeAll(data, nil), nil +} + +func validateFlags(flags *flags) error { + if err := validateLabelFlags(flags); err != nil { + return err + } + if err := validateTypeFlags(flags); err != nil { + return err + } + return nil +} + +func validateLabelFlags(flags *flags) error { + return validateLabelFlagValues(flags) +} + +func validateTypeFlags(flags *flags) error { + var typeFlags []string + if flags.Binary != "" { + typeFlags = append(typeFlags, binaryFlagName) + } + if len(typeFlags) > 1 { + usedFlagsErrStr := strings.Join( + slicesext.Map( + typeFlags, + func(flag string) string { return fmt.Sprintf("--%s", flag) }, + ), + ", ", + ) + return appcmd.NewInvalidArgumentErrorf("These flags cannot be used in combination with one another: %s", usedFlagsErrStr) + } + if len(typeFlags) == 0 { + return appcmd.NewInvalidArgumentErrorf("--%s must be set", binaryFlagName) + } + return nil +} + +func validateLabelFlagValues(flags *flags) error { + for _, label := range flags.Labels { + if label == "" { + return appcmd.NewInvalidArgumentErrorf("--%s requires a non-empty string", labelFlagName) + } + } + return nil +} diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go b/private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go new file mode 100644 index 0000000000..3184fae49e --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginpush/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package pluginpush + +import _ "github.com/bufbuild/buf/private/usage" From 717b0d514f607db1fff99d45c8a34ce9be34e28e Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Mon, 25 Nov 2024 18:41:27 -0500 Subject: [PATCH 2/4] Implement plugin uploader --- private/buf/bufcli/uploader.go | 30 +- .../command/plugin/pluginpush/pluginpush.go | 145 +++------ private/buf/cmd/buf/command/push/push.go | 6 +- .../bufplugin/bufpluginapi/bufpluginapi.go | 15 + .../bufplugin/bufpluginapi}/convert.go | 35 +- .../bufpkg/bufplugin/bufpluginapi/uploader.go | 308 ++++++++++++++++++ .../bufplugin/bufpluginapi/usage.gen.go | 19 ++ private/bufpkg/bufplugin/plugin.go | 243 ++++++++++++++ private/bufpkg/bufplugin/plugin_visibility.go | 43 +++ private/bufpkg/bufplugin/uploader.go | 146 +++++++++ 10 files changed, 865 insertions(+), 125 deletions(-) create mode 100644 private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go rename private/{buf/cmd/buf/command/plugin/pluginpush => bufpkg/bufplugin/bufpluginapi}/convert.go (73%) create mode 100644 private/bufpkg/bufplugin/bufpluginapi/uploader.go create mode 100644 private/bufpkg/bufplugin/bufpluginapi/usage.gen.go create mode 100644 private/bufpkg/bufplugin/plugin.go create mode 100644 private/bufpkg/bufplugin/plugin_visibility.go create mode 100644 private/bufpkg/bufplugin/uploader.go diff --git a/private/buf/bufcli/uploader.go b/private/buf/bufcli/uploader.go index df2d047a5e..4061c8114c 100644 --- a/private/buf/bufcli/uploader.go +++ b/private/buf/bufcli/uploader.go @@ -17,20 +17,32 @@ package bufcli import ( "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleapi" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginapi" "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapimodule" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" "github.com/bufbuild/buf/private/pkg/app/appext" ) -// NewUploader returns a new Uploader. -func NewUploader(container appext.Container) (bufmodule.Uploader, error) { +// NewModuleUploader returns a new Uploader for ModuleSets. +func NewModuleUploader(container appext.Container) (bufmodule.Uploader, error) { clientConfig, err := NewConnectClientConfig(container) if err != nil { return nil, err } - return newUploader(container, bufregistryapimodule.NewClientProvider(clientConfig)), nil + return newModuleUploader(container, bufregistryapimodule.NewClientProvider(clientConfig)), nil } -func newUploader( +// NewPluginUploader returns a new Uploader for Plugins. +func NewPluginUploader(container appext.Container) (bufplugin.Uploader, error) { + clientConfig, err := NewConnectClientConfig(container) + if err != nil { + return nil, err + } + return newPluginUploader(container, bufregistryapiplugin.NewClientProvider(clientConfig)), nil +} + +func newModuleUploader( container appext.Container, clientProvider bufregistryapimodule.ClientProvider, ) bufmodule.Uploader { @@ -41,3 +53,13 @@ func newUploader( bufmoduleapi.UploaderWithPublicRegistry(container.Env(publicRegistryEnvKey)), ) } + +func newPluginUploader( + container appext.Container, + clientProvider bufregistryapiplugin.ClientProvider, +) bufplugin.Uploader { + return bufpluginapi.NewUploader( + container.Logger(), + clientProvider, + ) +} diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go index 282477377e..a32fd2fdec 100644 --- a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go +++ b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go @@ -16,31 +16,25 @@ package pluginpush import ( "context" - "errors" "fmt" "os" "strings" - pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" - "connectrpc.com/connect" "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/bufpkg/bufparse" "github.com/bufbuild/buf/private/bufpkg/bufplugin" - "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" - "github.com/bufbuild/buf/private/pkg/connectclient" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" - "github.com/bufbuild/buf/private/pkg/uuidutil" - "github.com/bufbuild/buf/private/pkg/wasm" - "github.com/klauspost/compress/zstd" "github.com/spf13/pflag" ) const ( labelFlagName = "label" binaryFlagName = "binary" + createFlagName = "create" + createVisibilityFlagName = "create-visibility" sourceControlURLFlagName = "source-control-url" ) @@ -67,6 +61,8 @@ func NewCommand( type flags struct { Labels []string Binary string + Create bool + CreateVisibility string SourceControlURL string } @@ -75,6 +71,7 @@ func newFlags() *flags { } func (f *flags) Bind(flagSet *pflag.FlagSet) { + bufcli.BindCreateVisibility(flagSet, &f.CreateVisibility, createVisibilityFlagName, createFlagName) flagSet.StringSliceVar( &f.Labels, labelFlagName, @@ -87,6 +84,15 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { "", "The path to the Wasm binary file to push.", ) + flagSet.BoolVar( + &f.Create, + createFlagName, + false, + fmt.Sprintf( + "Create the plugin if it does not exist. Defaults to creating a private repository if --%s is not set.", + createVisibilityFlagName, + ), + ) flagSet.StringVar( &f.SourceControlURL, sourceControlURLFlagName, @@ -108,17 +114,12 @@ func run( if err != nil { return appcmd.WrapInvalidArgumentError(err) } - - clientConfig, err := bufcli.NewConnectClientConfig(container) - if err != nil { - return err - } - pluginKey, err := upload(ctx, container, flags, clientConfig, pluginFullName) + commit, err := upload(ctx, container, flags, pluginFullName) if err != nil { return err } - // Only one plugin key is returned. - if _, err := fmt.Fprintf(container.Stdout(), "%s\n", pluginKey.String()); err != nil { + // Only one commit is returned. + if _, err := fmt.Fprintf(container.Stdout(), "%s\n", commit.PluginKey().String()); err != nil { return syserror.Wrap(err) } return nil @@ -128,111 +129,41 @@ func upload( ctx context.Context, container appext.Container, flags *flags, - clientConfig *connectclient.Config, pluginFullName bufparse.FullName, -) (_ bufplugin.PluginKey, retErr error) { +) (_ bufplugin.Commit, retErr error) { + var plugin bufplugin.Plugin switch { case flags.Binary != "": - return uploadBinary(ctx, container, flags, clientConfig, pluginFullName) + var err error + plugin, err = bufplugin.NewLocalWasmPlugin( + pluginFullName, + func() ([]byte, error) { + wasmBinary, err := os.ReadFile(flags.Binary) + if err != nil { + return nil, fmt.Errorf("could not read Wasm binary %q: %w", flags.Binary, err) + } + return wasmBinary, nil + }, + ) + if err != nil { + return nil, err + } default: // This should never happen because the flags are validated. return nil, syserror.Newf("--%s must be set", binaryFlagName) } -} - -func uploadBinary( - ctx context.Context, - container appext.Container, - flags *flags, - clientConfig *connectclient.Config, - pluginFullName bufparse.FullName, -) (pluginKey bufplugin.PluginKey, retErr error) { - uploadServiceClient := bufregistryapiplugin.NewClientProvider(clientConfig). - V1Beta1UploadServiceClient(pluginFullName.Registry()) - - wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container) + uploader, err := bufcli.NewPluginUploader(container) if err != nil { return nil, err } - wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir)) + commits, err := uploader.Upload(ctx, []bufplugin.Plugin{plugin}) if err != nil { return nil, err } - defer func() { - retErr = errors.Join(retErr, wasmRuntime.Close(ctx)) - }() - // Load the binary from the `--binary` flag. - wasmBinary, err := os.ReadFile(flags.Binary) - if err != nil { - return nil, fmt.Errorf("could not read binary %q: %w", flags.Binary, err) - } - compressionType := pluginv1beta1.CompressionType_COMPRESSION_TYPE_ZSTD - compressedWasmBinary, err := zstdCompress(wasmBinary) - if err != nil { - return nil, fmt.Errorf("could not compress binary %q: %w", flags.Binary, err) - } - - // Defer validation of the plugin binary to the server, but compile the - // binary locally to catch any errors early. - _, err = wasmRuntime.Compile(ctx, pluginFullName.Name(), wasmBinary) - if err != nil { - return nil, fmt.Errorf("could not compile binary %q: %w", flags.Binary, err) - } - // Upload the binary to the registry. - content := &pluginv1beta1.UploadRequest_Content{ - PluginRef: &pluginv1beta1.PluginRef{ - Value: &pluginv1beta1.PluginRef_Name_{ - Name: &pluginv1beta1.PluginRef_Name{ - Owner: pluginFullName.Owner(), - Plugin: pluginFullName.Name(), - }, - }, - }, - CompressionType: compressionType, - Content: compressedWasmBinary, - ScopedLabelRefs: slicesext.Map(flags.Labels, func(label string) *pluginv1beta1.ScopedLabelRef { - return &pluginv1beta1.ScopedLabelRef{ - Value: &pluginv1beta1.ScopedLabelRef_Name{ - Name: label, - }, - } - }), - SourceControlUrl: flags.SourceControlURL, - } - uploadResponse, err := uploadServiceClient.Upload(ctx, connect.NewRequest(&pluginv1beta1.UploadRequest{ - Contents: []*pluginv1beta1.UploadRequest_Content{content}, - })) - if err != nil { - return nil, err - } - if len(uploadResponse.Msg.Commits) != 1 { - return nil, syserror.Newf("unexpected number of commits returned from server: %d", len(uploadResponse.Msg.Commits)) - } - protoCommit := uploadResponse.Msg.Commits[0] - commitID, err := uuidutil.FromDashless(protoCommit.Id) - if err != nil { - return nil, err - } - pluginKey, err = bufplugin.NewPluginKey( - pluginFullName, - commitID, - func() (bufplugin.Digest, error) { - return v1beta1ProtoToDigest(protoCommit.Digest) - }, - ) - if err != nil { - return nil, err - } - return pluginKey, nil -} - -func zstdCompress(data []byte) ([]byte, error) { - encoder, err := zstd.NewWriter(nil) - if err != nil { - return nil, fmt.Errorf("failed to create zstd encoder: %w", err) + if len(commits) != 1 { + return nil, syserror.Newf("unexpected number of commits returned from server: %d", len(commits)) } - defer encoder.Close() - return encoder.EncodeAll(data, nil), nil + return commits[0], nil } func validateFlags(flags *flags) error { diff --git a/private/buf/cmd/buf/command/push/push.go b/private/buf/cmd/buf/command/push/push.go index 213db7689e..53e3545a86 100644 --- a/private/buf/cmd/buf/command/push/push.go +++ b/private/buf/cmd/buf/command/push/push.go @@ -126,7 +126,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { createFlagName, false, fmt.Sprintf( - "Create the repository if it does not exist. Defaults to creating a private repository if --%s is not set.", + "Create the module if it does not exist. Defaults to creating a private module if --%s is not set.", createVisibilityFlagName, ), ) @@ -134,7 +134,7 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { &f.CreateDefaultLabel, createDefaultLabelFlagName, "", - `The repository's default label setting, if created. If this is not set, then the repository will be created with the default label "main".`, + `The module's default label setting, if created. If this is not set, then the module will be created with the default label "main".`, ) flagSet.StringVar( &f.SourceControlURL, @@ -218,7 +218,7 @@ func run( return err } - uploader, err := bufcli.NewUploader(container) + uploader, err := bufcli.NewModuleUploader(container) if err != nil { return err } diff --git a/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go new file mode 100644 index 0000000000..816d5770ad --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/bufpluginapi.go @@ -0,0 +1,15 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/convert.go b/private/bufpkg/bufplugin/bufpluginapi/convert.go similarity index 73% rename from private/buf/cmd/buf/command/plugin/pluginpush/convert.go rename to private/bufpkg/bufplugin/bufpluginapi/convert.go index b62df9a658..dcdd179ea7 100644 --- a/private/buf/cmd/buf/command/plugin/pluginpush/convert.go +++ b/private/bufpkg/bufplugin/bufpluginapi/convert.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pluginpush +package bufpluginapi import ( "fmt" @@ -28,19 +28,11 @@ var ( } ) -func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { - digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] - if !ok { - return 0, fmt.Errorf("unknown pluginv1beta1.DigestType: %v", protoDigestType) - } - return digestType, nil -} - -// v1beta1ProtoToDigest converts the given proto Digest to a Digest. +// V1Beta1ProtoToDigest converts the given proto Digest to a Digest. // // Validation is performed to ensure the DigestType is known, and the value // is a valid digest value for the given DigestType. -func v1beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, error) { +func V1Beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, error) { digestType, err := v1beta1ProtoToDigestType(protoDigest.Type) if err != nil { return nil, err @@ -51,3 +43,24 @@ func v1beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, } return bufplugin.NewDigest(digestType, bufcasDigest) } + +// *** PRIVATE *** + +func pluginVisibilityToV1Proto(pluginVisibility bufplugin.PluginVisibility) (pluginv1beta1.PluginVisibility, error) { + switch pluginVisibility { + case bufplugin.PluginVisibilityPublic: + return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PUBLIC, nil + case bufplugin.PluginVisibilityPrivate: + return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PRIVATE, nil + default: + return 0, fmt.Errorf("unknown PluginVisibility: %v", pluginVisibility) + } +} + +func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { + digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] + if !ok { + return 0, fmt.Errorf("unknown pluginv1beta1.DigestType: %v", protoDigestType) + } + return digestType, nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/uploader.go b/private/bufpkg/bufplugin/bufpluginapi/uploader.go new file mode 100644 index 0000000000..fe12a2bee2 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/uploader.go @@ -0,0 +1,308 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufpluginapi + +import ( + "context" + "fmt" + "log/slog" + "time" + + ownerv1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/owner/v1" + pluginv1beta1 "buf.build/gen/go/bufbuild/registry/protocolbuffers/go/buf/registry/plugin/v1beta1" + "connectrpc.com/connect" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/bufpkg/bufregistryapi/bufregistryapiplugin" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/bufbuild/buf/private/pkg/uuidutil" + "github.com/klauspost/compress/zstd" +) + +// NewUploader returns a new Uploader for the given API client. +func NewUploader( + logger *slog.Logger, + pluginClientProvider interface { + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + }, + options ...UploaderOption, +) bufplugin.Uploader { + return newUploader(logger, pluginClientProvider, options...) +} + +// UploaderOption is an option for a new Uploader. +type UploaderOption func(*uploader) + +// *** PRIVATE *** + +type uploader struct { + logger *slog.Logger + pluginClientProvider interface { + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + } +} + +func newUploader( + logger *slog.Logger, + pluginClientProvider interface { + bufregistryapiplugin.V1Beta1PluginServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + bufregistryapiplugin.V1Beta1UploadServiceClientProvider + }, + options ...UploaderOption, +) *uploader { + uploader := &uploader{ + logger: logger, + pluginClientProvider: pluginClientProvider, + } + for _, option := range options { + option(uploader) + } + return uploader +} + +func (u *uploader) Upload( + ctx context.Context, + plugins []bufplugin.Plugin, + options ...bufplugin.UploadOption, +) ([]bufplugin.Commit, error) { + uploadOptions, err := bufplugin.NewUploadOptions(options) + if err != nil { + return nil, err + } + registryToIndexedPluginKeys := slicesext.ToIndexedValuesMap( + plugins, + func(plugin bufplugin.Plugin) string { + return plugin.FullName().Registry() + }, + ) + indexedCommits := make([]slicesext.Indexed[bufplugin.Commit], 0, len(plugins)) + for registry, indexedPluginKeys := range registryToIndexedPluginKeys { + indexedRegistryPluginDatas, err := u.uploadIndexedPluginsForRegistry( + ctx, + registry, + indexedPluginKeys, + uploadOptions, + ) + if err != nil { + return nil, err + } + indexedCommits = append(indexedCommits, indexedRegistryPluginDatas...) + } + return slicesext.IndexedToSortedValues(indexedCommits), nil +} + +func (u *uploader) uploadIndexedPluginsForRegistry( + ctx context.Context, + registry string, + indexedPlugins []slicesext.Indexed[bufplugin.Plugin], + uploadOptions bufplugin.UploadOptions, +) ([]slicesext.Indexed[bufplugin.Commit], error) { + if uploadOptions.CreateIfNotExist() { + // We must attempt to create each Plugin one at a time, since CreatePlugins will return + // an `AlreadyExists` if any of the Plugins we are attempting to create already exists, + // and no new Plugins will be created. + for _, indexedPlugin := range indexedPlugins { + plugin := indexedPlugin.Value + if _, err := u.createPluginIfNotExist( + ctx, + registry, + plugin, + uploadOptions.CreatePluginVisibility(), + ); err != nil { + return nil, err + } + } + } + contents, err := slicesext.MapError(indexedPlugins, func(indexedPlugin slicesext.Indexed[bufplugin.Plugin]) (*pluginv1beta1.UploadRequest_Content, error) { + plugin := indexedPlugin.Value + if !plugin.IsLocal() { + return nil, syserror.New("expected local Plugin in uploadIndexedPluginsForRegistry") + } + if plugin.FullName() == nil { + return nil, syserror.Newf("expected Plugin name for local Plugin: %s", plugin.Description()) + } + data, err := plugin.Data() + if err != nil { + return nil, err + } + compressedWasmBinary, err := zstdCompress(data) + if err != nil { + return nil, fmt.Errorf("could not compress Plugin data %q: %w", plugin.OpaqueID(), err) + } + return &pluginv1beta1.UploadRequest_Content{ + PluginRef: &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: plugin.FullName().Owner(), + Plugin: plugin.FullName().Name(), + }, + }, + }, + CompressionType: pluginv1beta1.CompressionType_COMPRESSION_TYPE_ZSTD, + Content: compressedWasmBinary, + ScopedLabelRefs: slicesext.Map(uploadOptions.Labels(), func(label string) *pluginv1beta1.ScopedLabelRef { + return &pluginv1beta1.ScopedLabelRef{ + Value: &pluginv1beta1.ScopedLabelRef_Name{ + Name: label, + }, + } + }), + SourceControlUrl: uploadOptions.SourceControlURL(), + }, nil + }) + if err != nil { + return nil, err + } + + uploadResponse, err := u.pluginClientProvider.V1Beta1UploadServiceClient(registry).Upload( + ctx, + connect.NewRequest(&pluginv1beta1.UploadRequest{ + Contents: contents, + })) + if err != nil { + return nil, err + } + pluginCommits := uploadResponse.Msg.Commits + if len(pluginCommits) != len(indexedPlugins) { + return nil, syserror.New("did not get the expected number of plugin commits") + } + + indexedCommits := make([]slicesext.Indexed[bufplugin.Commit], 0, len(indexedPlugins)) + for i, pluginCommit := range pluginCommits { + pluginFullName := indexedPlugins[i].Value.FullName() + commitID, err := uuidutil.FromDashless(pluginCommit.Id) + if err != nil { + return nil, err + } + pluginKey, err := bufplugin.NewPluginKey( + pluginFullName, + commitID, + func() (bufplugin.Digest, error) { + return V1Beta1ProtoToDigest(pluginCommit.Digest) + }, + ) + if err != nil { + return nil, err + } + commit := bufplugin.NewCommit( + pluginKey, + func() (time.Time, error) { + return pluginCommit.CreateTime.AsTime(), nil + }, + ) + indexedCommits = append( + indexedCommits, + slicesext.Indexed[bufplugin.Commit]{ + Value: commit, + Index: i, + }, + ) + } + return indexedCommits, nil +} + +func (u *uploader) createPluginIfNotExist( + ctx context.Context, + primaryRegistry string, + plugin bufplugin.Plugin, + createPluginVisibility bufplugin.PluginVisibility, +) (*pluginv1beta1.Plugin, error) { + v1ProtoCreatePluginVisibility, err := pluginVisibilityToV1Proto(createPluginVisibility) + if err != nil { + return nil, err + } + response, err := u.pluginClientProvider.V1Beta1PluginServiceClient(primaryRegistry).CreatePlugins( + ctx, + connect.NewRequest( + &pluginv1beta1.CreatePluginsRequest{ + Values: []*pluginv1beta1.CreatePluginsRequest_Value{ + { + OwnerRef: &ownerv1.OwnerRef{ + Value: &ownerv1.OwnerRef_Name{ + Name: plugin.FullName().Owner(), + }, + }, + Name: plugin.FullName().Name(), + Visibility: v1ProtoCreatePluginVisibility, + }, + }, + }, + ), + ) + if err != nil { + if connect.CodeOf(err) == connect.CodeAlreadyExists { + // If a plugin already existed, then we check validate its contents. + plugins, err := u.validatePluginsExist(ctx, primaryRegistry, []bufplugin.Plugin{plugin}) + if err != nil { + return nil, err + } + if len(plugins) != 1 { + return nil, syserror.Newf("expected 1 Plugin, found %d", len(plugins)) + } + return plugins[0], nil + } + return nil, err + } + if len(response.Msg.Plugins) != 1 { + return nil, syserror.Newf("expected 1 Plugin, found %d", len(response.Msg.Plugins)) + } + // Otherwise we return the plugin we created. + return response.Msg.Plugins[0], nil +} + +func (u *uploader) validatePluginsExist( + ctx context.Context, + primaryRegistry string, + plugins []bufplugin.Plugin, +) ([]*pluginv1beta1.Plugin, error) { + response, err := u.pluginClientProvider.V1Beta1PluginServiceClient(primaryRegistry).GetPlugins( + ctx, + connect.NewRequest( + &pluginv1beta1.GetPluginsRequest{ + PluginRefs: slicesext.Map( + plugins, + func(plugin bufplugin.Plugin) *pluginv1beta1.PluginRef { + return &pluginv1beta1.PluginRef{ + Value: &pluginv1beta1.PluginRef_Name_{ + Name: &pluginv1beta1.PluginRef_Name{ + Owner: plugin.FullName().Owner(), + Plugin: plugin.FullName().Name(), + }, + }, + } + }, + ), + }, + ), + ) + if err != nil { + return nil, err + } + return response.Msg.Plugins, nil +} + +func zstdCompress(data []byte) ([]byte, error) { + encoder, err := zstd.NewWriter(nil) + if err != nil { + return nil, fmt.Errorf("failed to create zstd encoder: %w", err) + } + defer encoder.Close() + return encoder.EncodeAll(data, nil), nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go b/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go new file mode 100644 index 0000000000..cba34bb462 --- /dev/null +++ b/private/bufpkg/bufplugin/bufpluginapi/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufpluginapi + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufplugin/plugin.go b/private/bufpkg/bufplugin/plugin.go new file mode 100644 index 0000000000..fdc8b43cfc --- /dev/null +++ b/private/bufpkg/bufplugin/plugin.go @@ -0,0 +1,243 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "fmt" + "strings" + "sync" + + "github.com/bufbuild/buf/private/bufpkg/bufcas" + "github.com/bufbuild/buf/private/bufpkg/bufparse" + "github.com/bufbuild/buf/private/pkg/syserror" + "github.com/google/uuid" +) + +// Plugin presents a BSR plugin. +type Plugin interface { + // Data returns the bytes of the Plugin as a Wasm module. + // + // This is the raw bytes of the Wasm module in an uncompressed form. + // + // Will be nil if the plugin is not of type PluginTypeWasm. + Data() ([]byte, error) + // OpaqueID returns an unstructured ID that can uniquely identify a Plugin + // relative to the Workspace. + // + // An OpaqueID's structure should not be relied upon, and is not a + // globally-unique identifier. It's uniqueness property only applies to + // the lifetime of the Plugin, and only within Plugin commonly built + // from the Workspace root. + // + // If two Plugins have the same FullName, they will have the same OpaqueID. + OpaqueID() string + // Path returns the path, including arguments, to invoke the binary plugin. + // + // This is not empty only when the plugin is local. + Path() []string + // FullName returns the FullName of the Plugin. + // + // This is nil + FullName() bufparse.FullName + // CommitID returns the BSR ID of the Commit. + // + // It is up to the caller to convert this to a dashless ID when necessary. + // + // May be empty, that is CommitID() == uuid.Nil may be true. + // Callers should not rely on this value being present. + // + // If FullName is nil, this will always be empty. + CommitID() uuid.UUID + // Description returns a human-readable description of the Plugin. + // + // This is used to construct descriptive error messages pointing to configured plugins. + // + // This will never be empty. If a description was not explicitly set, this falls back to + // OpaqueID. + Description() string + // Digest returns the Plugin digest for the given DigestType. + // + // Note this is *not* a bufcas.Digest - this is a Digest. + // bufcas.Digests are a lower-level type that just deal in terms of + // files and content. A Digest is a specific algorithm applied to the + // content of a Plugin. + // + // Digest may return an error if the Plugin is not a Wasm plugin. + Digest(DigestType) (Digest, error) + // IsWasm returns true if the Plugin is a Wasm Plugin. + // + // Plugins are either Wasm or not Wasm. + // + // A Wasm Plugin is a Plugin that is a Wasm module. Wasm Plugins are invoked + // with a wasm.Runtime. + // + // Wasm Plugins will always have Data. + IsWasm() bool + // IsLocal returns true if the Plugin is a local Plugin. + // + // Plugins are either local or remote. + // + // A local Plugin is one that is built from sources from the "local context", + // such as a Workspace. Local Plugins are important for understanding what Plugins + // to push. + // + // Remote Plugins will always have FullNames. + IsLocal() bool + + isPlugin() +} + +// NewLocalWasmPlugin returns a new Plugin for a local Wasm plugin. +func NewLocalWasmPlugin( + pluginFullName bufparse.FullName, + getData func() ([]byte, error), +) (Plugin, error) { + return newPlugin( + "", // description + pluginFullName, + nil, // path + uuid.Nil, // commitID + true, // isWasm + true, // isLocal + getData, + ) +} + +// *** PRIVATE *** + +type plugin struct { + description string + pluginFullName bufparse.FullName + path []string + commitID uuid.UUID + isWasm bool + isLocal bool + getData func() ([]byte, error) + + digestTypeToGetDigest map[DigestType]func() (Digest, error) +} + +func newPlugin( + description string, + pluginFullName bufparse.FullName, + path []string, + commitID uuid.UUID, + isWasm bool, + isLocal bool, + getData func() ([]byte, error), +) (*plugin, error) { + if isWasm && getData == nil { + return nil, syserror.Newf("getData not present when constructing a Wasm Plugin") + } + if !isWasm && len(path) == 0 { + return nil, syserror.New("path not present when constructing a non-Wasm Plugin") + } + if !isLocal && pluginFullName == nil { + return nil, syserror.New("pluginFullName not present when constructing a remote Plugin") + } + if !isLocal && !isWasm { + return nil, syserror.New("non-Wasm remote Plugins are not supported") + } + if isLocal && commitID != uuid.Nil { + return nil, syserror.New("commitID present when constructing a local Plugin") + } + if pluginFullName == nil && commitID != uuid.Nil { + return nil, syserror.New("pluginFullName not present and commitID present when constructing a remote Plugin") + } + plugin := &plugin{ + description: description, + pluginFullName: pluginFullName, + path: path, + commitID: commitID, + isWasm: isWasm, + isLocal: isLocal, + getData: sync.OnceValues(getData), + } + plugin.digestTypeToGetDigest = newSyncOnceValueDigestTypeToGetDigestFuncForPlugin(plugin) + return plugin, nil +} + +func (p *plugin) OpaqueID() string { + if p.pluginFullName != nil { + return p.pluginFullName.String() + } + return strings.Join(p.path, " ") +} + +func (p *plugin) Path() []string { + return p.path +} + +func (p *plugin) FullName() bufparse.FullName { + return p.pluginFullName +} + +func (p *plugin) CommitID() uuid.UUID { + return p.commitID +} + +func (p *plugin) Description() string { + if p.description != "" { + return p.description + } + return p.OpaqueID() +} + +func (p *plugin) Data() ([]byte, error) { + if !p.isWasm { + return nil, fmt.Errorf("Plugin is not a Wasm Plugin") + } + return p.getData() +} + +func (p *plugin) Digest(digestType DigestType) (Digest, error) { + getDigest, ok := p.digestTypeToGetDigest[digestType] + if !ok { + return nil, syserror.Newf("DigestType %v was not in plugin.digestTypeToGetDigest", digestType) + } + return getDigest() +} + +func (p *plugin) IsWasm() bool { + return p.isWasm +} + +func (p *plugin) IsLocal() bool { + return p.isLocal +} + +func (p *plugin) isPlugin() {} + +func newSyncOnceValueDigestTypeToGetDigestFuncForPlugin(plugin *plugin) map[DigestType]func() (Digest, error) { + m := make(map[DigestType]func() (Digest, error)) + for digestType := range digestTypeToString { + m[digestType] = sync.OnceValues(newGetDigestFuncForPluginAndDigestType(plugin, digestType)) + } + return m +} + +func newGetDigestFuncForPluginAndDigestType(plugin *plugin, digestType DigestType) func() (Digest, error) { + return func() (Digest, error) { + data, err := plugin.getData() + if err != nil { + return nil, err + } + bufcasDigest, err := bufcas.NewDigest(data) + if err != nil { + return nil, err + } + return NewDigest(digestType, bufcasDigest) + } +} diff --git a/private/bufpkg/bufplugin/plugin_visibility.go b/private/bufpkg/bufplugin/plugin_visibility.go new file mode 100644 index 0000000000..d9523989c9 --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_visibility.go @@ -0,0 +1,43 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "fmt" +) + +const ( + // PluginVisibilityPublic says the Plugin is public on the registry. + PluginVisibilityPublic = iota + 1 + // PluginVisibilityPublic says the Plugin is private on the registry. + PluginVisibilityPrivate +) + +// PluginVisibility is the visibility of a Plugin on a registry. +// +// Only used for Upload for now. +type PluginVisibility int + +// ParsePluginVisibility parses the PluginVisibility from the string. +func ParsePluginVisibility(s string) (PluginVisibility, error) { + switch s { + case "public": + return PluginVisibilityPublic, nil + case "private": + return PluginVisibilityPrivate, nil + default: + return 0, fmt.Errorf("unknown PluginVisibility: %q", s) + } +} diff --git a/private/bufpkg/bufplugin/uploader.go b/private/bufpkg/bufplugin/uploader.go new file mode 100644 index 0000000000..c785400c61 --- /dev/null +++ b/private/bufpkg/bufplugin/uploader.go @@ -0,0 +1,146 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/bufbuild/buf/private/pkg/slicesext" +) + +var ( + // NopUploader is a no-op Uploader. + NopUploader Uploader = nopUploader{} +) + +// Uploader uploads Plugins. +type Uploader interface { + // Upload uploads the given Plugins. + Upload(ctx context.Context, plugins []Plugin, options ...UploadOption) ([]Commit, error) +} + +// UploadOption is an option for an Upload. +type UploadOption func(*uploadOptions) + +// UploadWithLabels returns a new UploadOption that adds the given labels. +// +// This can be called multiple times. The unique result set of labels will be used. +func UploadWithLabels(labels ...string) UploadOption { + return func(uploadOptions *uploadOptions) { + uploadOptions.labels = append(uploadOptions.labels, labels...) + } +} + +// UploadWithCreateIfNotExist returns a new UploadOption that will result in the +// Plugins being created on the registry with the given visibility and default label if they do not exist. +func UploadWithCreateIfNotExist(createPluginVisibility PluginVisibility, createDefaultLabel string) UploadOption { + return func(uploadOptions *uploadOptions) { + uploadOptions.createIfNotExist = true + uploadOptions.createPluginVisibility = createPluginVisibility + } +} + +// UploadWithSourceControlURL returns a new UploadOption that will set the source control +// url for the module contents uploaded. +func UploadWithSourceControlURL(sourceControlURL string) UploadOption { + return func(uploadOptions *uploadOptions) { + uploadOptions.sourceControlURL = sourceControlURL + } +} + +// UploadOptions are the possible options for upload. +// +// This is used by Uploader implementations. +type UploadOptions interface { + // Labels returns the unique and sorted set of labels to add. Labels + // are set using the `--label` flag when calling `buf plugin upload` + // and represent the labels that are set when uploading plugin data. + Labels() []string + // CreateIfNotExist says to create Plugins if they do not exist on the registry. + CreateIfNotExist() bool + // CreatePluginVisibility returns the visibility to create Plugins with. + // + // Will always be present if CreateIfNotExist() is true. + CreatePluginVisibility() PluginVisibility + // SourceControlURL returns the source control URL set by the user for the module + // contents uploaded. We set the same source control URL for all module contents. + SourceControlURL() string + + isUploadOptions() +} + +// NewUploadOptions returns a new UploadOptions. +func NewUploadOptions(options []UploadOption) (UploadOptions, error) { + uploadOptions := newUploadOptions() + for _, option := range options { + option(uploadOptions) + } + if err := uploadOptions.validate(); err != nil { + return nil, err + } + return uploadOptions, nil +} + +// *** PRIVATE *** + +type nopUploader struct{} + +func (nopUploader) Upload(context.Context, []Plugin, ...UploadOption) ([]Commit, error) { + return nil, errors.New("unimplemented: no-op Uploader called") +} + +type uploadOptions struct { + labels []string + createIfNotExist bool + createPluginVisibility PluginVisibility + sourceControlURL string +} + +func newUploadOptions() *uploadOptions { + return &uploadOptions{} +} + +func (u *uploadOptions) Labels() []string { + return slicesext.ToUniqueSorted(u.labels) +} + +func (u *uploadOptions) CreateIfNotExist() bool { + return u.createIfNotExist +} + +func (u *uploadOptions) CreatePluginVisibility() PluginVisibility { + return u.createPluginVisibility +} + +func (u *uploadOptions) SourceControlURL() string { + return u.sourceControlURL +} + +func (u *uploadOptions) isUploadOptions() {} + +func (u *uploadOptions) validate() error { + if u.createIfNotExist && u.createPluginVisibility == 0 { + return errors.New("must set a valid PluginVisibility if CreateIfNotExist was specified") + } + if u.sourceControlURL != "" { + if _, err := url.Parse(u.sourceControlURL); err != nil { + return fmt.Errorf("must set a valid url for the source control url: %w", err) + } + } + return nil +} From 97934e2733f74126d4415f4774a837266c064a6f Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 26 Nov 2024 18:06:20 -0500 Subject: [PATCH 3/4] Fix docs --- .../bufpkg/bufplugin/bufpluginapi/uploader.go | 4 ++-- private/bufpkg/bufplugin/plugin.go | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/private/bufpkg/bufplugin/bufpluginapi/uploader.go b/private/bufpkg/bufplugin/bufpluginapi/uploader.go index fe12a2bee2..966603be43 100644 --- a/private/bufpkg/bufplugin/bufpluginapi/uploader.go +++ b/private/bufpkg/bufplugin/bufpluginapi/uploader.go @@ -181,7 +181,7 @@ func (u *uploader) uploadIndexedPluginsForRegistry( } pluginCommits := uploadResponse.Msg.Commits if len(pluginCommits) != len(indexedPlugins) { - return nil, syserror.New("did not get the expected number of plugin commits") + return nil, syserror.Newf("expected %d Commits, found %d", len(indexedPlugins), len(pluginCommits)) } indexedCommits := make([]slicesext.Indexed[bufplugin.Commit], 0, len(indexedPlugins)) @@ -301,7 +301,7 @@ func (u *uploader) validatePluginsExist( func zstdCompress(data []byte) ([]byte, error) { encoder, err := zstd.NewWriter(nil) if err != nil { - return nil, fmt.Errorf("failed to create zstd encoder: %w", err) + return nil, err } defer encoder.Close() return encoder.EncodeAll(data, nil), nil diff --git a/private/bufpkg/bufplugin/plugin.go b/private/bufpkg/bufplugin/plugin.go index fdc8b43cfc..438f52a60f 100644 --- a/private/bufpkg/bufplugin/plugin.go +++ b/private/bufpkg/bufplugin/plugin.go @@ -27,12 +27,6 @@ import ( // Plugin presents a BSR plugin. type Plugin interface { - // Data returns the bytes of the Plugin as a Wasm module. - // - // This is the raw bytes of the Wasm module in an uncompressed form. - // - // Will be nil if the plugin is not of type PluginTypeWasm. - Data() ([]byte, error) // OpaqueID returns an unstructured ID that can uniquely identify a Plugin // relative to the Workspace. // @@ -74,14 +68,21 @@ type Plugin interface { // files and content. A Digest is a specific algorithm applied to the // content of a Plugin. // - // Digest may return an error if the Plugin is not a Wasm plugin. + // Will return an error if the Plugin is not a Wasm Plugin. Digest(DigestType) (Digest, error) + // Data returns the bytes of the Plugin as a Wasm module. + // + // This is the raw bytes of the Wasm module in an uncompressed form. + // + // Will return an error if the Plugin is not a Wasm Plugin. + Data() ([]byte, error) // IsWasm returns true if the Plugin is a Wasm Plugin. // // Plugins are either Wasm or not Wasm. // // A Wasm Plugin is a Plugin that is a Wasm module. Wasm Plugins are invoked - // with a wasm.Runtime. + // with the wasm.Runtime. The Plugin will have Data and will be able to + // calculate Digests. // // Wasm Plugins will always have Data. IsWasm() bool From 854095a8985aa1e94834593b302333d6f17aa3fc Mon Sep 17 00:00:00 2001 From: Edward McFarlane Date: Tue, 26 Nov 2024 19:25:07 -0500 Subject: [PATCH 4/4] Fix plugin type on create --- .../command/plugin/pluginpush/pluginpush.go | 56 ++++++++++++++++++- .../bufpkg/bufplugin/bufpluginapi/convert.go | 11 +++- .../bufpkg/bufplugin/bufpluginapi/uploader.go | 11 +++- private/bufpkg/bufplugin/plugin_type.go | 46 +++++++++++++++ private/bufpkg/bufplugin/uploader.go | 14 ++++- 5 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 private/bufpkg/bufplugin/plugin_type.go diff --git a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go index a32fd2fdec..5a68efbe48 100644 --- a/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go +++ b/private/buf/cmd/buf/command/plugin/pluginpush/pluginpush.go @@ -26,6 +26,7 @@ import ( "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/spf13/pflag" ) @@ -35,6 +36,7 @@ const ( binaryFlagName = "binary" createFlagName = "create" createVisibilityFlagName = "create-visibility" + createTypeFlagName = "create-type" sourceControlURLFlagName = "source-control-url" ) @@ -63,6 +65,7 @@ type flags struct { Binary string Create bool CreateVisibility string + CreateType string SourceControlURL string } @@ -89,8 +92,19 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { createFlagName, false, fmt.Sprintf( - "Create the plugin if it does not exist. Defaults to creating a private repository if --%s is not set.", + "Create the plugin if it does not exist. Defaults to creating a private repository if --%s is not set. Must be used with --%s.", createVisibilityFlagName, + createTypeFlagName, + ), + ) + flagSet.StringVar( + &f.CreateType, + createTypeFlagName, + "", + fmt.Sprintf( + "The plugin's type setting, if created. Can only be set with --%s. Must be one of %s", + createTypeFlagName, + stringutil.SliceToString(bufplugin.AllPluginTypeStrings), ), ) flagSet.StringVar( @@ -156,7 +170,22 @@ func upload( if err != nil { return nil, err } - commits, err := uploader.Upload(ctx, []bufplugin.Plugin{plugin}) + var options []bufplugin.UploadOption + if flags.Create { + createPluginVisibility, err := bufplugin.ParsePluginVisibility(flags.CreateVisibility) + if err != nil { + return nil, err + } + createPluginType, err := bufplugin.ParsePluginType(flags.CreateType) + if err != nil { + return nil, err + } + options = append(options, bufplugin.UploadWithCreateIfNotExist( + createPluginVisibility, + createPluginType, + )) + } + commits, err := uploader.Upload(ctx, []bufplugin.Plugin{plugin}, options...) if err != nil { return nil, err } @@ -173,6 +202,9 @@ func validateFlags(flags *flags) error { if err := validateTypeFlags(flags); err != nil { return err } + if err := validateCreateFlags(flags); err != nil { + return err + } return nil } @@ -209,3 +241,23 @@ func validateLabelFlagValues(flags *flags) error { } return nil } + +func validateCreateFlags(flags *flags) error { + if flags.Create { + if flags.CreateVisibility == "" { + return appcmd.NewInvalidArgumentErrorf("--%s must be set if --%s is set", createVisibilityFlagName, createFlagName) + } + if _, err := bufplugin.ParsePluginVisibility(flags.CreateVisibility); err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + } + if flags.Create { + if flags.CreateType == "" { + return appcmd.NewInvalidArgumentErrorf("--%s must be set if --%s is set", createTypeFlagName, createFlagName) + } + if _, err := bufplugin.ParsePluginType(flags.CreateType); err != nil { + return appcmd.WrapInvalidArgumentError(err) + } + } + return nil +} diff --git a/private/bufpkg/bufplugin/bufpluginapi/convert.go b/private/bufpkg/bufplugin/bufpluginapi/convert.go index dcdd179ea7..8b15660e68 100644 --- a/private/bufpkg/bufplugin/bufpluginapi/convert.go +++ b/private/bufpkg/bufplugin/bufpluginapi/convert.go @@ -46,7 +46,7 @@ func V1Beta1ProtoToDigest(protoDigest *pluginv1beta1.Digest) (bufplugin.Digest, // *** PRIVATE *** -func pluginVisibilityToV1Proto(pluginVisibility bufplugin.PluginVisibility) (pluginv1beta1.PluginVisibility, error) { +func pluginVisibilityToV1Beta1Proto(pluginVisibility bufplugin.PluginVisibility) (pluginv1beta1.PluginVisibility, error) { switch pluginVisibility { case bufplugin.PluginVisibilityPublic: return pluginv1beta1.PluginVisibility_PLUGIN_VISIBILITY_PUBLIC, nil @@ -57,6 +57,15 @@ func pluginVisibilityToV1Proto(pluginVisibility bufplugin.PluginVisibility) (plu } } +func pluginTypeToV1Beta1Proto(pluginType bufplugin.PluginType) (pluginv1beta1.PluginType, error) { + switch pluginType { + case bufplugin.PluginTypeCheck: + return pluginv1beta1.PluginType_PLUGIN_TYPE_CHECK, nil + default: + return 0, fmt.Errorf("unknown PluginType: %v", pluginType) + } +} + func v1beta1ProtoToDigestType(protoDigestType pluginv1beta1.DigestType) (bufplugin.DigestType, error) { digestType, ok := v1beta1ProtoDigestTypeToDigestType[protoDigestType] if !ok { diff --git a/private/bufpkg/bufplugin/bufpluginapi/uploader.go b/private/bufpkg/bufplugin/bufpluginapi/uploader.go index 966603be43..f0900152e2 100644 --- a/private/bufpkg/bufplugin/bufpluginapi/uploader.go +++ b/private/bufpkg/bufplugin/bufpluginapi/uploader.go @@ -125,6 +125,7 @@ func (u *uploader) uploadIndexedPluginsForRegistry( registry, plugin, uploadOptions.CreatePluginVisibility(), + uploadOptions.CreatePluginType(), ); err != nil { return nil, err } @@ -223,8 +224,13 @@ func (u *uploader) createPluginIfNotExist( primaryRegistry string, plugin bufplugin.Plugin, createPluginVisibility bufplugin.PluginVisibility, + createPluginType bufplugin.PluginType, ) (*pluginv1beta1.Plugin, error) { - v1ProtoCreatePluginVisibility, err := pluginVisibilityToV1Proto(createPluginVisibility) + v1Beta1ProtoCreatePluginVisibility, err := pluginVisibilityToV1Beta1Proto(createPluginVisibility) + if err != nil { + return nil, err + } + v1Beta1ProtoCreatePluginType, err := pluginTypeToV1Beta1Proto(createPluginType) if err != nil { return nil, err } @@ -240,7 +246,8 @@ func (u *uploader) createPluginIfNotExist( }, }, Name: plugin.FullName().Name(), - Visibility: v1ProtoCreatePluginVisibility, + Visibility: v1Beta1ProtoCreatePluginVisibility, + Type: v1Beta1ProtoCreatePluginType, }, }, }, diff --git a/private/bufpkg/bufplugin/plugin_type.go b/private/bufpkg/bufplugin/plugin_type.go new file mode 100644 index 0000000000..2d06454ac4 --- /dev/null +++ b/private/bufpkg/bufplugin/plugin_type.go @@ -0,0 +1,46 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufplugin + +import ( + "fmt" +) + +const ( + // PluginTypeCheck says the Plugin is a check plugin. + PluginTypeCheck = iota + 1 +) + +var ( + // AllPluginTypeStrings is all format strings without aliases. + // + // Sorted in the order we want to display them. + AllPluginTypeStrings = []string{ + "check", + } +) + +// PluginType is the type of a Plugin. +type PluginType int + +// ParsePluginType parses the PluginType from the string. +func ParsePluginType(s string) (PluginType, error) { + switch s { + case "check": + return PluginVisibilityPublic, nil + default: + return 0, fmt.Errorf("unknown PluginType: %q", s) + } +} diff --git a/private/bufpkg/bufplugin/uploader.go b/private/bufpkg/bufplugin/uploader.go index c785400c61..c5d2d09b44 100644 --- a/private/bufpkg/bufplugin/uploader.go +++ b/private/bufpkg/bufplugin/uploader.go @@ -47,11 +47,12 @@ func UploadWithLabels(labels ...string) UploadOption { } // UploadWithCreateIfNotExist returns a new UploadOption that will result in the -// Plugins being created on the registry with the given visibility and default label if they do not exist. -func UploadWithCreateIfNotExist(createPluginVisibility PluginVisibility, createDefaultLabel string) UploadOption { +// Plugins being created on the registry with the given visibility if they do not exist. +func UploadWithCreateIfNotExist(createPluginVisibility PluginVisibility, createPluginType PluginType) UploadOption { return func(uploadOptions *uploadOptions) { uploadOptions.createIfNotExist = true uploadOptions.createPluginVisibility = createPluginVisibility + uploadOptions.createPluginType = createPluginType } } @@ -77,6 +78,10 @@ type UploadOptions interface { // // Will always be present if CreateIfNotExist() is true. CreatePluginVisibility() PluginVisibility + // CreatePluginType returns the type to create Plugins with. + // + // Will always be present if CreateIfNotExist() is true. + CreatePluginType() PluginType // SourceControlURL returns the source control URL set by the user for the module // contents uploaded. We set the same source control URL for all module contents. SourceControlURL() string @@ -108,6 +113,7 @@ type uploadOptions struct { labels []string createIfNotExist bool createPluginVisibility PluginVisibility + createPluginType PluginType sourceControlURL string } @@ -127,6 +133,10 @@ func (u *uploadOptions) CreatePluginVisibility() PluginVisibility { return u.createPluginVisibility } +func (u *uploadOptions) CreatePluginType() PluginType { + return u.createPluginType +} + func (u *uploadOptions) SourceControlURL() string { return u.sourceControlURL }