From 4f01250f6a773045678951e34ab22c6a567f6e69 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 21 Nov 2023 17:30:16 -0500 Subject: [PATCH 01/66] init --- private/bufnew/bufconfig/generate_config.go | 45 ++++------ .../bufconfig/generate_managed_config.go | 82 +++++++++++++++++++ .../bufconfig/generate_plugin_config.go | 76 +++++++++++++++++ 3 files changed, 176 insertions(+), 27 deletions(-) create mode 100644 private/bufnew/bufconfig/generate_managed_config.go create mode 100644 private/bufnew/bufconfig/generate_plugin_config.go diff --git a/private/bufnew/bufconfig/generate_config.go b/private/bufnew/bufconfig/generate_config.go index 4b5bd15667..6f3d8e87e3 100644 --- a/private/bufnew/bufconfig/generate_config.go +++ b/private/bufnew/bufconfig/generate_config.go @@ -18,44 +18,35 @@ package bufconfig // // TODO type GenerateConfig interface { + // GeneratePluginConfigs returns the plugin configurations. This will always be + // non-empty. Zero plugin configs will cause an error at construction time. GeneratePluginConfigs() []GeneratePluginConfig - // may be nil + // GenerateManagedConfig returns the managed mode configuration. This is never nil. + // If not defined by user, this config will return false from Enabled(). GenerateManagedConfig() GenerateManagedConfig - // may be empty - // will always be empty in v2 + // GenerateTypeConfig returns the types to generate code for. This overrides other type + // filters from input configurations, which exist in v2. + // This will always be nil in v2 + GenerateTypeConfig() GenerateTypeConfig // TODO: we may need a way to attach inputs to make this consistent, but // can deal with that for v2. - //GenerateInputConfigs() []GenerateInputConfig + // GenerateInputConfigs() []GenerateInputConfig - // may be nil - // will always be nil in v2 - GenerateTypeConfig() GenerateTypeConfig isGenerateConfig() } -type GeneratePluginConfig interface { - Plugin() string - Revision() int - Out() string - // TODO define enum in same pattern as FileVersion - // GenerateStrategy() GenerateStrategy - // TODO finish - // TODO: figure out what to do with TypesConfig - isGeneratePluginConfig() -} - -type GenerateManagedConfig interface { - // second value is whether or not this was present - CCEnableArenas() (bool, bool) - // TODO finish - isGenerateManagedConfig() +// GenerateInputConfig is an input configuration. +type GenerateInputConfig interface { + // TODO: implement this in v2 + isGenerateInputConfig() } -//type GenerateInputConfig interface { -//isGenerateInputConfig() -//} - +// GenerateTypeConfig is a type filter configuration. type GenerateTypeConfig interface { + // If IncludeTypes returns a non-empty list, it means that only those types are + // generated. Otherwise all types are generated. + IncludeTypes() []string + isGenerateTypeConfig() } diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go new file mode 100644 index 0000000000..910f3a6bc4 --- /dev/null +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -0,0 +1,82 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfig + +import ( + "github.com/bufbuild/buf/private/bufnew/bufmodule" + "google.golang.org/protobuf/types/descriptorpb" +) + +// GenerateManagedConfig is a managed mode configuration. +type GenerateManagedConfig interface { + // Enabled returns whether managed mode is enabled. + Enabled() bool + // The first return value is the value to modify the file's option to. For + // example, if a config has set java_package_prefix to foo for a file with + // package bar, this returns "foo.bar" for the first value. + // The second value indicates whether this option should be modified. + // Reasons not to modify: + // 1. Some options are not supposed to be modified unless a value is specified, + // such as optimize_for. + // 2. In the config the user has specified not to modify this option for this file. + // 3. The file already has the desired file option value. + // 4. The file is a WKT. + // TODO: 3 and 4 are debatable, asking the question what is the responsibility + // of a managed config. + // + // Note: this means a GenerateManagedConfig's interface does not deal with + // options like java_package_prefix and java_package_suffix, because it returns + // the final value to modify java_package to. + ValueForFileOption(imageFile, fileOption) (interface{}, bool) + + isGenerateManagedConfig() +} + +// bufimage.ImageFile implements this, but this also allows private implementation +// for testing. +type imageFile interface { + ModuleFullName() bufmodule.ModuleFullName + Path() string + FileDescriptorProto() *descriptorpb.FileDescriptorProto +} + +type fileOption int + +const ( + // fileOptionJavaPackage is the file option java_package. + fileOptionJavaPackage fileOption = iota + 1 + // fileOptionJavaOuterClassname is the file option java_outer_classname. + fileOptionJavaOuterClassname + // fileOptionJavaMultipleFiles is the file option java_multiple_files. + fileOptionJavaMultipleFiles + // fileOptionJavaStringCheckUtf8 is the file option java_string_check_utf8. + fileOptionJavaStringCheckUtf8 + // fileOptionOptimizeFor is the file option optimize_for. + fileOptionOptimizeFor + // fileOptionGoPackage is the file option go_package. + fileOptionGoPackage + // fileOptionCcEnableArenas is the file option cc_enable_arenas. + fileOptionCcEnableArenas + // fileOptionObjcClassPrefix is the file option objc_class_prefix. + fileOptionObjcClassPrefix + // fileOptionCsharpNamespace is the file option csharp_namespace. + fileOptionCsharpNamespace + // fileOptionPhpNamespace is the file option php_namespace. + fileOptionPhpNamespace + // fileOptionPhpMetadataNamespace is the file option php_metadata_namespace. + fileOptionPhpMetadataNamespace + // fileOptionRubyPackage is the file option ruby_package. + fileOptionRubyPackage +) diff --git a/private/bufnew/bufconfig/generate_plugin_config.go b/private/bufnew/bufconfig/generate_plugin_config.go new file mode 100644 index 0000000000..cadd3ec99c --- /dev/null +++ b/private/bufnew/bufconfig/generate_plugin_config.go @@ -0,0 +1,76 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfig + +// TODO: should this type live here? Perhaps it should live in the package that handles generation? +// GenerateStrategy is the generation strategy for a protoc plugin. +type GenerateStrategy int + +const ( + // GenerateStrategyDirectory is the strategy to generate per directory. + // + // This is the default value for local plugins. + GenerateStrategyDirectory GenerateStrategy = 1 + // GenerateStrategyAll is the strategy to generate with all files at once. + // + // This is the only strategy for remote plugins. + GenerateStrategyAll GenerateStrategy = 2 +) + +// GeneratePluginConfig is a configuration for a plugin. +type GeneratePluginConfig interface { + Name() string + Out() string + Opt() string + IncludeImports() bool + IncludeWKT() bool + + isPluginConfig() +} + +// LocalPluginConfig is a plugin configuration for a local plugin. It maybe a +// binary or protoc builtin plugin, but its type is undetermined. We defer to +// the plugin executor to decide which type it is. +type LocalPluginConfig interface { + GeneratePluginConfig + Strategy() GenerateStrategy + + isLocalPluginConfig() +} + +// BinaryPluginConfig is a binary plugin configuration. +type BinaryPluginConfig interface { + LocalPluginConfig + Path() []string + + isBinaryPluginConfig() +} + +// ProtocBuiltinPluginConfig is a protoc builtin plugin configuration. +type ProtocBuiltinPluginConfig interface { + LocalPluginConfig + ProtocPath() string + + isProtocBuiltinPluginConfig() +} + +// RemotePluginConfig is a remote plugin configuration. +type RemotePluginConfig interface { + GeneratePluginConfig + RemoteHost() string + Revision() int + + isRemotePluginConfig() +} From dc09dabcef6dee870c6339b95c6f3a2b876a3105 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 21 Nov 2023 17:31:47 -0500 Subject: [PATCH 02/66] commit --- private/bufnew/bufconfig/generate_config.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/private/bufnew/bufconfig/generate_config.go b/private/bufnew/bufconfig/generate_config.go index 6f3d8e87e3..968e5de8ab 100644 --- a/private/bufnew/bufconfig/generate_config.go +++ b/private/bufnew/bufconfig/generate_config.go @@ -15,8 +15,6 @@ package bufconfig // GenerateConfig is a generation configuration. -// -// TODO type GenerateConfig interface { // GeneratePluginConfigs returns the plugin configurations. This will always be // non-empty. Zero plugin configs will cause an error at construction time. From d80be026d33acfbddc365e4d09966b08c8ed8d16 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 21 Nov 2023 20:04:38 -0500 Subject: [PATCH 03/66] update input config and managed config --- private/bufnew/bufconfig/generate_config.go | 15 +- .../bufnew/bufconfig/generate_input_config.go | 32 ++++ .../bufconfig/generate_managed_config.go | 154 +++++++++++++----- 3 files changed, 145 insertions(+), 56 deletions(-) create mode 100644 private/bufnew/bufconfig/generate_input_config.go diff --git a/private/bufnew/bufconfig/generate_config.go b/private/bufnew/bufconfig/generate_config.go index 968e5de8ab..431fb5083d 100644 --- a/private/bufnew/bufconfig/generate_config.go +++ b/private/bufnew/bufconfig/generate_config.go @@ -19,26 +19,19 @@ type GenerateConfig interface { // GeneratePluginConfigs returns the plugin configurations. This will always be // non-empty. Zero plugin configs will cause an error at construction time. GeneratePluginConfigs() []GeneratePluginConfig - // GenerateManagedConfig returns the managed mode configuration. This is never nil. - // If not defined by user, this config will return false from Enabled(). + // GenerateManagedConfig returns the managed mode configuration. + // This may be nil. GenerateManagedConfig() GenerateManagedConfig // GenerateTypeConfig returns the types to generate code for. This overrides other type // filters from input configurations, which exist in v2. // This will always be nil in v2 GenerateTypeConfig() GenerateTypeConfig - // TODO: we may need a way to attach inputs to make this consistent, but - // can deal with that for v2. - // GenerateInputConfigs() []GenerateInputConfig + // GenerateInputConfigs returns the input config. + GenerateInputConfigs() []GenerateInputConfig isGenerateConfig() } -// GenerateInputConfig is an input configuration. -type GenerateInputConfig interface { - // TODO: implement this in v2 - isGenerateInputConfig() -} - // GenerateTypeConfig is a type filter configuration. type GenerateTypeConfig interface { // If IncludeTypes returns a non-empty list, it means that only those types are diff --git a/private/bufnew/bufconfig/generate_input_config.go b/private/bufnew/bufconfig/generate_input_config.go new file mode 100644 index 0000000000..14a08cf2a8 --- /dev/null +++ b/private/bufnew/bufconfig/generate_input_config.go @@ -0,0 +1,32 @@ +// GenerateInputConfig is an input configuration. +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfig + +import "github.com/bufbuild/buf/private/buf/buffetch" + +type GenerateInputConfig interface { + // Ref returns the input ref. + Ref() buffetch.Ref + // Types returns the types to generate. If GenerateConfig.GenerateTypeConfig() + // returns a non-empty list of types. + Types() []string + // ExcludePaths returns paths not to generate for. + ExcludePaths() []string + // IncludePaths returns paths to generate for. + IncludePaths() []string + + isGenerateInputConfig() +} diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 910f3a6bc4..4ad9ac79fc 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -21,62 +21,126 @@ import ( // GenerateManagedConfig is a managed mode configuration. type GenerateManagedConfig interface { - // Enabled returns whether managed mode is enabled. - Enabled() bool - // The first return value is the value to modify the file's option to. For - // example, if a config has set java_package_prefix to foo for a file with - // package bar, this returns "foo.bar" for the first value. - // The second value indicates whether this option should be modified. - // Reasons not to modify: - // 1. Some options are not supposed to be modified unless a value is specified, - // such as optimize_for. - // 2. In the config the user has specified not to modify this option for this file. - // 3. The file already has the desired file option value. - // 4. The file is a WKT. - // TODO: 3 and 4 are debatable, asking the question what is the responsibility - // of a managed config. - // - // Note: this means a GenerateManagedConfig's interface does not deal with - // options like java_package_prefix and java_package_suffix, because it returns - // the final value to modify java_package to. - ValueForFileOption(imageFile, fileOption) (interface{}, bool) + // Disables returns the disable rules in the configuration. + Disables() []ManagedDisableRule + // Overrides returns the override rules in the configuration. + Overrides() []ManagedOverrideRule isGenerateManagedConfig() } +// ManagedDisableRule is a disable rule. A disable rule describes: +// +// - The options to not modify. If not specified, it means all options (both +// file options and field options) are not modified. +// - The files/fields for which these options are not modified. If not specified, +// it means for all files/fields the specified options are not modified. +// +// A ManagedDisableRule is guaranteed to specify at least one of the two aspects. +// i.e. At least one of Path, ModuleFullNameString, FieldName, FileOption and +// FieldOption is not empty. A rule can disable all options for certain files/fields, +// disable certains options for all files/fields, or disable certain options for +// certain files/fields. To disable all options for all files/fields, turn off managed mode. +type ManagedDisableRule interface { + // Path returns the file path, relative to its module, to disable managed mode for. + Path() string + // ModuleFullNameString returns the full name string of the module to disable + // managed mode for. + ModuleFullNameString() string + // FieldName returns the fully qualified name for the field to disable managed + // mode for. This is guaranteed to be empty if FileOption is not empty. + FieldName() string + // FileOption returns the file option to disable managed mode for. This is + // guaranteed to be empty if FieldName is not empty. + FileOption() FileOption + // FieldOption returns the field option to disalbe managed mode for. + FieldOption() FieldOption + + isManagedDisable() +} + +// ManagedOverrideRule is an override rule. An override describes: +// +// - The options to modify. Exactly one of FileOption and FieldOption is not empty. +// - The value, prefix or suffix to modify these options with. Exactly one of +// Value, Prefix and Suffix is not empty. +// - The files/fields for which the options are modified. If all of Path, ModuleFullNameString +// - or FieldName are empty, all files/fields are modified. Otherwise, only +// file/fields that match the specified Path, ModuleFullNameString and FieldName +// is modified. +type ManagedOverrideRule interface { + // Path is the file path, relative to its module, to disable managed mode for. + Path() string + // ModuleFullNameString is the full name string of the module to disable + // managed mode for. + ModuleFullNameString() string + // FieldName is the fully qualified name for the field to disable managed + // mode for. This is guranteed to be empty is FileOption is not empty. + FieldName() string + // FileOption returns the file option to disable managed mode for. This is + // guaranteed to be empty (FileOptionUnspecified) if FieldName is empty. + FileOption() FileOption + // FieldOption returns the field option to disable managed mode for. + FieldOption() FieldOption + // Value returns the override value. + Value() interface{} + // Prefix returns the override prefix. + Prefix() string + // Suffix returns the override suffix. + Suffix() string + + isManagedOverride() +} + // bufimage.ImageFile implements this, but this also allows private implementation // for testing. -type imageFile interface { +type ImageFile interface { + // ModuleFullName returns its module's full name ModuleFullName() bufmodule.ModuleFullName + // Path returns the path relative to the its module's root. Path() string + // FileDescriptorProto returns the its field descriptor. FileDescriptorProto() *descriptorpb.FileDescriptorProto } -type fileOption int +// FileOption is a file option. +type FileOption int + +const ( + // FileOptionUnspecified is an unspecified file option. + FileOptionUnspecified FileOption = iota + // FileOptionJavaPackage is the file option java_package. + FileOptionJavaPackage + // FileOptionJavaOuterClassname is the file option java_outer_classname. + FileOptionJavaOuterClassname + // FileOptionJavaMultipleFiles is the file option java_multiple_files. + FileOptionJavaMultipleFiles + // FileOptionJavaStringCheckUtf8 is the file option java_string_check_utf8. + FileOptionJavaStringCheckUtf8 + // FileOptionOptimizeFor is the file option optimize_for. + FileOptionOptimizeFor + // FileOptionGoPackage is the file option go_package. + FileOptionGoPackage + // FileOptionCcEnableArenas is the file option cc_enable_arenas. + FileOptionCcEnableArenas + // FileOptionObjcClassPrefix is the file option objc_class_prefix. + FileOptionObjcClassPrefix + // FileOptionCsharpNamespace is the file option csharp_namespace. + FileOptionCsharpNamespace + // FileOptionPhpNamespace is the file option php_namespace. + FileOptionPhpNamespace + // FileOptionPhpMetadataNamespace is the file option php_metadata_namespace. + FileOptionPhpMetadataNamespace + // FileOptionRubyPackage is the file option ruby_package. + FileOptionRubyPackage +) + +// FieldOption is a field option. +type FieldOption int const ( - // fileOptionJavaPackage is the file option java_package. - fileOptionJavaPackage fileOption = iota + 1 - // fileOptionJavaOuterClassname is the file option java_outer_classname. - fileOptionJavaOuterClassname - // fileOptionJavaMultipleFiles is the file option java_multiple_files. - fileOptionJavaMultipleFiles - // fileOptionJavaStringCheckUtf8 is the file option java_string_check_utf8. - fileOptionJavaStringCheckUtf8 - // fileOptionOptimizeFor is the file option optimize_for. - fileOptionOptimizeFor - // fileOptionGoPackage is the file option go_package. - fileOptionGoPackage - // fileOptionCcEnableArenas is the file option cc_enable_arenas. - fileOptionCcEnableArenas - // fileOptionObjcClassPrefix is the file option objc_class_prefix. - fileOptionObjcClassPrefix - // fileOptionCsharpNamespace is the file option csharp_namespace. - fileOptionCsharpNamespace - // fileOptionPhpNamespace is the file option php_namespace. - fileOptionPhpNamespace - // fileOptionPhpMetadataNamespace is the file option php_metadata_namespace. - fileOptionPhpMetadataNamespace - // fileOptionRubyPackage is the file option ruby_package. - fileOptionRubyPackage + // FieldOptionUnspecified is an unspecified field option. + FieldOptionUnspecified FieldOption = iota + // FieldOptionJSType is the field option js_type. + FieldOptionJSType ) From 0715e66b57ed8650e6f38998a4afe7a659c1a12e Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 10:34:35 -0500 Subject: [PATCH 04/66] ModuleFullNameString -> ModuleFullName --- .../bufnew/bufconfig/generate_managed_config.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 4ad9ac79fc..46ba91b660 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -37,16 +37,16 @@ type GenerateManagedConfig interface { // it means for all files/fields the specified options are not modified. // // A ManagedDisableRule is guaranteed to specify at least one of the two aspects. -// i.e. At least one of Path, ModuleFullNameString, FieldName, FileOption and +// i.e. At least one of Path, ModuleFullName, FieldName, FileOption and // FieldOption is not empty. A rule can disable all options for certain files/fields, // disable certains options for all files/fields, or disable certain options for // certain files/fields. To disable all options for all files/fields, turn off managed mode. type ManagedDisableRule interface { // Path returns the file path, relative to its module, to disable managed mode for. Path() string - // ModuleFullNameString returns the full name string of the module to disable + // ModuleFullName returns the full name string of the module to disable // managed mode for. - ModuleFullNameString() string + ModuleFullName() string // FieldName returns the fully qualified name for the field to disable managed // mode for. This is guaranteed to be empty if FileOption is not empty. FieldName() string @@ -64,16 +64,16 @@ type ManagedDisableRule interface { // - The options to modify. Exactly one of FileOption and FieldOption is not empty. // - The value, prefix or suffix to modify these options with. Exactly one of // Value, Prefix and Suffix is not empty. -// - The files/fields for which the options are modified. If all of Path, ModuleFullNameString +// - The files/fields for which the options are modified. If all of Path, ModuleFullName // - or FieldName are empty, all files/fields are modified. Otherwise, only -// file/fields that match the specified Path, ModuleFullNameString and FieldName +// file/fields that match the specified Path, ModuleFullName and FieldName // is modified. type ManagedOverrideRule interface { // Path is the file path, relative to its module, to disable managed mode for. Path() string - // ModuleFullNameString is the full name string of the module to disable + // ModuleFullName is the full name string of the module to disable // managed mode for. - ModuleFullNameString() string + ModuleFullName() string // FieldName is the fully qualified name for the field to disable managed // mode for. This is guranteed to be empty is FileOption is not empty. FieldName() string From 7b50b7943c21aac7a2fb97092325e22748b7bf44 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 10:35:27 -0500 Subject: [PATCH 05/66] remove ImageFile interface --- .../bufnew/bufconfig/generate_managed_config.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 46ba91b660..e20aa9252d 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -14,11 +14,6 @@ package bufconfig -import ( - "github.com/bufbuild/buf/private/bufnew/bufmodule" - "google.golang.org/protobuf/types/descriptorpb" -) - // GenerateManagedConfig is a managed mode configuration. type GenerateManagedConfig interface { // Disables returns the disable rules in the configuration. @@ -92,17 +87,6 @@ type ManagedOverrideRule interface { isManagedOverride() } -// bufimage.ImageFile implements this, but this also allows private implementation -// for testing. -type ImageFile interface { - // ModuleFullName returns its module's full name - ModuleFullName() bufmodule.ModuleFullName - // Path returns the path relative to the its module's root. - Path() string - // FileDescriptorProto returns the its field descriptor. - FileDescriptorProto() *descriptorpb.FileDescriptorProto -} - // FileOption is a file option. type FileOption int From 99b374156a497ef13e2e63b720d1bd3bb89b4e69 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 10:38:43 -0500 Subject: [PATCH 06/66] move const to the top --- .../bufconfig/generate_managed_config.go | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index e20aa9252d..f0d91a9d73 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -14,6 +14,48 @@ package bufconfig +// FileOption is a file option. +type FileOption int + +const ( + // FileOptionUnspecified is an unspecified file option. + FileOptionUnspecified FileOption = iota + // FileOptionJavaPackage is the file option java_package. + FileOptionJavaPackage + // FileOptionJavaOuterClassname is the file option java_outer_classname. + FileOptionJavaOuterClassname + // FileOptionJavaMultipleFiles is the file option java_multiple_files. + FileOptionJavaMultipleFiles + // FileOptionJavaStringCheckUtf8 is the file option java_string_check_utf8. + FileOptionJavaStringCheckUtf8 + // FileOptionOptimizeFor is the file option optimize_for. + FileOptionOptimizeFor + // FileOptionGoPackage is the file option go_package. + FileOptionGoPackage + // FileOptionCcEnableArenas is the file option cc_enable_arenas. + FileOptionCcEnableArenas + // FileOptionObjcClassPrefix is the file option objc_class_prefix. + FileOptionObjcClassPrefix + // FileOptionCsharpNamespace is the file option csharp_namespace. + FileOptionCsharpNamespace + // FileOptionPhpNamespace is the file option php_namespace. + FileOptionPhpNamespace + // FileOptionPhpMetadataNamespace is the file option php_metadata_namespace. + FileOptionPhpMetadataNamespace + // FileOptionRubyPackage is the file option ruby_package. + FileOptionRubyPackage +) + +// FieldOption is a field option. +type FieldOption int + +const ( + // FieldOptionUnspecified is an unspecified field option. + FieldOptionUnspecified FieldOption = iota + // FieldOptionJSType is the field option js_type. + FieldOptionJSType +) + // GenerateManagedConfig is a managed mode configuration. type GenerateManagedConfig interface { // Disables returns the disable rules in the configuration. @@ -86,45 +128,3 @@ type ManagedOverrideRule interface { isManagedOverride() } - -// FileOption is a file option. -type FileOption int - -const ( - // FileOptionUnspecified is an unspecified file option. - FileOptionUnspecified FileOption = iota - // FileOptionJavaPackage is the file option java_package. - FileOptionJavaPackage - // FileOptionJavaOuterClassname is the file option java_outer_classname. - FileOptionJavaOuterClassname - // FileOptionJavaMultipleFiles is the file option java_multiple_files. - FileOptionJavaMultipleFiles - // FileOptionJavaStringCheckUtf8 is the file option java_string_check_utf8. - FileOptionJavaStringCheckUtf8 - // FileOptionOptimizeFor is the file option optimize_for. - FileOptionOptimizeFor - // FileOptionGoPackage is the file option go_package. - FileOptionGoPackage - // FileOptionCcEnableArenas is the file option cc_enable_arenas. - FileOptionCcEnableArenas - // FileOptionObjcClassPrefix is the file option objc_class_prefix. - FileOptionObjcClassPrefix - // FileOptionCsharpNamespace is the file option csharp_namespace. - FileOptionCsharpNamespace - // FileOptionPhpNamespace is the file option php_namespace. - FileOptionPhpNamespace - // FileOptionPhpMetadataNamespace is the file option php_metadata_namespace. - FileOptionPhpMetadataNamespace - // FileOptionRubyPackage is the file option ruby_package. - FileOptionRubyPackage -) - -// FieldOption is a field option. -type FieldOption int - -const ( - // FieldOptionUnspecified is an unspecified field option. - FieldOptionUnspecified FieldOption = iota - // FieldOptionJSType is the field option js_type. - FieldOptionJSType -) From 175a45ace0b25be63598c485efde0a63134e2999 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 11:02:48 -0500 Subject: [PATCH 07/66] remove type switch from plugin config --- .../bufconfig/generate_plugin_config.go | 77 +++++++++++-------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/private/bufnew/bufconfig/generate_plugin_config.go b/private/bufnew/bufconfig/generate_plugin_config.go index cadd3ec99c..47a1b3b0ab 100644 --- a/private/bufnew/bufconfig/generate_plugin_config.go +++ b/private/bufnew/bufconfig/generate_plugin_config.go @@ -14,8 +14,10 @@ package bufconfig -// TODO: should this type live here? Perhaps it should live in the package that handles generation? // GenerateStrategy is the generation strategy for a protoc plugin. +// +// TODO: Should this type live in this package? Perhaps it should live in the package that handles generation? +// TODO: The same question can be asked for FieldOption and FileOption. type GenerateStrategy int const ( @@ -29,48 +31,59 @@ const ( GenerateStrategyAll GenerateStrategy = 2 ) +type PluginConfigType int + +const ( + // PluginConfigTypeRemote is the remote plugin config type. + PluginConfigTypeRemote PluginConfigType = iota + 1 + // PluginConfigTypeBinary is the binary plugin config type. + PluginConfigTypeBinary + // PluginConfigTypeProtocBuiltin is the protoc built-in plugin config type. + PluginConfigTypeProtocBuiltin + // PluginConfigTypeLocal is the local plugin config type. This type indicates + // it is to be determined whether the plugin is binary or protoc built-in. + // We defer further classification to the plugin executor. In v2 the exact + // plugin config type is always specified and it will never be just local. + PluginConfigTypeLocal +) + // GeneratePluginConfig is a configuration for a plugin. type GeneratePluginConfig interface { + // Type returns the plugin type. This is never the zero value. + Type() PluginConfigType + // Name returns the plugin name. This is never empty. Name() string + // Out returns the output directory for generation. This is never empty. Out() string + // Opt returns the plugin options as a comma seperated string. Opt() string + // IncludeImports returns whether to generate code for imported files. This + // is always false in v1. IncludeImports() bool + // IncludeWKT returns whether to generate code for the well-known types. + // This returns true only if IncludeImports returns true. This is always + // false in v1. IncludeWKT() bool - - isPluginConfig() -} - -// LocalPluginConfig is a plugin configuration for a local plugin. It maybe a -// binary or protoc builtin plugin, but its type is undetermined. We defer to -// the plugin executor to decide which type it is. -type LocalPluginConfig interface { - GeneratePluginConfig + // Strategy returns the generation strategy. + // + // This is not empty only when the plugin is local, binary or protoc builtin. Strategy() GenerateStrategy - - isLocalPluginConfig() -} - -// BinaryPluginConfig is a binary plugin configuration. -type BinaryPluginConfig interface { - LocalPluginConfig + // Path returns the path, including arguments, to invoke the binary plugin. + // + // This is not empty only when the plugin is binary. Path() []string - - isBinaryPluginConfig() -} - -// ProtocBuiltinPluginConfig is a protoc builtin plugin configuration. -type ProtocBuiltinPluginConfig interface { - LocalPluginConfig + // ProtocPath returns a path to protoc. + // + // This is not empty only when the plugin is protoc-builtin ProtocPath() string - - isProtocBuiltinPluginConfig() -} - -// RemotePluginConfig is a remote plugin configuration. -type RemotePluginConfig interface { - GeneratePluginConfig + // RemoteHost returns the remote host of the remote plugin. + // + // This is not empty only when the plugin is remote. RemoteHost() string + // Revision returns the revision of the remote plugin. + // + // This is not empty only when the plugin is remote. Revision() int - isRemotePluginConfig() + isPluginConfig() } From 4d6b7bb91d16f1ec9c5010ef9bc2ad54df583a04 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 11:05:41 -0500 Subject: [PATCH 08/66] godoc --- private/bufnew/bufconfig/generate_input_config.go | 1 + private/bufnew/bufconfig/generate_plugin_config.go | 1 + 2 files changed, 2 insertions(+) diff --git a/private/bufnew/bufconfig/generate_input_config.go b/private/bufnew/bufconfig/generate_input_config.go index 14a08cf2a8..b504fa0f2b 100644 --- a/private/bufnew/bufconfig/generate_input_config.go +++ b/private/bufnew/bufconfig/generate_input_config.go @@ -17,6 +17,7 @@ package bufconfig import "github.com/bufbuild/buf/private/buf/buffetch" +// GenerateInputConfig is an input configuration for code generation. type GenerateInputConfig interface { // Ref returns the input ref. Ref() buffetch.Ref diff --git a/private/bufnew/bufconfig/generate_plugin_config.go b/private/bufnew/bufconfig/generate_plugin_config.go index 47a1b3b0ab..361c8d2db1 100644 --- a/private/bufnew/bufconfig/generate_plugin_config.go +++ b/private/bufnew/bufconfig/generate_plugin_config.go @@ -31,6 +31,7 @@ const ( GenerateStrategyAll GenerateStrategy = 2 ) +// PluginConfigType is a plugin configuration type. type PluginConfigType int const ( From 48c2747e36b74787e0d60958c2dcd7f090b992af Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 12:03:35 -0500 Subject: [PATCH 09/66] add some concrete types --- .../bufconfig/generate_external_config.go | 256 +++++++++++++++ .../bufconfig/generate_managed_config.go | 54 ++++ .../bufconfig/generate_plugin_config.go | 300 ++++++++++++++++++ 3 files changed, 610 insertions(+) create mode 100644 private/bufnew/bufconfig/generate_external_config.go diff --git a/private/bufnew/bufconfig/generate_external_config.go b/private/bufnew/bufconfig/generate_external_config.go new file mode 100644 index 0000000000..4b879e8fff --- /dev/null +++ b/private/bufnew/bufconfig/generate_external_config.go @@ -0,0 +1,256 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfig + +import "encoding/json" + +// TODO: this is a temporary file to avoid crowing other files. We can choose to move stuff from this file over. +// TODO: this is also completely copied over from bufgen.go, the only change made to it so far is unexporting the type. + +// externalGenerateConfigV1 is a v1 external generate configuration. +type externalGenerateConfigV1 struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Plugins []externalGeneratePluginConfigV1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Managed externalGenerateManagedConfigV1 `json:"managed,omitempty" yaml:"managed,omitempty"` + Types externalTypesConfigV1 `json:"types,omitempty" yaml:"types,omitempty"` +} + +// externalGeneratePluginConfigV1 is an external plugin configuration. +type externalGeneratePluginConfigV1 struct { + Plugin string `json:"plugin,omitempty" yaml:"plugin,omitempty"` + Revision int `json:"revision,omitempty" yaml:"revision,omitempty"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Remote string `json:"remote,omitempty" yaml:"remote,omitempty"` + Out string `json:"out,omitempty" yaml:"out,omitempty"` + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + Path interface{} `json:"path,omitempty" yaml:"path,omitempty"` + ProtocPath string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalGenerateManagedConfigV1 is an external managed mode configuration. +// +// Only use outside of this package for testing. +type externalGenerateManagedConfigV1 struct { + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` + JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` + JavaStringCheckUtf8 *bool `json:"java_string_check_utf8,omitempty" yaml:"java_string_check_utf8,omitempty"` + JavaPackagePrefix externalJavaPackagePrefixConfigV1 `json:"java_package_prefix,omitempty" yaml:"java_package_prefix,omitempty"` + CsharpNamespace externalCsharpNamespaceConfigV1 `json:"csharp_namespace,omitempty" yaml:"csharp_namespace,omitempty"` + OptimizeFor externalOptimizeForConfigV1 `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` + GoPackagePrefix externalGoPackagePrefixConfigV1 `json:"go_package_prefix,omitempty" yaml:"go_package_prefix,omitempty"` + ObjcClassPrefix externalObjcClassPrefixConfigV1 `json:"objc_class_prefix,omitempty" yaml:"objc_class_prefix,omitempty"` + RubyPackage externalRubyPackageConfigV1 `json:"ruby_package,omitempty" yaml:"ruby_package,omitempty"` + Override map[string]map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty, excluding the 'Enabled' setting. +func (e externalGenerateManagedConfigV1) isEmpty() bool { + return e.CcEnableArenas == nil && + e.JavaMultipleFiles == nil && + e.JavaStringCheckUtf8 == nil && + e.JavaPackagePrefix.isEmpty() && + e.CsharpNamespace.isEmpty() && + e.CsharpNamespace.isEmpty() && + e.OptimizeFor.isEmpty() && + e.GoPackagePrefix.isEmpty() && + e.ObjcClassPrefix.isEmpty() && + e.RubyPackage.isEmpty() && + len(e.Override) == 0 +} + +// externalJavaPackagePrefixConfigV1 is the external java_package prefix configuration. +type externalJavaPackagePrefixConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty. +func (e externalJavaPackagePrefixConfigV1) isEmpty() bool { + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// UnmarshalYAML satisfies the yaml.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for java_package_prefix. +func (e *externalJavaPackagePrefixConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { + return e.unmarshalWith(unmarshal) +} + +// UnmarshalJSON satisfies the json.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for java_package_prefix. +func (e *externalJavaPackagePrefixConfigV1) UnmarshalJSON(data []byte) error { + unmarshal := func(v interface{}) error { + return json.Unmarshal(data, v) + } + + return e.unmarshalWith(unmarshal) +} + +// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. +func (e *externalJavaPackagePrefixConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { + var prefix string + if err := unmarshal(&prefix); err == nil { + e.Default = prefix + return nil + } + + type rawExternalJavaPackagePrefixConfigV1 externalJavaPackagePrefixConfigV1 + if err := unmarshal((*rawExternalJavaPackagePrefixConfigV1)(e)); err != nil { + return err + } + + return nil +} + +// externalOptimizeForConfigV1 is the external optimize_for configuration. +type externalOptimizeForConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty +func (e externalOptimizeForConfigV1) isEmpty() bool { // TODO: does it need to be public? + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// UnmarshalYAML satisfies the yaml.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for optimize_for. +func (e *externalOptimizeForConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { + return e.unmarshalWith(unmarshal) +} + +// UnmarshalJSON satisfies the json.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for optimize_for. +func (e *externalOptimizeForConfigV1) UnmarshalJSON(data []byte) error { + unmarshal := func(v interface{}) error { + return json.Unmarshal(data, v) + } + + return e.unmarshalWith(unmarshal) +} + +// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. +func (e *externalOptimizeForConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { + var optimizeFor string + if err := unmarshal(&optimizeFor); err == nil { + e.Default = optimizeFor + return nil + } + + type rawExternalOptimizeForConfigV1 externalOptimizeForConfigV1 + if err := unmarshal((*rawExternalOptimizeForConfigV1)(e)); err != nil { + return err + } + + return nil +} + +// externalGoPackagePrefixConfigV1 is the external go_package prefix configuration. +type externalGoPackagePrefixConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty. +func (e externalGoPackagePrefixConfigV1) isEmpty() bool { + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalCsharpNamespaceConfigV1 is the external csharp_namespace configuration. +type externalCsharpNamespaceConfigV1 struct { + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty. +func (e externalCsharpNamespaceConfigV1) isEmpty() bool { + return len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalRubyPackageConfigV1 is the external ruby_package configuration +type externalRubyPackageConfigV1 struct { + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true is the config is empty +func (e externalRubyPackageConfigV1) isEmpty() bool { // TODO: does this need to be public? same with other IsEmpty() + return len(e.Except) == 0 && len(e.Override) == 0 +} + +// externalObjcClassPrefixConfigV1 is the external objc_class_prefix configuration. +type externalObjcClassPrefixConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +func (e externalObjcClassPrefixConfigV1) isEmpty() bool { + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalConfigV1Beta1 is a v1 external generate configuration. +type externalConfigV1Beta1 struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Managed bool `json:"managed,omitempty" yaml:"managed,omitempty"` + Plugins []externalGeneratePluginConfigV1Beta1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Options externalOptionsConfigV1Beta1 `json:"options,omitempty" yaml:"options,omitempty"` +} + +// externalGeneratePluginConfigV1Beta1 is an external plugin configuration. +type externalGeneratePluginConfigV1Beta1 struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Out string `json:"out,omitempty" yaml:"out,omitempty"` + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + Path string `json:"path,omitempty" yaml:"path,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalOptionsConfigV1Beta1 is an external options configuration. +type externalOptionsConfigV1Beta1 struct { + CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` + JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` + OptimizeFor string `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` +} + +// TODO: remove this +// externalConfigVersion defines the subset of all config +// file versions that is used to determine the configuration version. +type externalConfigVersion struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` +} + +// externalTypesConfigV1 is an external types configuration. +type externalTypesConfigV1 struct { + Include []string `json:"include,omitempty" yaml:"include"` +} + +// isEmpty returns true if e is empty. +func (e externalTypesConfigV1) isEmpty() bool { + return len(e.Include) == 0 +} diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index f0d91a9d73..5b6030dcd5 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -128,3 +128,57 @@ type ManagedOverrideRule interface { isManagedOverride() } + +type generateManagedConfig struct { + disables []ManagedDisableRule + overrides []ManagedOverrideRule +} + +func (g *generateManagedConfig) Disables() []ManagedDisableRule { + return g.disables +} + +func (g *generateManagedConfig) Overrides() []ManagedOverrideRule { + return g.overrides +} + +type managedDisableRule struct { + path string + moduleFullName string + fieldName string + fileOption FileOption + fieldOption FieldOption +} + +func (m *managedDisableRule) Path() string { + return m.path +} + +func (m *managedDisableRule) ModuleFullName() string { + return m.moduleFullName +} + +func (m *managedDisableRule) FieldName() string { + return m.fieldName +} + +func (m *managedDisableRule) FileOption() FileOption { + return m.fileOption +} + +func (m *managedDisableRule) FieldOption() FieldOption { + return m.fieldOption +} + +func (m *managedDisableRule) isManagedDisable() {} + +type managedOverrideRule struct { + path string + moduleFullName string + fieldName string + fileOption FileOption + fieldOption FieldOption + value interface{} + prefix string + suffix string +} diff --git a/private/bufnew/bufconfig/generate_plugin_config.go b/private/bufnew/bufconfig/generate_plugin_config.go index 361c8d2db1..35e13e75d0 100644 --- a/private/bufnew/bufconfig/generate_plugin_config.go +++ b/private/bufnew/bufconfig/generate_plugin_config.go @@ -14,6 +14,19 @@ package bufconfig +import ( + "errors" + "fmt" + + "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" + "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" + "github.com/bufbuild/buf/private/pkg/encoding" +) + +const remoteAlphaPluginDeprecationMessage = "the remote field no longer works as " + + "the remote generation alpha has been deprecated, see the migration guide to " + + "now-stable remote plugins: https://buf.build/docs/migration-guides/migrate-remote-generation-alpha/#migrate-to-remote-plugins" + // GenerateStrategy is the generation strategy for a protoc plugin. // // TODO: Should this type live in this package? Perhaps it should live in the package that handles generation? @@ -88,3 +101,290 @@ type GeneratePluginConfig interface { isPluginConfig() } + +func parseStrategy(s string) (GenerateStrategy, error) { + switch s { + case "", "directory": + return GenerateStrategyDirectory, nil + case "all": + return GenerateStrategyAll, nil + default: + return 0, fmt.Errorf("unknown strategy: %s", s) + } +} + +type pluginConfig struct { + pluginConfigType PluginConfigType + name string + out string + opt string + includeImports bool + includeWKT bool + strategy GenerateStrategy + path []string + protocPath string + remoteHost string + revision int +} + +func (p *pluginConfig) PluginConfigType() PluginConfigType { + return p.pluginConfigType +} + +func (p *pluginConfig) Name() string { + return p.name +} + +func (p *pluginConfig) Out() string { + return p.out +} + +func (p *pluginConfig) Opt() string { + return p.opt +} + +func (p *pluginConfig) IncludeImports() bool { + return p.includeImports +} + +func (p *pluginConfig) IncludeWKT() bool { + return p.includeWKT +} + +func (p *pluginConfig) Strategy() GenerateStrategy { + return p.strategy +} + +func (p *pluginConfig) Path() []string { + return p.path +} + +func (p *pluginConfig) ProtocPath() string { + return p.protocPath +} + +func (p *pluginConfig) RemoteHost() string { + return p.remoteHost +} + +func (p *pluginConfig) Revision() int { + return p.revision +} + +// TODO: figure out where is the best place to do parameter validation, here or in new*plugin. +func newPluginConfigFromExternalV1( + externalConfig externalGeneratePluginConfigV1, +) (*pluginConfig, error) { + if externalConfig.Remote != "" { + return nil, errors.New(remoteAlphaPluginDeprecationMessage) + } + // In v1 config, only plugin and name are allowed, since remote alpha plugin + // has been deprecated. + if externalConfig.Plugin == "" && externalConfig.Name == "" { + return nil, fmt.Errorf("one of plugin or name is required") + } + if externalConfig.Plugin != "" && externalConfig.Name != "" { + return nil, fmt.Errorf("only one of plugin or name can be set") + } + var pluginIdentifier string + switch { + case externalConfig.Plugin != "": + pluginIdentifier = externalConfig.Plugin + if _, _, _, _, err := bufremoteplugin.ParsePluginVersionPath(pluginIdentifier); err == nil { + // A remote alpha plugin name is not a valid remote plugin reference. + return nil, fmt.Errorf("invalid remote plugin reference: %s", pluginIdentifier) + } + case externalConfig.Name != "": + pluginIdentifier = externalConfig.Name + if _, _, _, _, err := bufremoteplugin.ParsePluginVersionPath(pluginIdentifier); err == nil { + return nil, fmt.Errorf("invalid plugin name %s, did you mean to use a remote plugin?", pluginIdentifier) + } + if bufpluginref.IsPluginReferenceOrIdentity(pluginIdentifier) { + // A remote alpha plugin name is not a valid local plugin name. + return nil, fmt.Errorf("invalid local plugin name: %s", pluginIdentifier) + } + } + if externalConfig.Out == "" { + return nil, fmt.Errorf("out is required for plugin %s", pluginIdentifier) + } + strategy, err := parseStrategy(externalConfig.Strategy) + if err != nil { + return nil, err + } + opt, err := encoding.InterfaceSliceOrStringToCommaSepString(externalConfig.Opt) + if err != nil { + return nil, err + } + path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Path) + if err != nil { + return nil, err + } + if externalConfig.Plugin != "" && bufpluginref.IsPluginReferenceOrIdentity(pluginIdentifier) { + // TODO: Is checkPathAndStrategyUnset the best way to validate this? + if err := checkPathAndStrategyUnset(externalConfig, pluginIdentifier); err != nil { + return nil, err + } + return newRemotePluginConfig( + externalConfig.Plugin, + externalConfig.Revision, + externalConfig.Out, + opt, + false, + false, + ) + } + // At this point the plugin must be local, regardless whehter it's specified + // by key 'plugin' or 'name'. + if len(path) > 0 { + return newBinaryPluginConfig( + pluginIdentifier, + path, + strategy, + externalConfig.Out, + opt, + false, + false, + ) + } + if externalConfig.ProtocPath != "" { + return newProtocBuiltinPluginConfig( + pluginIdentifier, + externalConfig.ProtocPath, + externalConfig.Out, + opt, + false, + false, + strategy, + ) + } + // It could be either binary or protoc built-in. We defer to the plugin executor + // to decide whether the plugin is protoc-builtin or binary. + return newLocalPluginConfig( + pluginIdentifier, + strategy, + externalConfig.Out, + opt, + false, + false, + ) +} + +func newLocalPluginConfig( + name string, + strategy GenerateStrategy, + out string, + opt string, + includeImports bool, + includeWKT bool, +) (*pluginConfig, error) { + if includeWKT && !includeImports { + return nil, errors.New("cannot include well-known types without including imports") + } + return &pluginConfig{ + name: name, + strategy: strategy, + out: out, + opt: opt, + includeImports: includeImports, + includeWKT: includeWKT, + }, nil +} + +func newBinaryPluginConfig( + name string, + path []string, + strategy GenerateStrategy, + out string, + opt string, + includeImports bool, + includeWKT bool, +) (*pluginConfig, error) { + if len(path) == 0 { + return nil, errors.New("must specify a path to the plugin") + } + if includeWKT && !includeImports { + return nil, errors.New("cannot include well-known types without including imports") + } + return &pluginConfig{ + name: name, + path: path, + strategy: strategy, + out: out, + opt: opt, + includeImports: includeImports, + includeWKT: includeWKT, + }, nil +} + +func newProtocBuiltinPluginConfig( + name string, + protocPath string, + out string, + opt string, + includeImports bool, + includeWKT bool, + strategy GenerateStrategy, +) (*pluginConfig, error) { + if includeWKT && !includeImports { + return nil, errors.New("cannot include well-known types without including imports") + } + return &pluginConfig{ + name: name, + protocPath: protocPath, + out: out, + opt: opt, + strategy: strategy, + includeImports: includeImports, + includeWKT: includeWKT, + }, nil +} + +func newRemotePluginConfig( + name string, + revision int, + out string, + opt string, + includeImports bool, + includeWKT bool, +) (*pluginConfig, error) { + if includeWKT && !includeImports { + return nil, errors.New("cannot include well-known types without including imports") + } + remoteHost, err := parseRemoteHostName(name) + if err != nil { + return nil, err + } + return &pluginConfig{ + name: name, + remoteHost: remoteHost, + revision: revision, + out: out, + opt: opt, + includeImports: includeImports, + includeWKT: includeWKT, + }, nil +} + +func parseRemoteHostName(fullName string) (string, error) { + if identity, err := bufpluginref.PluginIdentityForString(fullName); err == nil { + return identity.Remote(), nil + } + reference, err := bufpluginref.PluginReferenceForString(fullName, 0) + if err == nil { + return reference.Remote(), nil + } + return "", err +} + +func checkPathAndStrategyUnset(plugin externalGeneratePluginConfigV1, pluginIdentifier string) error { + if plugin.Path != nil { + return fmt.Errorf("remote plugin %s cannot specify a path", pluginIdentifier) + } + if plugin.Strategy != "" { + return fmt.Errorf("remote plugin %s cannot specify a strategy", pluginIdentifier) + } + if plugin.ProtocPath != "" { + return fmt.Errorf("remote plugin %s cannot specify a protoc path", pluginIdentifier) + } + return nil +} From e73b9c7640699e3e88b6f263eac8f221b1debd9e Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 12:04:16 -0500 Subject: [PATCH 10/66] rename --- private/bufnew/bufconfig/generate_managed_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index f0d91a9d73..23189899d9 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -93,7 +93,7 @@ type ManagedDisableRule interface { // FieldOption returns the field option to disalbe managed mode for. FieldOption() FieldOption - isManagedDisable() + isManagedDisableRule() } // ManagedOverrideRule is an override rule. An override describes: @@ -126,5 +126,5 @@ type ManagedOverrideRule interface { // Suffix returns the override suffix. Suffix() string - isManagedOverride() + isManagedOverrideRule() } From 33698bd9233d4b7a4e28a56e7dc26bc202786198 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 12:27:40 -0500 Subject: [PATCH 11/66] commit --- .../bufconfig/generate_managed_config.go | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 9d5af3ff90..c9aae52a0c 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -170,7 +170,7 @@ func (m *managedDisableRule) FieldOption() FieldOption { return m.fieldOption } -func (m *managedDisableRule) isManagedDisable() {} +func (m *managedDisableRule) isManagedDisableRule() {} type managedOverrideRule struct { path string @@ -182,3 +182,52 @@ type managedOverrideRule struct { prefix string suffix string } + +// TODO: decide where to validate path and module full name +func newManagedOverrideRule( + path string, + moduleFullName string, + fileOption FileOption, + value interface{}, +) (*managedOverrideRule, error) { + return &managedOverrideRule{ + path: path, + moduleFullName: moduleFullName, + fileOption: fileOption, + value: value, + }, nil +} + +func (m *managedOverrideRule) Path() string { + return m.path +} + +func (m *managedOverrideRule) ModuleFullName() string { + return m.moduleFullName +} + +func (m *managedOverrideRule) FieldName() string { + return m.fieldName +} + +func (m *managedOverrideRule) FileOption() FileOption { + return m.fileOption +} + +func (m *managedOverrideRule) FieldOption() FieldOption { + return m.fieldOption +} + +func (m *managedOverrideRule) Value() interface{} { + return m.value +} + +func (m *managedOverrideRule) Prefix() string { + return m.prefix +} + +func (m *managedOverrideRule) Suffix() string { + return m.suffix +} + +func (m *managedOverrideRule) isManagedOverrideRule() {} From bac4b044e428ab65e0aa3f4bf27b47b12359c875 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 12:31:59 -0500 Subject: [PATCH 12/66] make java_package_prefix, java_package_suffix and etc. file options as well; remove Prefix() and Suffix() from ManagedOverrideRule --- .../bufconfig/generate_managed_config.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 23189899d9..7e6a64f3f8 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -22,6 +22,10 @@ const ( FileOptionUnspecified FileOption = iota // FileOptionJavaPackage is the file option java_package. FileOptionJavaPackage + // FileOptionJavaPackagePrefix is the file option java_package_prefix. + FileOptionJavaPackagePrefix + // FileOptionJavaPackageSuffix is the file option java_package_suffix. + FileOptionJavaPackageSuffix // FileOptionJavaOuterClassname is the file option java_outer_classname. FileOptionJavaOuterClassname // FileOptionJavaMultipleFiles is the file option java_multiple_files. @@ -32,18 +36,26 @@ const ( FileOptionOptimizeFor // FileOptionGoPackage is the file option go_package. FileOptionGoPackage + // FileOptionGoPackagePrefix is the file option go_package_prefix. + FileOptionGoPackagePrefix // FileOptionCcEnableArenas is the file option cc_enable_arenas. FileOptionCcEnableArenas // FileOptionObjcClassPrefix is the file option objc_class_prefix. FileOptionObjcClassPrefix // FileOptionCsharpNamespace is the file option csharp_namespace. FileOptionCsharpNamespace + // FileOptionCsharpNamespacePrefix is the file option csharp_namespace_prefix. + FileOptionCsharpNamespacePrefix // FileOptionPhpNamespace is the file option php_namespace. FileOptionPhpNamespace // FileOptionPhpMetadataNamespace is the file option php_metadata_namespace. FileOptionPhpMetadataNamespace + // FileOptionPhpMetadataNamespaceSuffix is the file option php_metadata_namespace_suffix. + FileOptionPhpMetadataNamespaceSuffix // FileOptionRubyPackage is the file option ruby_package. FileOptionRubyPackage + // FileOptionRubyPackageSuffix is the file option ruby_package_suffix. + FileOptionRubyPackageSuffix ) // FieldOption is a field option. @@ -99,8 +111,7 @@ type ManagedDisableRule interface { // ManagedOverrideRule is an override rule. An override describes: // // - The options to modify. Exactly one of FileOption and FieldOption is not empty. -// - The value, prefix or suffix to modify these options with. Exactly one of -// Value, Prefix and Suffix is not empty. +// - The value to modify these options with. // - The files/fields for which the options are modified. If all of Path, ModuleFullName // - or FieldName are empty, all files/fields are modified. Otherwise, only // file/fields that match the specified Path, ModuleFullName and FieldName @@ -121,10 +132,6 @@ type ManagedOverrideRule interface { FieldOption() FieldOption // Value returns the override value. Value() interface{} - // Prefix returns the override prefix. - Prefix() string - // Suffix returns the override suffix. - Suffix() string isManagedOverrideRule() } From 0d80c90240de59cdf6eda73a7e94550b3455c326 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 16:40:50 -0500 Subject: [PATCH 13/66] parse managed v1 config into the new config interface; need testing --- private/bufnew/bufconfig/buf_gen_yaml_file.go | 18 +- private/bufnew/bufconfig/generate_config.go | 23 +- .../bufconfig/generate_external_config.go | 10 +- .../bufconfig/generate_managed_config.go | 433 +++++++++++++++--- .../bufconfig/generate_managed_option.go | 228 +++++++++ 5 files changed, 642 insertions(+), 70 deletions(-) create mode 100644 private/bufnew/bufconfig/generate_managed_option.go diff --git a/private/bufnew/bufconfig/buf_gen_yaml_file.go b/private/bufnew/bufconfig/buf_gen_yaml_file.go index a04d0d263c..6aab61a4d8 100644 --- a/private/bufnew/bufconfig/buf_gen_yaml_file.go +++ b/private/bufnew/bufconfig/buf_gen_yaml_file.go @@ -122,7 +122,23 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error case FileVersionV1Beta1: return nil, errors.New("TODO") case FileVersionV1: - return nil, errors.New("TODO") + var externalGenYAMLFile externalBufGenYAMLV1 + if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { + return nil, fmt.Errorf("invalid as version %v: %w", fileVersion, err) + } + return &bufGenYAMLFile{ + fileVersion: FileVersionV1, + GenerateConfig: &generateConfig{ + // TODO + pluginConfigs: nil, + // TODO + managedConfig: nil, + // TODO + typeConfig: nil, + // TODO + inputConfigs: nil, + }, + }, nil case FileVersionV2: return nil, newUnsupportedFileVersionError(fileVersion) default: diff --git a/private/bufnew/bufconfig/generate_config.go b/private/bufnew/bufconfig/generate_config.go index 431fb5083d..f4acefa792 100644 --- a/private/bufnew/bufconfig/generate_config.go +++ b/private/bufnew/bufconfig/generate_config.go @@ -43,10 +43,31 @@ type GenerateTypeConfig interface { // *** PRIVATE *** -type generateConfig struct{} +type generateConfig struct { + pluginConfigs []GeneratePluginConfig + managedConfig GenerateManagedConfig + typeConfig GenerateTypeConfig + inputConfigs []GenerateInputConfig +} func newGenerateConfig() *generateConfig { return &generateConfig{} } +func (*generateConfig) GeneratePluginConfigs() []GeneratePluginConfig { + return nil +} + +func (*generateConfig) GenerateManagedConfig() GenerateManagedConfig { + return nil +} + +func (*generateConfig) GenerateTypeConfig() GenerateTypeConfig { + return nil +} + +func (*generateConfig) GenerateInputConfigs() []GenerateInputConfig { + return nil +} + func (*generateConfig) isGenerateConfig() {} diff --git a/private/bufnew/bufconfig/generate_external_config.go b/private/bufnew/bufconfig/generate_external_config.go index 4b879e8fff..e62dc8b567 100644 --- a/private/bufnew/bufconfig/generate_external_config.go +++ b/private/bufnew/bufconfig/generate_external_config.go @@ -18,9 +18,11 @@ import "encoding/json" // TODO: this is a temporary file to avoid crowing other files. We can choose to move stuff from this file over. // TODO: this is also completely copied over from bufgen.go, the only change made to it so far is unexporting the type. +// TODO: update struct type names to externalXYZFileV1/2/1Beta1 +// TODO: update GODOCs to the style of '// externalBufLockFileV2 represents the v2 buf.lock file.' -// externalGenerateConfigV1 is a v1 external generate configuration. -type externalGenerateConfigV1 struct { +// externalBufGenYAMLV1 is a v1 external generate configuration. +type externalBufGenYAMLV1 struct { Version string `json:"version,omitempty" yaml:"version,omitempty"` Plugins []externalGeneratePluginConfigV1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` Managed externalGenerateManagedConfigV1 `json:"managed,omitempty" yaml:"managed,omitempty"` @@ -214,8 +216,8 @@ func (e externalObjcClassPrefixConfigV1) isEmpty() bool { len(e.Override) == 0 } -// externalConfigV1Beta1 is a v1 external generate configuration. -type externalConfigV1Beta1 struct { +// externalBufGenYAMLV1Beta1 is a v1 external generate configuration. +type externalBufGenYAMLV1Beta1 struct { Version string `json:"version,omitempty" yaml:"version,omitempty"` Managed bool `json:"managed,omitempty" yaml:"managed,omitempty"` Plugins []externalGeneratePluginConfigV1Beta1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index b00ae5588b..291a9e58d8 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -14,58 +14,14 @@ package bufconfig -// FileOption is a file option. -type FileOption int - -const ( - // FileOptionUnspecified is an unspecified file option. - FileOptionUnspecified FileOption = iota - // FileOptionJavaPackage is the file option java_package. - FileOptionJavaPackage - // FileOptionJavaPackagePrefix is the file option java_package_prefix. - FileOptionJavaPackagePrefix - // FileOptionJavaPackageSuffix is the file option java_package_suffix. - FileOptionJavaPackageSuffix - // FileOptionJavaOuterClassname is the file option java_outer_classname. - FileOptionJavaOuterClassname - // FileOptionJavaMultipleFiles is the file option java_multiple_files. - FileOptionJavaMultipleFiles - // FileOptionJavaStringCheckUtf8 is the file option java_string_check_utf8. - FileOptionJavaStringCheckUtf8 - // FileOptionOptimizeFor is the file option optimize_for. - FileOptionOptimizeFor - // FileOptionGoPackage is the file option go_package. - FileOptionGoPackage - // FileOptionGoPackagePrefix is the file option go_package_prefix. - FileOptionGoPackagePrefix - // FileOptionCcEnableArenas is the file option cc_enable_arenas. - FileOptionCcEnableArenas - // FileOptionObjcClassPrefix is the file option objc_class_prefix. - FileOptionObjcClassPrefix - // FileOptionCsharpNamespace is the file option csharp_namespace. - FileOptionCsharpNamespace - // FileOptionCsharpNamespacePrefix is the file option csharp_namespace_prefix. - FileOptionCsharpNamespacePrefix - // FileOptionPhpNamespace is the file option php_namespace. - FileOptionPhpNamespace - // FileOptionPhpMetadataNamespace is the file option php_metadata_namespace. - FileOptionPhpMetadataNamespace - // FileOptionPhpMetadataNamespaceSuffix is the file option php_metadata_namespace_suffix. - FileOptionPhpMetadataNamespaceSuffix - // FileOptionRubyPackage is the file option ruby_package. - FileOptionRubyPackage - // FileOptionRubyPackageSuffix is the file option ruby_package_suffix. - FileOptionRubyPackageSuffix -) - -// FieldOption is a field option. -type FieldOption int - -const ( - // FieldOptionUnspecified is an unspecified field option. - FieldOptionUnspecified FieldOption = iota - // FieldOptionJSType is the field option js_type. - FieldOptionJSType +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/bufbuild/buf/private/bufnew/bufmodule" + "github.com/bufbuild/buf/private/pkg/normalpath" ) // GenerateManagedConfig is a managed mode configuration. @@ -141,6 +97,230 @@ type generateManagedConfig struct { overrides []ManagedOverrideRule } +func newManagedOverrideRuleFromExternalV1( + externalConfig externalGenerateManagedConfigV1, +) (*generateManagedConfig, error) { + if externalConfig.isEmpty() || !externalConfig.Enabled { + return nil, nil + } + var ( + disables []ManagedDisableRule + overrides []ManagedOverrideRule + ) + if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { + override, err := newFileOptionOverrideRule( + "", + "", + FileOptionCcEnableArenas, + *externalCCEnableArenas, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { + override, err := newFileOptionOverrideRule( + "", + "", + FileOptionJavaMultipleFiles, + *externalJavaMultipleFiles, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + if externalJavaStringCheckUtf8 := externalConfig.JavaStringCheckUtf8; externalJavaStringCheckUtf8 != nil { + override, err := newFileOptionOverrideRule( + "", + "", + FileOptionJavaStringCheckUtf8, + *externalJavaStringCheckUtf8, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + if externalJavaPackagePrefix := externalConfig.JavaPackagePrefix; !externalJavaPackagePrefix.isEmpty() { + if externalJavaPackagePrefix.Default == "" { + // TODO: resolve this: this message has been updated, compared to the one in bufgen/config.go: + // "java_package_prefix setting requires a default value" + return nil, errors.New("java_package_prefix must have a default value") + } + defaultOverride, err := newFileOptionOverrideRule( + "", + "", + FileOptionJavaPackagePrefix, + externalJavaPackagePrefix.Default, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, defaultOverride) + javaPackagePrefixDisables, javaPackagePrefixOverrides, err := getDisablesAndOverrides( + FileOptionJavaPackage, + externalJavaPackagePrefix.Except, + FileOptionJavaPackagePrefix, + externalJavaPackagePrefix.Override, + ) + if err != nil { + return nil, err + } + disables = append(disables, javaPackagePrefixDisables...) + overrides = append(overrides, javaPackagePrefixOverrides...) + } + if externalCsharpNamespace := externalConfig.CsharpNamespace; !externalCsharpNamespace.isEmpty() { + csharpNamespaceDisables, csharpNamespaceOverrides, err := getDisablesAndOverrides( + FileOptionCsharpNamespace, + externalCsharpNamespace.Except, + FileOptionCsharpNamespace, + externalCsharpNamespace.Override, + ) + if err != nil { + return nil, err + } + disables = append(disables, csharpNamespaceDisables...) + overrides = append(overrides, csharpNamespaceOverrides...) + } + if externalOptimizeFor := externalConfig.OptimizeFor; !externalOptimizeFor.isEmpty() { + if externalOptimizeFor.Default == "" { + return nil, errors.New("optimize_for must have a default value") + } + defaultOverride, err := newFileOptionOverrideRule( + "", + "", + FileOptionOptimizeFor, + externalOptimizeFor.Default, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, defaultOverride) + optimizeForDisables, optimizeForOverrides, err := getDisablesAndOverrides( + FileOptionOptimizeFor, + externalOptimizeFor.Except, + FileOptionOptimizeFor, + externalOptimizeFor.Override, + ) + if err != nil { + return nil, err + } + disables = append(disables, optimizeForDisables...) + overrides = append(overrides, optimizeForOverrides...) + } + if externalGoPackagePrefix := externalConfig.GoPackagePrefix; !externalGoPackagePrefix.isEmpty() { + if externalGoPackagePrefix.Default != "" { + return nil, errors.New("go_package_prefix must have a default value") + } + defaultOverride, err := newFileOptionOverrideRule( + "", + "", + FileOptionGoPackagePrefix, + externalGoPackagePrefix.Default, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, defaultOverride) + goPackagePrefixDisables, goPackagePrefixOverrides, err := getDisablesAndOverrides( + FileOptionGoPackage, + externalGoPackagePrefix.Except, + FileOptionGoPackagePrefix, + externalGoPackagePrefix.Override, + ) + disables = append(disables, goPackagePrefixDisables...) + overrides = append(overrides, goPackagePrefixOverrides...) + } + if externalObjcClassPrefix := externalConfig.ObjcClassPrefix; !externalObjcClassPrefix.isEmpty() { + if externalObjcClassPrefix.Default != "" { + // objc class prefix allows empty default + defaultOverride, err := newFileOptionOverrideRule( + "", + "", + FileOptionObjcClassPrefix, + externalObjcClassPrefix.Default, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, defaultOverride) + } + objcClassPrefixDisables, objcClassPrefixOverrides, err := getDisablesAndOverrides( + FileOptionObjcClassPrefix, + externalObjcClassPrefix.Except, + FileOptionObjcClassPrefix, + externalObjcClassPrefix.Override, + ) + if err != nil { + return nil, err + } + disables = append(disables, objcClassPrefixDisables...) + overrides = append(overrides, objcClassPrefixOverrides...) + } + if externalRubyPackage := externalConfig.RubyPackage; !externalRubyPackage.isEmpty() { + rubyPackageDisables, rubyPackageOverrides, err := getDisablesAndOverrides( + FileOptionRubyPackage, + externalRubyPackage.Except, + FileOptionRubyPackage, + externalRubyPackage.Override, + ) + if err != nil { + return nil, err + } + disables = append(disables, rubyPackageDisables...) + overrides = append(overrides, rubyPackageOverrides...) + } + for upperCaseFileOption, fileToOverride := range externalConfig.Override { + lowerCaseFileOption := strings.ToLower(upperCaseFileOption) + fileOption, ok := stringToFileOption[lowerCaseFileOption] + if !ok { + return nil, fmt.Errorf("%q is not a valid file option", upperCaseFileOption) + } + for filePath, override := range fileToOverride { + normalizedFilePath, err := normalpath.NormalizeAndValidate(filePath) + if err != nil { + return nil, fmt.Errorf( + "failed to normalize import path: %s provided for override: %s", + filePath, + upperCaseFileOption, + ) + } + if filePath != normalizedFilePath { + return nil, fmt.Errorf( + "override can only take normalized import paths, invalid import path: %s provided for override: %s", + filePath, + upperCaseFileOption, + ) + } + var overrideValue interface{} = override + switch fileOption { + case FileOptionCcEnableArenas, FileOptionJavaMultipleFiles, FileOptionJavaStringCheckUtf8: + parseOverrideValue, err := strconv.ParseBool(override) + if err != nil { + return nil, fmt.Errorf("") + } + overrideValue = parseOverrideValue + } + overrideRule, err := newFileOptionOverrideRule( + filePath, + "", + fileOption, + overrideValue, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, overrideRule) + } + } + return &generateManagedConfig{ + disables: disables, + overrides: overrides, + }, nil +} + func (g *generateManagedConfig) Disables() []ManagedDisableRule { return g.disables } @@ -157,6 +337,38 @@ type managedDisableRule struct { fieldOption FieldOption } +func newDisableRule( + path string, + moduleFullName string, + fieldName string, + fileOption FileOption, + fieldOption FieldOption, +) (*managedDisableRule, error) { + if path == "" && moduleFullName == "" && fieldName == "" && fileOption == FileOptionUnspecified && fieldOption == FieldOptionUnspecified { + // This should never happen to parsing configs from provided by users. + return nil, errors.New("empty disable rule is not allowed") + } + if fileOption != FileOptionUnspecified && fieldOption != FieldOptionUnspecified { + return nil, errors.New("at most one of file_option and field_option can be specified") + } + if fieldName != "" && fileOption != FileOptionUnspecified { + return nil, errors.New("cannot disable a file option for a field") + } + // TODO: validate path here? Was it validated in v1/main? + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, err + } + } + return &managedDisableRule{ + path: path, + moduleFullName: moduleFullName, + fieldName: fieldName, + fileOption: fileOption, + fieldOption: fieldOption, + }, nil +} + func (m *managedDisableRule) Path() string { return m.path } @@ -186,22 +398,71 @@ type managedOverrideRule struct { fileOption FileOption fieldOption FieldOption value interface{} - prefix string - suffix string } -// TODO: decide where to validate path and module full name -func newManagedOverrideRule( +func newFileOptionOverrideRule( path string, moduleFullName string, fileOption FileOption, value interface{}, ) (*managedOverrideRule, error) { + // TODO: validate path here? Was it validated in v1/main? + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, err + } + } + // All valid file options have a parse func. This lookup implicitly validates the option. + parseOverrideValueFunc, ok := fileOptionToParseOverrideValueFunc[fileOption] + if !ok { + return nil, fmt.Errorf("invalid fileOption: %v", fileOption) + } + if value == nil { + return nil, fmt.Errorf("value must be specified for override") + } + parsedValue, err := parseOverrideValueFunc(value) + if err != nil { + return nil, fmt.Errorf("invalid value %v for %v: %w", value, fileOption, err) + } return &managedOverrideRule{ path: path, moduleFullName: moduleFullName, fileOption: fileOption, - value: value, + value: parsedValue, + }, nil +} + +func newFieldOptionOverrideRule( + path string, + moduleFullName string, + fieldName string, + fieldOption FieldOption, + value interface{}, +) (*managedOverrideRule, error) { + // TODO: validate path here? Was it validated in v1/main? + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, err + } + } + // All valid field options have a parse func. This lookup implicitly validates the option. + parseOverrideValueFunc, ok := fieldOptionToParseOverrideValueFunc[fieldOption] + if !ok { + return nil, fmt.Errorf("invalid fieldOption: %v", fieldOption) + } + if value == nil { + return nil, fmt.Errorf("value must be specified for override") + } + parsedValue, err := parseOverrideValueFunc(value) + if err != nil { + return nil, fmt.Errorf("invalid value %v for %v: %w", value, fieldOption, err) + } + return &managedOverrideRule{ + path: path, + moduleFullName: moduleFullName, + fieldName: fieldName, + fieldOption: fieldOption, + value: parsedValue, }, nil } @@ -229,12 +490,56 @@ func (m *managedOverrideRule) Value() interface{} { return m.value } -func (m *managedOverrideRule) Prefix() string { - return m.prefix -} +func (m *managedOverrideRule) isManagedOverrideRule() {} -func (m *managedOverrideRule) Suffix() string { - return m.suffix +func getDisablesAndOverrides( + exceptFileOption FileOption, + exceptModuleFullNames []string, + overrideFileOption FileOption, + moduleFullNameToOverride map[string]string, +) ([]ManagedDisableRule, []ManagedOverrideRule, error) { + var ( + disables []ManagedDisableRule + overrides []ManagedOverrideRule + ) + seenExceptModuleFullNames := make(map[string]struct{}, len(exceptModuleFullNames)) + for _, exceptModuleFullName := range exceptModuleFullNames { + if _, err := bufmodule.ParseModuleFullName(exceptModuleFullName); err != nil { + return nil, nil, err + } + if _, ok := seenExceptModuleFullNames[exceptModuleFullName]; ok { + return nil, nil, fmt.Errorf("%q is defined multiple times in except", exceptModuleFullName) + } + seenExceptModuleFullNames[exceptModuleFullName] = struct{}{} + disable, err := newDisableRule( + "", + exceptModuleFullName, + "", + exceptFileOption, + FieldOptionUnspecified, + ) + if err != nil { + return nil, nil, err + } + disables = append(disables, disable) + } + for overrideModuleFullName, overrideValue := range moduleFullNameToOverride { + if _, err := bufmodule.ParseModuleFullName(overrideModuleFullName); err != nil { + return nil, nil, err + } + if _, ok := seenExceptModuleFullNames[overrideModuleFullName]; ok { + return nil, nil, fmt.Errorf("override %q is already defined as an except", overrideModuleFullName) + } + override, err := newFileOptionOverrideRule( + "", + overrideModuleFullName, + overrideFileOption, + overrideValue, + ) + if err != nil { + return nil, nil, err + } + overrides = append(overrides, override) + } + return disables, overrides, nil } - -func (m *managedOverrideRule) isManagedOverrideRule() {} diff --git a/private/bufnew/bufconfig/generate_managed_option.go b/private/bufnew/bufconfig/generate_managed_option.go new file mode 100644 index 0000000000..848983e7b4 --- /dev/null +++ b/private/bufnew/bufconfig/generate_managed_option.go @@ -0,0 +1,228 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfig + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "google.golang.org/protobuf/types/descriptorpb" +) + +// FileOption is a file option. +type FileOption int + +const ( + // FileOptionUnspecified is an unspecified file option. + FileOptionUnspecified FileOption = iota + // FileOptionJavaPackage is the file option java_package. + FileOptionJavaPackage + // FileOptionJavaPackagePrefix is the file option java_package_prefix. + FileOptionJavaPackagePrefix + // FileOptionJavaPackageSuffix is the file option java_package_suffix. + FileOptionJavaPackageSuffix + // FileOptionJavaOuterClassname is the file option java_outer_classname. + FileOptionJavaOuterClassname + // FileOptionJavaMultipleFiles is the file option java_multiple_files. + FileOptionJavaMultipleFiles + // FileOptionJavaStringCheckUtf8 is the file option java_string_check_utf8. + FileOptionJavaStringCheckUtf8 + // FileOptionOptimizeFor is the file option optimize_for. + FileOptionOptimizeFor + // FileOptionGoPackage is the file option go_package. + FileOptionGoPackage + // FileOptionGoPackagePrefix is the file option go_package_prefix. + FileOptionGoPackagePrefix + // FileOptionCcEnableArenas is the file option cc_enable_arenas. + FileOptionCcEnableArenas + // FileOptionObjcClassPrefix is the file option objc_class_prefix. + FileOptionObjcClassPrefix + // FileOptionCsharpNamespace is the file option csharp_namespace. + FileOptionCsharpNamespace + // FileOptionCsharpNamespacePrefix is the file option csharp_namespace_prefix. + FileOptionCsharpNamespacePrefix + // FileOptionPhpNamespace is the file option php_namespace. + FileOptionPhpNamespace + // FileOptionPhpMetadataNamespace is the file option php_metadata_namespace. + FileOptionPhpMetadataNamespace + // FileOptionPhpMetadataNamespaceSuffix is the file option php_metadata_namespace_suffix. + FileOptionPhpMetadataNamespaceSuffix + // FileOptionRubyPackage is the file option ruby_package. + FileOptionRubyPackage + // FileOptionRubyPackageSuffix is the file option ruby_package_suffix. + FileOptionRubyPackageSuffix +) + +// FieldOption is a field option. +type FieldOption int + +const ( + // FieldOptionUnspecified is an unspecified field option. + FieldOptionUnspecified FieldOption = iota + // FieldOptionJSType is the field option js_type. + FieldOptionJSType +) + +var ( + fileOptionToString = map[FileOption]string{ + FileOptionJavaPackage: "java_package", + FileOptionJavaPackagePrefix: "java_package_prefix", + FileOptionJavaPackageSuffix: "java_package_suffix", + FileOptionJavaOuterClassname: "java_outer_classname", + FileOptionJavaMultipleFiles: "java_multiple_files", + FileOptionJavaStringCheckUtf8: "java_string_check_utf8", + FileOptionOptimizeFor: "optimize_for", + FileOptionGoPackage: "go_package", + FileOptionGoPackagePrefix: "go_package_prefix", + FileOptionCcEnableArenas: "cc_enable_arenas", + FileOptionObjcClassPrefix: "objc_class_prefix", + FileOptionCsharpNamespace: "csharp_namespace", + FileOptionCsharpNamespacePrefix: "csharp_namespace_prefix", + FileOptionPhpNamespace: "php_namespace", + FileOptionPhpMetadataNamespace: "php_metadata_namespace", + FileOptionPhpMetadataNamespaceSuffix: "php_metadata_namespace_suffix", + FileOptionRubyPackage: "ruby_package", + FileOptionRubyPackageSuffix: "ruby_package_suffix", + } + stringToFileOption = map[string]FileOption{ + "java_package": FileOptionJavaPackage, + "java_package_prefix": FileOptionJavaPackagePrefix, + "java_package_suffix": FileOptionJavaPackageSuffix, + "java_outer_classname": FileOptionJavaOuterClassname, + "java_multiple_files": FileOptionJavaMultipleFiles, + "java_string_check_utf8": FileOptionJavaStringCheckUtf8, + "optimize_for": FileOptionOptimizeFor, + "go_package": FileOptionGoPackage, + "go_package_prefix": FileOptionGoPackagePrefix, + "cc_enable_arenas": FileOptionCcEnableArenas, + "objc_class_prefix": FileOptionObjcClassPrefix, + "csharp_namespace": FileOptionCsharpNamespace, + "csharp_namespace_prefix": FileOptionCsharpNamespacePrefix, + "php_namespace": FileOptionPhpNamespace, + "php_metadata_namespace": FileOptionPhpMetadataNamespace, + "php_metadata_namespace_suffix": FileOptionPhpMetadataNamespaceSuffix, + "ruby_package": FileOptionRubyPackage, + "ruby_package_suffix": FileOptionRubyPackageSuffix, + } + fileOptionToParseOverrideValueFunc = map[FileOption]func(interface{}) (interface{}, error){ + FileOptionJavaPackage: parseOverrideValue[string], + FileOptionJavaPackagePrefix: parseOverrideValue[string], + FileOptionJavaPackageSuffix: parseOverrideValue[string], + FileOptionOptimizeFor: parseOverrideValueOptimizeMode, + FileOptionJavaOuterClassname: parseOverrideValue[string], + FileOptionJavaMultipleFiles: parseOverrideValue[bool], + FileOptionJavaStringCheckUtf8: parseOverrideValue[bool], + FileOptionGoPackage: parseOverrideValue[string], + FileOptionGoPackagePrefix: parseOverrideValue[string], + FileOptionCcEnableArenas: parseOverrideValue[bool], + FileOptionObjcClassPrefix: parseOverrideValue[string], // objc_class_prefix is in descriptor.proto + FileOptionCsharpNamespace: parseOverrideValue[string], + FileOptionCsharpNamespacePrefix: parseOverrideValue[string], + FileOptionPhpNamespace: parseOverrideValue[string], + FileOptionPhpMetadataNamespace: parseOverrideValue[string], + FileOptionPhpMetadataNamespaceSuffix: parseOverrideValue[string], + FileOptionRubyPackage: parseOverrideValue[string], + FileOptionRubyPackageSuffix: parseOverrideValue[string], + } + allFieldOptions = []FieldOption{ + FieldOptionJSType, + } + fieldOptionToString = map[FieldOption]string{ + FieldOptionJSType: "jstype", + } + stringToFieldOption = map[string]FieldOption{ + "jstype": FieldOptionJSType, + } + fieldOptionToParseOverrideValueFunc = map[FieldOption]func(interface{}) (interface{}, error){ + FieldOptionJSType: parseOverrideValueJSType, + } +) + +// String implements fmt.Stringer. +func (f FileOption) String() string { + s, ok := fileOptionToString[f] + if !ok { + return strconv.Itoa(int(f)) + } + return s +} + +func parseFileOption(s string) (FileOption, error) { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + return 0, errors.New("empty fileOption") + } + f, ok := stringToFileOption[s] + if ok { + return f, nil + } + return 0, fmt.Errorf("unknown fileOption: %q", s) +} + +// String implements fmt.Stringer. +func (f FieldOption) String() string { + s, ok := fieldOptionToString[f] + if !ok { + return strconv.Itoa(int(f)) + } + return s +} + +func parseFieldOption(s string) (FieldOption, error) { + s = strings.ToLower(strings.TrimSpace(s)) + if s == "" { + return 0, errors.New("empty field option") + } + f, ok := stringToFieldOption[s] + if ok { + return f, nil + } + return 0, fmt.Errorf("unknown field option: %q", s) +} + +func parseOverrideValue[T string | bool](overrideValue interface{}) (interface{}, error) { + parsedValue, ok := overrideValue.(T) + if !ok { + // TODO: test out this message + return nil, fmt.Errorf("must be a %T", overrideValue) + } + return parsedValue, nil +} + +func parseOverrideValueOptimizeMode(overrideValue interface{}) (interface{}, error) { + optimizeModeName, ok := overrideValue.(string) + if !ok { + return nil, errors.New("must be one of SPEED, CODE_SIZE or LITE_RUNTIME") + } + optimizeMode, ok := descriptorpb.FileOptions_OptimizeMode_value[optimizeModeName] + if !ok { + return nil, errors.New("must be one of SPEED, CODE_SIZE or LITE_RUNTIME") + } + return optimizeMode, nil +} + +func parseOverrideValueJSType(override interface{}) (interface{}, error) { + jsTypeName, ok := override.(string) + if !ok { + return nil, errors.New("must be one of JS_NORMAL, JS_STRING or JS_NUMBER") + } + jsTypeEnum, ok := descriptorpb.FieldOptions_JSType_value[jsTypeName] + if !ok { + return nil, errors.New("must be one of JS_NORMAL, JS_STRING or JS_NUMBER") + } + return descriptorpb.FieldOptions_JSType(jsTypeEnum), nil +} From 5bea9fed0c5bd781114dee52a2dcbe527bf85937 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 16:41:28 -0500 Subject: [PATCH 14/66] commit --- private/bufnew/bufconfig/generate_managed_config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 291a9e58d8..aa6a1c2492 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -230,6 +230,9 @@ func newManagedOverrideRuleFromExternalV1( FileOptionGoPackagePrefix, externalGoPackagePrefix.Override, ) + if err != nil { + return nil, err + } disables = append(disables, goPackagePrefixDisables...) overrides = append(overrides, goPackagePrefixOverrides...) } From 1a440e0d8ff39b9d7117b7a627740445bd1d1e16 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 17:20:33 -0500 Subject: [PATCH 15/66] type config and plugin config --- private/bufnew/bufconfig/buf_gen_yaml_file.go | 26 ++++++++--- private/bufnew/bufconfig/generate_config.go | 9 ---- .../bufconfig/generate_managed_config.go | 2 + .../bufconfig/generate_plugin_config.go | 8 ++-- .../bufnew/bufconfig/generate_type_config.go | 44 +++++++++++++++++++ 5 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 private/bufnew/bufconfig/generate_type_config.go diff --git a/private/bufnew/bufconfig/buf_gen_yaml_file.go b/private/bufnew/bufconfig/buf_gen_yaml_file.go index 6aab61a4d8..a4ab2a83ae 100644 --- a/private/bufnew/bufconfig/buf_gen_yaml_file.go +++ b/private/bufnew/bufconfig/buf_gen_yaml_file.go @@ -20,6 +20,7 @@ import ( "fmt" "io" + "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" ) @@ -126,16 +127,27 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { return nil, fmt.Errorf("invalid as version %v: %w", fileVersion, err) } + managedConfig, err := newManagedOverrideRuleFromExternalV1(externalGenYAMLFile.Managed) + if err != nil { + return nil, err + } + if len(externalGenYAMLFile.Plugins) == 0 { + return nil, errors.New("must specifiy at least one plugin") + } + pluginConfigs, err := slicesext.MapError( + externalGenYAMLFile.Plugins, + newPluginConfigFromExternalV1, + ) + if err != nil { + return nil, err + } return &bufGenYAMLFile{ fileVersion: FileVersionV1, GenerateConfig: &generateConfig{ - // TODO - pluginConfigs: nil, - // TODO - managedConfig: nil, - // TODO - typeConfig: nil, - // TODO + pluginConfigs: pluginConfigs, + managedConfig: managedConfig, + typeConfig: newGenerateTypeConfig(externalGenYAMLFile.Types.Include), + // TODO for v2 inputConfigs: nil, }, }, nil diff --git a/private/bufnew/bufconfig/generate_config.go b/private/bufnew/bufconfig/generate_config.go index f4acefa792..144bfc8ad5 100644 --- a/private/bufnew/bufconfig/generate_config.go +++ b/private/bufnew/bufconfig/generate_config.go @@ -32,15 +32,6 @@ type GenerateConfig interface { isGenerateConfig() } -// GenerateTypeConfig is a type filter configuration. -type GenerateTypeConfig interface { - // If IncludeTypes returns a non-empty list, it means that only those types are - // generated. Otherwise all types are generated. - IncludeTypes() []string - - isGenerateTypeConfig() -} - // *** PRIVATE *** type generateConfig struct { diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index aa6a1c2492..238628ef73 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -332,6 +332,8 @@ func (g *generateManagedConfig) Overrides() []ManagedOverrideRule { return g.overrides } +func (g *generateManagedConfig) isGenerateManagedConfig() {} + type managedDisableRule struct { path string moduleFullName string diff --git a/private/bufnew/bufconfig/generate_plugin_config.go b/private/bufnew/bufconfig/generate_plugin_config.go index 35e13e75d0..ce407820b8 100644 --- a/private/bufnew/bufconfig/generate_plugin_config.go +++ b/private/bufnew/bufconfig/generate_plugin_config.go @@ -99,7 +99,7 @@ type GeneratePluginConfig interface { // This is not empty only when the plugin is remote. Revision() int - isPluginConfig() + isGeneratePluginConfig() } func parseStrategy(s string) (GenerateStrategy, error) { @@ -127,7 +127,7 @@ type pluginConfig struct { revision int } -func (p *pluginConfig) PluginConfigType() PluginConfigType { +func (p *pluginConfig) Type() PluginConfigType { return p.pluginConfigType } @@ -171,10 +171,12 @@ func (p *pluginConfig) Revision() int { return p.revision } +func (p *pluginConfig) isGeneratePluginConfig() {} + // TODO: figure out where is the best place to do parameter validation, here or in new*plugin. func newPluginConfigFromExternalV1( externalConfig externalGeneratePluginConfigV1, -) (*pluginConfig, error) { +) (GeneratePluginConfig, error) { if externalConfig.Remote != "" { return nil, errors.New(remoteAlphaPluginDeprecationMessage) } diff --git a/private/bufnew/bufconfig/generate_type_config.go b/private/bufnew/bufconfig/generate_type_config.go new file mode 100644 index 0000000000..ae1ef26d43 --- /dev/null +++ b/private/bufnew/bufconfig/generate_type_config.go @@ -0,0 +1,44 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfig + +// GenerateTypeConfig is a type filter configuration. +type GenerateTypeConfig interface { + // If IncludeTypes returns a non-empty list, it means that only those types are + // generated. Otherwise all types are generated. + IncludeTypes() []string + + isGenerateTypeConfig() +} + +type generateTypeConfig struct { + includeTypes []string +} + +// TODO: it seems like this isn't validated in main, but we should do some validation +func newGenerateTypeConfig(includeTypes []string) *generateTypeConfig { + if len(includeTypes) == 0 { + return nil + } + return &generateTypeConfig{ + includeTypes: includeTypes, + } +} + +func (g *generateTypeConfig) IncludeTypes() []string { + return g.includeTypes +} + +func (g *generateTypeConfig) isGenerateTypeConfig() {} From 930e373a268a6ac37d904da0177cf8f495f80d69 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 17:21:02 -0500 Subject: [PATCH 16/66] TODO --- private/bufnew/bufconfig/buf_gen_yaml_file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/private/bufnew/bufconfig/buf_gen_yaml_file.go b/private/bufnew/bufconfig/buf_gen_yaml_file.go index a4ab2a83ae..18b31a6408 100644 --- a/private/bufnew/bufconfig/buf_gen_yaml_file.go +++ b/private/bufnew/bufconfig/buf_gen_yaml_file.go @@ -152,7 +152,7 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error }, }, nil case FileVersionV2: - return nil, newUnsupportedFileVersionError(fileVersion) + return nil, errors.New("TODO") default: // This is a system error since we've already parsed. return nil, fmt.Errorf("unknown FileVersion: %v", fileVersion) From 193f4ac1c864bce71d9d7ec20fe0b8c2ca3572dc Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 17:28:55 -0500 Subject: [PATCH 17/66] rename --- private/bufnew/bufconfig/buf_gen_yaml_file.go | 2 +- .../bufnew/bufconfig/generate_config_test.go | 26 +++++++++++++++++++ .../bufconfig/generate_external_config.go | 4 +-- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 private/bufnew/bufconfig/generate_config_test.go diff --git a/private/bufnew/bufconfig/buf_gen_yaml_file.go b/private/bufnew/bufconfig/buf_gen_yaml_file.go index 18b31a6408..02aa3c6ee1 100644 --- a/private/bufnew/bufconfig/buf_gen_yaml_file.go +++ b/private/bufnew/bufconfig/buf_gen_yaml_file.go @@ -123,7 +123,7 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error case FileVersionV1Beta1: return nil, errors.New("TODO") case FileVersionV1: - var externalGenYAMLFile externalBufGenYAMLV1 + var externalGenYAMLFile externalBufGenYAMLFileV1 if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { return nil, fmt.Errorf("invalid as version %v: %w", fileVersion, err) } diff --git a/private/bufnew/bufconfig/generate_config_test.go b/private/bufnew/bufconfig/generate_config_test.go new file mode 100644 index 0000000000..a9e3e6a6ef --- /dev/null +++ b/private/bufnew/bufconfig/generate_config_test.go @@ -0,0 +1,26 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfig + +import "testing" + +func TestParseConfigFromExternalV1(t *testing.T) { + t.Parallel() + + // testcases := []struct { + // description string + // externalConfig externalBufGenYAMLFileV1 + // }{} +} diff --git a/private/bufnew/bufconfig/generate_external_config.go b/private/bufnew/bufconfig/generate_external_config.go index e62dc8b567..1e5592f882 100644 --- a/private/bufnew/bufconfig/generate_external_config.go +++ b/private/bufnew/bufconfig/generate_external_config.go @@ -21,8 +21,8 @@ import "encoding/json" // TODO: update struct type names to externalXYZFileV1/2/1Beta1 // TODO: update GODOCs to the style of '// externalBufLockFileV2 represents the v2 buf.lock file.' -// externalBufGenYAMLV1 is a v1 external generate configuration. -type externalBufGenYAMLV1 struct { +// externalBufGenYAMLFileV1 is a v1 external generate configuration. +type externalBufGenYAMLFileV1 struct { Version string `json:"version,omitempty" yaml:"version,omitempty"` Plugins []externalGeneratePluginConfigV1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` Managed externalGenerateManagedConfigV1 `json:"managed,omitempty" yaml:"managed,omitempty"` From 2bd20c2e0b5701ba20e4ba692c493317f7c92178 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 18:22:19 -0500 Subject: [PATCH 18/66] plugin config tests --- private/bufnew/bufconfig/buf_gen_yaml_file.go | 23 +- private/bufnew/bufconfig/generate_config.go | 32 ++ .../bufnew/bufconfig/generate_config_test.go | 372 +++++++++++++++++- .../bufconfig/generate_managed_config.go | 2 +- .../bufconfig/generate_plugin_config.go | 68 ++-- .../bufnew/bufconfig/generate_type_config.go | 2 +- 6 files changed, 445 insertions(+), 54 deletions(-) diff --git a/private/bufnew/bufconfig/buf_gen_yaml_file.go b/private/bufnew/bufconfig/buf_gen_yaml_file.go index 02aa3c6ee1..ae135dab2c 100644 --- a/private/bufnew/bufconfig/buf_gen_yaml_file.go +++ b/private/bufnew/bufconfig/buf_gen_yaml_file.go @@ -20,7 +20,6 @@ import ( "fmt" "io" - "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" ) @@ -127,29 +126,13 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { return nil, fmt.Errorf("invalid as version %v: %w", fileVersion, err) } - managedConfig, err := newManagedOverrideRuleFromExternalV1(externalGenYAMLFile.Managed) - if err != nil { - return nil, err - } - if len(externalGenYAMLFile.Plugins) == 0 { - return nil, errors.New("must specifiy at least one plugin") - } - pluginConfigs, err := slicesext.MapError( - externalGenYAMLFile.Plugins, - newPluginConfigFromExternalV1, - ) + generateConfig, err := newGenerateConfigFromExternalFileV1(externalGenYAMLFile) if err != nil { return nil, err } return &bufGenYAMLFile{ - fileVersion: FileVersionV1, - GenerateConfig: &generateConfig{ - pluginConfigs: pluginConfigs, - managedConfig: managedConfig, - typeConfig: newGenerateTypeConfig(externalGenYAMLFile.Types.Include), - // TODO for v2 - inputConfigs: nil, - }, + fileVersion: fileVersion, + GenerateConfig: generateConfig, }, nil case FileVersionV2: return nil, errors.New("TODO") diff --git a/private/bufnew/bufconfig/generate_config.go b/private/bufnew/bufconfig/generate_config.go index 144bfc8ad5..b83ba4fa39 100644 --- a/private/bufnew/bufconfig/generate_config.go +++ b/private/bufnew/bufconfig/generate_config.go @@ -14,6 +14,12 @@ package bufconfig +import ( + "errors" + + "github.com/bufbuild/buf/private/pkg/slicesext" +) + // GenerateConfig is a generation configuration. type GenerateConfig interface { // GeneratePluginConfigs returns the plugin configurations. This will always be @@ -32,6 +38,32 @@ type GenerateConfig interface { isGenerateConfig() } +func newGenerateConfigFromExternalFileV1( + externalFile externalBufGenYAMLFileV1, +) (GenerateConfig, error) { + managedConfig, err := newManagedOverrideRuleFromExternalV1(externalFile.Managed) + if err != nil { + return nil, err + } + if len(externalFile.Plugins) == 0 { + return nil, errors.New("must specifiy at least one plugin") + } + pluginConfigs, err := slicesext.MapError( + externalFile.Plugins, + newPluginConfigFromExternalV1, + ) + if err != nil { + return nil, err + } + return &generateConfig{ + pluginConfigs: pluginConfigs, + managedConfig: managedConfig, + typeConfig: newGenerateTypeConfig(externalFile.Types.Include), + // TODO for v2 + inputConfigs: nil, + }, nil +} + // *** PRIVATE *** type generateConfig struct { diff --git a/private/bufnew/bufconfig/generate_config_test.go b/private/bufnew/bufconfig/generate_config_test.go index a9e3e6a6ef..5d8ece6d18 100644 --- a/private/bufnew/bufconfig/generate_config_test.go +++ b/private/bufnew/bufconfig/generate_config_test.go @@ -14,13 +14,375 @@ package bufconfig -import "testing" +import ( + "testing" + + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/stretchr/testify/require" +) func TestParseConfigFromExternalV1(t *testing.T) { t.Parallel() + testcases := []struct { + description string + externalConfig externalBufGenYAMLFileV1 + expectedConfig GenerateConfig + }{ + { + description: "name_local_plugin_strategy", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Name: "java", + Out: "java/out", + Opt: "a=b,c", + Strategy: "all", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "java", + out: "java/out", + opt: "a=b,c", + strategy: GenerateStrategyAll, + }, + }, + }, + }, + { + description: "name_local_plugin_strategy", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "java", + Out: "java/out", + Opt: "a=b,c", + Strategy: "all", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "java", + out: "java/out", + opt: "a=b,c", + strategy: GenerateStrategyAll, + }, + }, + }, + }, + { + description: "name_binary_plugin_with_string_slice_path_and_opts", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Name: "go", + Out: "go/out", + Path: slicesext.Map([]string{"go", "run", "goplugin"}, func(s string) interface{} { return s }), + Opt: slicesext.Map([]string{"a=b", "c"}, func(s string) interface{} { return s }), + Strategy: "directory", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeBinary, + name: "go", + out: "go/out", + path: []string{"go", "run", "goplugin"}, + opt: "a=b,c", + strategy: GenerateStrategyDirectory, + }, + }, + }, + }, + { + description: "plugin_binary_plugin_with_string_slice_path_and_opts", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + Path: slicesext.Map([]string{"go", "run", "goplugin"}, func(s string) interface{} { return s }), + Opt: slicesext.Map([]string{"a=b", "c"}, func(s string) interface{} { return s }), + Strategy: "directory", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeBinary, + name: "go", + out: "go/out", + path: []string{"go", "run", "goplugin"}, + opt: "a=b,c", + strategy: GenerateStrategyDirectory, + }, + }, + }, + }, + { + description: "name_binary_plugin_with_string_path", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Name: "go2", + Out: "go2/out", + Path: "protoc-gen-go", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeBinary, + name: "go2", + out: "go2/out", + path: []string{"protoc-gen-go"}, + strategy: GenerateStrategyDirectory, + }, + }, + }, + }, + { + description: "plugin_binary_plugin_with_string_path", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go2", + Out: "go2/out", + Path: "protoc-gen-go", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeBinary, + name: "go2", + out: "go2/out", + path: []string{"protoc-gen-go"}, + strategy: GenerateStrategyDirectory, + }, + }, + }, + }, + { + description: "name_protoc_builtin_plugin", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Name: "cpp", + Out: "cpp/out", + ProtocPath: "path/to/protoc", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeProtocBuiltin, + name: "cpp", + out: "cpp/out", + protocPath: "path/to/protoc", + strategy: GenerateStrategyDirectory, + }, + }, + }, + }, + { + description: "plugin_protoc_builtin_plugin", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "cpp", + Out: "cpp/out", + ProtocPath: "path/to/protoc", + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeProtocBuiltin, + name: "cpp", + out: "cpp/out", + protocPath: "path/to/protoc", + strategy: GenerateStrategyDirectory, + }, + }, + }, + }, + { + description: "remote_plugin_reference", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "buf.build/protocolbuffers/go:v1.31.0", + Out: "go/out", + Revision: 1, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeRemote, + remoteHost: "buf.build", + revision: 1, + name: "buf.build/protocolbuffers/go:v1.31.0", + out: "go/out", + }, + }, + }, + }, + { + description: "remote_plugin_identity", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "buf.build/protocolbuffers/go", + Out: "go/out", + Revision: 1, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeRemote, + remoteHost: "buf.build", + revision: 1, + name: "buf.build/protocolbuffers/go", + out: "go/out", + }, + }, + }, + }, + } + for _, testcase := range testcases { + testcase := testcase + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + parsedConfig, err := newGenerateConfigFromExternalFileV1(testcase.externalConfig) + require.NoError(t, err) + require.Equal(t, testcase.expectedConfig, parsedConfig) + }) + } +} - // testcases := []struct { - // description string - // externalConfig externalBufGenYAMLFileV1 - // }{} +func TestParseConfigFromExternalV1Fail(t *testing.T) { + t.Parallel() + testcases := []struct { + description string + externalConfig externalBufGenYAMLFileV1 + }{ + { + description: "empty_out", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Name: "java", + Out: "", + Opt: "a=b,c", + Strategy: "all", + }, + }, + }, + }, + { + description: "both_plugin_and_name", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "java", + Name: "go", + Out: "java/out", + }, + }, + }, + }, + { + description: "neither_plugin_nor_name", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Out: "java/out", + }, + }, + }, + }, + { + description: "no_plugins", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: nil, + }, + }, + { + description: "invalid_strategy", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + Strategy: "invalid", + }, + }, + }, + }, + { + description: "deprecated_alpha_plugin", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Remote: "buf.build/bufbuild/plugins/connect-go:v1.3.1-1", + Out: "connect/out", + }, + }, + }, + }, + { + description: "plugin_with_deprecated_alpha_plugin_name", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "buf.build/bufbuild/plugins/connect-go:v1.3.1-1", + Out: "connect/out", + }, + }, + }, + }, + } + for _, testcase := range testcases { + testcase := testcase + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + _, err := newGenerateConfigFromExternalFileV1(testcase.externalConfig) + require.Error(t, err) + }) + } } diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 238628ef73..700e5ce035 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -99,7 +99,7 @@ type generateManagedConfig struct { func newManagedOverrideRuleFromExternalV1( externalConfig externalGenerateManagedConfigV1, -) (*generateManagedConfig, error) { +) (GenerateManagedConfig, error) { if externalConfig.isEmpty() || !externalConfig.Enabled { return nil, nil } diff --git a/private/bufnew/bufconfig/generate_plugin_config.go b/private/bufnew/bufconfig/generate_plugin_config.go index ce407820b8..e6112825cc 100644 --- a/private/bufnew/bufconfig/generate_plugin_config.go +++ b/private/bufnew/bufconfig/generate_plugin_config.go @@ -282,13 +282,17 @@ func newLocalPluginConfig( if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } + if strategy == 0 { + strategy = GenerateStrategyDirectory + } return &pluginConfig{ - name: name, - strategy: strategy, - out: out, - opt: opt, - includeImports: includeImports, - includeWKT: includeWKT, + pluginConfigType: PluginConfigTypeLocal, + name: name, + strategy: strategy, + out: out, + opt: opt, + includeImports: includeImports, + includeWKT: includeWKT, }, nil } @@ -307,14 +311,18 @@ func newBinaryPluginConfig( if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } + if strategy == 0 { + strategy = GenerateStrategyDirectory + } return &pluginConfig{ - name: name, - path: path, - strategy: strategy, - out: out, - opt: opt, - includeImports: includeImports, - includeWKT: includeWKT, + pluginConfigType: PluginConfigTypeBinary, + name: name, + path: path, + strategy: strategy, + out: out, + opt: opt, + includeImports: includeImports, + includeWKT: includeWKT, }, nil } @@ -330,14 +338,18 @@ func newProtocBuiltinPluginConfig( if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } + if strategy == 0 { + strategy = GenerateStrategyDirectory + } return &pluginConfig{ - name: name, - protocPath: protocPath, - out: out, - opt: opt, - strategy: strategy, - includeImports: includeImports, - includeWKT: includeWKT, + pluginConfigType: PluginConfigTypeProtocBuiltin, + name: name, + protocPath: protocPath, + out: out, + opt: opt, + strategy: strategy, + includeImports: includeImports, + includeWKT: includeWKT, }, nil } @@ -356,14 +368,16 @@ func newRemotePluginConfig( if err != nil { return nil, err } + // TODO: validate revision return &pluginConfig{ - name: name, - remoteHost: remoteHost, - revision: revision, - out: out, - opt: opt, - includeImports: includeImports, - includeWKT: includeWKT, + pluginConfigType: PluginConfigTypeRemote, + name: name, + remoteHost: remoteHost, + revision: revision, + out: out, + opt: opt, + includeImports: includeImports, + includeWKT: includeWKT, }, nil } diff --git a/private/bufnew/bufconfig/generate_type_config.go b/private/bufnew/bufconfig/generate_type_config.go index ae1ef26d43..ccf0ded554 100644 --- a/private/bufnew/bufconfig/generate_type_config.go +++ b/private/bufnew/bufconfig/generate_type_config.go @@ -28,7 +28,7 @@ type generateTypeConfig struct { } // TODO: it seems like this isn't validated in main, but we should do some validation -func newGenerateTypeConfig(includeTypes []string) *generateTypeConfig { +func newGenerateTypeConfig(includeTypes []string) GenerateTypeConfig { if len(includeTypes) == 0 { return nil } From 088cd69d761f47ce2950fdc179c7c6b0f1d1ecc7 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 22 Nov 2023 18:34:37 -0500 Subject: [PATCH 19/66] add some tests; need more for managed; need to debug the code commented out in gnenerate_managed_config.go --- .../bufnew/bufconfig/generate_config_test.go | 65 +++++++++++++++++ .../bufconfig/generate_managed_config.go | 72 +++++++++---------- 2 files changed, 101 insertions(+), 36 deletions(-) diff --git a/private/bufnew/bufconfig/generate_config_test.go b/private/bufnew/bufconfig/generate_config_test.go index 5d8ece6d18..8910993497 100644 --- a/private/bufnew/bufconfig/generate_config_test.go +++ b/private/bufnew/bufconfig/generate_config_test.go @@ -19,6 +19,7 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" ) func TestParseConfigFromExternalV1(t *testing.T) { @@ -276,6 +277,70 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, }, + { + description: "managed mode", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + }, + }, + Managed: externalGenerateManagedConfigV1{ + Enabled: true, + CcEnableArenas: proto.Bool(true), + JavaMultipleFiles: proto.Bool(true), + JavaStringCheckUtf8: proto.Bool(true), + JavaPackagePrefix: externalJavaPackagePrefixConfigV1{ + Default: "foo", + Except: []string{"buf.build/acme/foo", "buf.build/acme/bar"}, + Override: map[string]string{ + "buf.build/acme/baz": "baz", + "buf.build/acme/bat": "bat", + }, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "go", + out: "go/out", + strategy: GenerateStrategyDirectory, + }, + }, + managedConfig: &generateManagedConfig{ + disables: []ManagedDisableRule{ + &managedDisableRule{ + fileOption: FileOptionJavaPackage, + moduleFullName: "buf.build/acme/foo", + }, + &managedDisableRule{ + fileOption: FileOptionJavaPackage, + moduleFullName: "buf.build/acme/bar", + }, + }, + overrides: []ManagedOverrideRule{ + &managedOverrideRule{ + fileOption: FileOptionJavaPackagePrefix, + value: "foo", + }, + &managedOverrideRule{ + fileOption: FileOptionJavaPackagePrefix, + moduleFullName: "buf.build/acme/baz", + value: "baz", + }, + &managedOverrideRule{ + fileOption: FileOptionJavaPackagePrefix, + moduleFullName: "buf.build/acme/bat", + value: "bat", + }, + }, + }, + }, + }, } for _, testcase := range testcases { testcase := testcase diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 700e5ce035..06e58ed1e6 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -107,42 +107,42 @@ func newManagedOverrideRuleFromExternalV1( disables []ManagedDisableRule overrides []ManagedOverrideRule ) - if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { - override, err := newFileOptionOverrideRule( - "", - "", - FileOptionCcEnableArenas, - *externalCCEnableArenas, - ) - if err != nil { - return nil, err - } - overrides = append(overrides, override) - } - if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { - override, err := newFileOptionOverrideRule( - "", - "", - FileOptionJavaMultipleFiles, - *externalJavaMultipleFiles, - ) - if err != nil { - return nil, err - } - overrides = append(overrides, override) - } - if externalJavaStringCheckUtf8 := externalConfig.JavaStringCheckUtf8; externalJavaStringCheckUtf8 != nil { - override, err := newFileOptionOverrideRule( - "", - "", - FileOptionJavaStringCheckUtf8, - *externalJavaStringCheckUtf8, - ) - if err != nil { - return nil, err - } - overrides = append(overrides, override) - } + // if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { + // override, err := newFileOptionOverrideRule( + // "", + // "", + // FileOptionCcEnableArenas, + // *externalCCEnableArenas, + // ) + // if err != nil { + // return nil, err + // } + // overrides = append(overrides, override) + // } + // if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { + // override, err := newFileOptionOverrideRule( + // "", + // "", + // FileOptionJavaMultipleFiles, + // *externalJavaMultipleFiles, + // ) + // if err != nil { + // return nil, err + // } + // overrides = append(overrides, override) + // } + // if externalJavaStringCheckUtf8 := externalConfig.JavaStringCheckUtf8; externalJavaStringCheckUtf8 != nil { + // override, err := newFileOptionOverrideRule( + // "", + // "", + // FileOptionJavaStringCheckUtf8, + // *externalJavaStringCheckUtf8, + // ) + // if err != nil { + // return nil, err + // } + // overrides = append(overrides, override) + // } if externalJavaPackagePrefix := externalConfig.JavaPackagePrefix; !externalJavaPackagePrefix.isEmpty() { if externalJavaPackagePrefix.Default == "" { // TODO: resolve this: this message has been updated, compared to the one in bufgen/config.go: From cd3b0267e7577727a7cfb625cef5e17680c6a7a7 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 23 Nov 2023 00:04:07 -0500 Subject: [PATCH 20/66] tests for managed mode config v1 -> parsed config --- .../bufnew/bufconfig/generate_config_test.go | 351 +++++++++++++++++- .../bufconfig/generate_managed_config.go | 197 +++++----- .../bufconfig/generate_managed_option.go | 2 +- 3 files changed, 453 insertions(+), 97 deletions(-) diff --git a/private/bufnew/bufconfig/generate_config_test.go b/private/bufnew/bufconfig/generate_config_test.go index 8910993497..2a13a3f6f4 100644 --- a/private/bufnew/bufconfig/generate_config_test.go +++ b/private/bufnew/bufconfig/generate_config_test.go @@ -20,6 +20,7 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" ) func TestParseConfigFromExternalV1(t *testing.T) { @@ -278,7 +279,33 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, { - description: "managed mode", + description: "managed_mode_empty", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + }, + }, + Managed: externalGenerateManagedConfigV1{ + Enabled: true, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "go", + out: "go/out", + strategy: GenerateStrategyDirectory, + }, + }, + managedConfig: &generateManagedConfig{}, + }, + }, + { + description: "managed_mode_bools_and_java_package", externalConfig: externalBufGenYAMLFileV1{ Version: "v1", Plugins: []externalGeneratePluginConfigV1{ @@ -296,8 +323,9 @@ func TestParseConfigFromExternalV1(t *testing.T) { Default: "foo", Except: []string{"buf.build/acme/foo", "buf.build/acme/bar"}, Override: map[string]string{ - "buf.build/acme/baz": "baz", - "buf.build/acme/bat": "bat", + "buf.build/acme/weatherapis": "weather", + "buf.build/acme/paymentapis": "payment", + "buf.build/acme/petapis": "pet", }, }, }, @@ -323,19 +351,328 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, overrides: []ManagedOverrideRule{ + &managedOverrideRule{ + fileOption: FileOptionCcEnableArenas, + value: true, + }, + &managedOverrideRule{ + fileOption: FileOptionJavaMultipleFiles, + value: true, + }, + &managedOverrideRule{ + fileOption: FileOptionJavaStringCheckUtf8, + value: true, + }, &managedOverrideRule{ fileOption: FileOptionJavaPackagePrefix, value: "foo", }, + // the next three rules are ordered by their module names + &managedOverrideRule{ + fileOption: FileOptionJavaPackagePrefix, + moduleFullName: "buf.build/acme/paymentapis", + value: "payment", + }, &managedOverrideRule{ fileOption: FileOptionJavaPackagePrefix, - moduleFullName: "buf.build/acme/baz", - value: "baz", + moduleFullName: "buf.build/acme/petapis", + value: "pet", }, &managedOverrideRule{ fileOption: FileOptionJavaPackagePrefix, - moduleFullName: "buf.build/acme/bat", - value: "bat", + moduleFullName: "buf.build/acme/weatherapis", + value: "weather", + }, + }, + }, + }, + }, + { + description: "managed_mode_optimize_for", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + }, + }, + Managed: externalGenerateManagedConfigV1{ + Enabled: true, + OptimizeFor: externalOptimizeForConfigV1{ + Default: "LITE_RUNTIME", + Except: []string{"buf.build/acme/foo"}, + Override: map[string]string{ + "buf.build/acme/petapis": "CODE_SIZE", + "buf.build/acme/paymentapis": "SPEED", + }, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "go", + out: "go/out", + strategy: GenerateStrategyDirectory, + }, + }, + managedConfig: &generateManagedConfig{ + disables: []ManagedDisableRule{ + &managedDisableRule{ + fileOption: FileOptionOptimizeFor, + moduleFullName: "buf.build/acme/foo", + }, + }, + overrides: []ManagedOverrideRule{ + &managedOverrideRule{ + fileOption: FileOptionOptimizeFor, + value: descriptorpb.FileOptions_LITE_RUNTIME, + }, + &managedOverrideRule{ + fileOption: FileOptionOptimizeFor, + moduleFullName: "buf.build/acme/paymentapis", + value: descriptorpb.FileOptions_SPEED, + }, + &managedOverrideRule{ + fileOption: FileOptionOptimizeFor, + moduleFullName: "buf.build/acme/petapis", + value: descriptorpb.FileOptions_CODE_SIZE, + }, + }, + }, + }, + }, + { + description: "managed_mode_go_package_prefix", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + }, + }, + Managed: externalGenerateManagedConfigV1{ + Enabled: true, + GoPackagePrefix: externalGoPackagePrefixConfigV1{ + Default: "foo", + Except: []string{"buf.build/acme/foo"}, + Override: map[string]string{ + "buf.build/acme/petapis": "pet", + }, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "go", + out: "go/out", + strategy: GenerateStrategyDirectory, + }, + }, + managedConfig: &generateManagedConfig{ + disables: []ManagedDisableRule{ + &managedDisableRule{ + fileOption: FileOptionGoPackage, + moduleFullName: "buf.build/acme/foo", + }, + }, + overrides: []ManagedOverrideRule{ + &managedOverrideRule{ + fileOption: FileOptionGoPackagePrefix, + value: "foo", + }, + &managedOverrideRule{ + fileOption: FileOptionGoPackagePrefix, + moduleFullName: "buf.build/acme/petapis", + value: "pet", + }, + }, + }, + }, + }, + { + description: "managed_mode_objc_class_prefix", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + }, + }, + Managed: externalGenerateManagedConfigV1{ + Enabled: true, + ObjcClassPrefix: externalObjcClassPrefixConfigV1{ + Default: "foo", + Except: []string{"buf.build/acme/foo"}, + Override: map[string]string{ + "buf.build/acme/petapis": "pet", + }, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "go", + out: "go/out", + strategy: GenerateStrategyDirectory, + }, + }, + managedConfig: &generateManagedConfig{ + disables: []ManagedDisableRule{ + &managedDisableRule{ + fileOption: FileOptionObjcClassPrefix, + moduleFullName: "buf.build/acme/foo", + }, + }, + overrides: []ManagedOverrideRule{ + &managedOverrideRule{ + fileOption: FileOptionObjcClassPrefix, + value: "foo", + }, + &managedOverrideRule{ + fileOption: FileOptionObjcClassPrefix, + moduleFullName: "buf.build/acme/petapis", + value: "pet", + }, + }, + }, + }, + }, + { + description: "managed_mode_ruby_package", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + }, + }, + Managed: externalGenerateManagedConfigV1{ + Enabled: true, + RubyPackage: externalRubyPackageConfigV1{ + Except: []string{"buf.build/acme/foo"}, + Override: map[string]string{ + "buf.build/acme/petapis": "pet", + }, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "go", + out: "go/out", + strategy: GenerateStrategyDirectory, + }, + }, + managedConfig: &generateManagedConfig{ + disables: []ManagedDisableRule{ + &managedDisableRule{ + fileOption: FileOptionRubyPackage, + moduleFullName: "buf.build/acme/foo", + }, + }, + overrides: []ManagedOverrideRule{ + &managedOverrideRule{ + fileOption: FileOptionRubyPackage, + moduleFullName: "buf.build/acme/petapis", + value: "pet", + }, + }, + }, + }, + }, + { + description: "managed_mode_per_file_override", + externalConfig: externalBufGenYAMLFileV1{ + Version: "v1", + Plugins: []externalGeneratePluginConfigV1{ + { + Plugin: "go", + Out: "go/out", + }, + }, + Managed: externalGenerateManagedConfigV1{ + Enabled: true, + Override: map[string]map[string]string{ + "JAVA_PACKAGE": { + "foo.proto": "foo", + "bar.proto": "bar", + "baz.proto": "baz", + }, + "CC_ENABLE_ARENAS": { + "foo.proto": "false", + "baz.proto": "true", + }, + "OPTIMIZE_FOR": { + "dir/baz.proto": "SPEED", + "dir/foo.proto": "CODE_SIZE", + "dir/bar.proto": "LITE_RUNTIME", + }, + }, + }, + }, + expectedConfig: &generateConfig{ + pluginConfigs: []GeneratePluginConfig{ + &pluginConfig{ + pluginConfigType: PluginConfigTypeLocal, + name: "go", + out: "go/out", + strategy: GenerateStrategyDirectory, + }, + }, + managedConfig: &generateManagedConfig{ + overrides: []ManagedOverrideRule{ + // ordered by file option names and then by file paths + &managedOverrideRule{ + fileOption: FileOptionCcEnableArenas, + path: "baz.proto", + value: true, + }, + &managedOverrideRule{ + fileOption: FileOptionCcEnableArenas, + path: "foo.proto", + value: false, + }, + &managedOverrideRule{ + fileOption: FileOptionJavaPackage, + path: "bar.proto", + value: "bar", + }, + &managedOverrideRule{ + fileOption: FileOptionJavaPackage, + path: "baz.proto", + value: "baz", + }, + &managedOverrideRule{ + fileOption: FileOptionJavaPackage, + path: "foo.proto", + value: "foo", + }, + &managedOverrideRule{ + fileOption: FileOptionOptimizeFor, + path: "dir/bar.proto", + value: descriptorpb.FileOptions_LITE_RUNTIME, + }, + &managedOverrideRule{ + fileOption: FileOptionOptimizeFor, + path: "dir/baz.proto", + value: descriptorpb.FileOptions_SPEED, + }, + &managedOverrideRule{ + fileOption: FileOptionOptimizeFor, + path: "dir/foo.proto", + value: descriptorpb.FileOptions_CODE_SIZE, }, }, }, diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index 06e58ed1e6..fead8fa4b6 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -22,6 +22,7 @@ import ( "github.com/bufbuild/buf/private/bufnew/bufmodule" "github.com/bufbuild/buf/private/pkg/normalpath" + "github.com/bufbuild/buf/private/pkg/slicesext" ) // GenerateManagedConfig is a managed mode configuration. @@ -100,49 +101,49 @@ type generateManagedConfig struct { func newManagedOverrideRuleFromExternalV1( externalConfig externalGenerateManagedConfigV1, ) (GenerateManagedConfig, error) { - if externalConfig.isEmpty() || !externalConfig.Enabled { + if !externalConfig.Enabled { return nil, nil } var ( disables []ManagedDisableRule overrides []ManagedOverrideRule ) - // if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { - // override, err := newFileOptionOverrideRule( - // "", - // "", - // FileOptionCcEnableArenas, - // *externalCCEnableArenas, - // ) - // if err != nil { - // return nil, err - // } - // overrides = append(overrides, override) - // } - // if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { - // override, err := newFileOptionOverrideRule( - // "", - // "", - // FileOptionJavaMultipleFiles, - // *externalJavaMultipleFiles, - // ) - // if err != nil { - // return nil, err - // } - // overrides = append(overrides, override) - // } - // if externalJavaStringCheckUtf8 := externalConfig.JavaStringCheckUtf8; externalJavaStringCheckUtf8 != nil { - // override, err := newFileOptionOverrideRule( - // "", - // "", - // FileOptionJavaStringCheckUtf8, - // *externalJavaStringCheckUtf8, - // ) - // if err != nil { - // return nil, err - // } - // overrides = append(overrides, override) - // } + if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { + override, err := newFileOptionOverrideRule( + "", + "", + FileOptionCcEnableArenas, + *externalCCEnableArenas, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { + override, err := newFileOptionOverrideRule( + "", + "", + FileOptionJavaMultipleFiles, + *externalJavaMultipleFiles, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + if externalJavaStringCheckUtf8 := externalConfig.JavaStringCheckUtf8; externalJavaStringCheckUtf8 != nil { + override, err := newFileOptionOverrideRule( + "", + "", + FileOptionJavaStringCheckUtf8, + *externalJavaStringCheckUtf8, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } if externalJavaPackagePrefix := externalConfig.JavaPackagePrefix; !externalJavaPackagePrefix.isEmpty() { if externalJavaPackagePrefix.Default == "" { // TODO: resolve this: this message has been updated, compared to the one in bufgen/config.go: @@ -159,7 +160,7 @@ func newManagedOverrideRuleFromExternalV1( return nil, err } overrides = append(overrides, defaultOverride) - javaPackagePrefixDisables, javaPackagePrefixOverrides, err := getDisablesAndOverrides( + javaPackagePrefixDisables, javaPackagePrefixOverrides, err := disablesAndOverridesFromExceptAndOverrideV1( FileOptionJavaPackage, externalJavaPackagePrefix.Except, FileOptionJavaPackagePrefix, @@ -172,7 +173,7 @@ func newManagedOverrideRuleFromExternalV1( overrides = append(overrides, javaPackagePrefixOverrides...) } if externalCsharpNamespace := externalConfig.CsharpNamespace; !externalCsharpNamespace.isEmpty() { - csharpNamespaceDisables, csharpNamespaceOverrides, err := getDisablesAndOverrides( + csharpNamespaceDisables, csharpNamespaceOverrides, err := disablesAndOverridesFromExceptAndOverrideV1( FileOptionCsharpNamespace, externalCsharpNamespace.Except, FileOptionCsharpNamespace, @@ -198,7 +199,7 @@ func newManagedOverrideRuleFromExternalV1( return nil, err } overrides = append(overrides, defaultOverride) - optimizeForDisables, optimizeForOverrides, err := getDisablesAndOverrides( + optimizeForDisables, optimizeForOverrides, err := disablesAndOverridesFromExceptAndOverrideV1( FileOptionOptimizeFor, externalOptimizeFor.Except, FileOptionOptimizeFor, @@ -211,7 +212,7 @@ func newManagedOverrideRuleFromExternalV1( overrides = append(overrides, optimizeForOverrides...) } if externalGoPackagePrefix := externalConfig.GoPackagePrefix; !externalGoPackagePrefix.isEmpty() { - if externalGoPackagePrefix.Default != "" { + if externalGoPackagePrefix.Default == "" { return nil, errors.New("go_package_prefix must have a default value") } defaultOverride, err := newFileOptionOverrideRule( @@ -224,7 +225,7 @@ func newManagedOverrideRuleFromExternalV1( return nil, err } overrides = append(overrides, defaultOverride) - goPackagePrefixDisables, goPackagePrefixOverrides, err := getDisablesAndOverrides( + goPackagePrefixDisables, goPackagePrefixOverrides, err := disablesAndOverridesFromExceptAndOverrideV1( FileOptionGoPackage, externalGoPackagePrefix.Except, FileOptionGoPackagePrefix, @@ -250,7 +251,7 @@ func newManagedOverrideRuleFromExternalV1( } overrides = append(overrides, defaultOverride) } - objcClassPrefixDisables, objcClassPrefixOverrides, err := getDisablesAndOverrides( + objcClassPrefixDisables, objcClassPrefixOverrides, err := disablesAndOverridesFromExceptAndOverrideV1( FileOptionObjcClassPrefix, externalObjcClassPrefix.Except, FileOptionObjcClassPrefix, @@ -263,7 +264,7 @@ func newManagedOverrideRuleFromExternalV1( overrides = append(overrides, objcClassPrefixOverrides...) } if externalRubyPackage := externalConfig.RubyPackage; !externalRubyPackage.isEmpty() { - rubyPackageDisables, rubyPackageOverrides, err := getDisablesAndOverrides( + rubyPackageDisables, rubyPackageOverrides, err := disablesAndOverridesFromExceptAndOverrideV1( FileOptionRubyPackage, externalRubyPackage.Except, FileOptionRubyPackage, @@ -275,49 +276,11 @@ func newManagedOverrideRuleFromExternalV1( disables = append(disables, rubyPackageDisables...) overrides = append(overrides, rubyPackageOverrides...) } - for upperCaseFileOption, fileToOverride := range externalConfig.Override { - lowerCaseFileOption := strings.ToLower(upperCaseFileOption) - fileOption, ok := stringToFileOption[lowerCaseFileOption] - if !ok { - return nil, fmt.Errorf("%q is not a valid file option", upperCaseFileOption) - } - for filePath, override := range fileToOverride { - normalizedFilePath, err := normalpath.NormalizeAndValidate(filePath) - if err != nil { - return nil, fmt.Errorf( - "failed to normalize import path: %s provided for override: %s", - filePath, - upperCaseFileOption, - ) - } - if filePath != normalizedFilePath { - return nil, fmt.Errorf( - "override can only take normalized import paths, invalid import path: %s provided for override: %s", - filePath, - upperCaseFileOption, - ) - } - var overrideValue interface{} = override - switch fileOption { - case FileOptionCcEnableArenas, FileOptionJavaMultipleFiles, FileOptionJavaStringCheckUtf8: - parseOverrideValue, err := strconv.ParseBool(override) - if err != nil { - return nil, fmt.Errorf("") - } - overrideValue = parseOverrideValue - } - overrideRule, err := newFileOptionOverrideRule( - filePath, - "", - fileOption, - overrideValue, - ) - if err != nil { - return nil, err - } - overrides = append(overrides, overrideRule) - } + perFileOverrides, err := overrideRulesForPerFileOverridesV1(externalConfig.Override) + if err != nil { + return nil, err } + overrides = append(overrides, perFileOverrides...) return &generateManagedConfig{ disables: disables, overrides: overrides, @@ -497,7 +460,7 @@ func (m *managedOverrideRule) Value() interface{} { func (m *managedOverrideRule) isManagedOverrideRule() {} -func getDisablesAndOverrides( +func disablesAndOverridesFromExceptAndOverrideV1( exceptFileOption FileOption, exceptModuleFullNames []string, overrideFileOption FileOption, @@ -528,7 +491,9 @@ func getDisablesAndOverrides( } disables = append(disables, disable) } - for overrideModuleFullName, overrideValue := range moduleFullNameToOverride { + // Sort by keys for deterministic order. + sortedModuleFullNames := slicesext.MapKeysToSortedSlice(moduleFullNameToOverride) + for _, overrideModuleFullName := range sortedModuleFullNames { if _, err := bufmodule.ParseModuleFullName(overrideModuleFullName); err != nil { return nil, nil, err } @@ -539,7 +504,7 @@ func getDisablesAndOverrides( "", overrideModuleFullName, overrideFileOption, - overrideValue, + moduleFullNameToOverride[overrideModuleFullName], ) if err != nil { return nil, nil, err @@ -548,3 +513,57 @@ func getDisablesAndOverrides( } return disables, overrides, nil } + +func overrideRulesForPerFileOverridesV1( + fileOptionToFilePathToOverride map[string]map[string]string, +) ([]ManagedOverrideRule, error) { + var overrideRules []ManagedOverrideRule + sortedFileOptionStrings := slicesext.MapKeysToSortedSlice(fileOptionToFilePathToOverride) + for _, fileOptionString := range sortedFileOptionStrings { + fileOption, ok := stringToFileOption[strings.ToLower(fileOptionString)] + if !ok { + return nil, fmt.Errorf("%q is not a valid file option", fileOptionString) + } + filePathToOverride := fileOptionToFilePathToOverride[fileOptionString] + sortedFilePaths := slicesext.MapKeysToSortedSlice(filePathToOverride) + for _, filePath := range sortedFilePaths { + normalizedFilePath, err := normalpath.NormalizeAndValidate(filePath) + if err != nil { + return nil, fmt.Errorf( + "%s for override %s is not a valid import path: %w", + filePath, + fileOptionString, + err, + ) + } + if filePath != normalizedFilePath { + return nil, fmt.Errorf( + "import path %s for override %s is not normalized, use %s instead", + filePath, + fileOptionString, + normalizedFilePath, + ) + } + overrideString := filePathToOverride[filePath] + var overrideValue interface{} = overrideString + switch fileOption { + case FileOptionCcEnableArenas, FileOptionJavaMultipleFiles, FileOptionJavaStringCheckUtf8: + overrideValue, err = strconv.ParseBool(overrideString) + if err != nil { + return nil, fmt.Errorf("") + } + } + overrideRule, err := newFileOptionOverrideRule( + filePath, + "", + fileOption, + overrideValue, + ) + if err != nil { + return nil, err + } + overrideRules = append(overrideRules, overrideRule) + } + } + return overrideRules, nil +} diff --git a/private/bufnew/bufconfig/generate_managed_option.go b/private/bufnew/bufconfig/generate_managed_option.go index 848983e7b4..e08e13ad34 100644 --- a/private/bufnew/bufconfig/generate_managed_option.go +++ b/private/bufnew/bufconfig/generate_managed_option.go @@ -212,7 +212,7 @@ func parseOverrideValueOptimizeMode(overrideValue interface{}) (interface{}, err if !ok { return nil, errors.New("must be one of SPEED, CODE_SIZE or LITE_RUNTIME") } - return optimizeMode, nil + return descriptorpb.FileOptions_OptimizeMode(optimizeMode), nil } func parseOverrideValueJSType(override interface{}) (interface{}, error) { From 5f3e185aefd9d364c14ae8495fe4a5e860b20295 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 23 Nov 2023 16:36:54 -0500 Subject: [PATCH 21/66] add logic to bufimagemodify to support the new config; the old stuff are stil there; need testing --- private/buf/bufgen/bufgen.go | 3 +- private/buf/bufgen/features.go | 5 +- private/buf/bufgen/generator.go | 261 ++------- .../bufpkg/bufimage/bufimagemodify/modify.go | 554 ++++++++++++++++++ 4 files changed, 601 insertions(+), 222 deletions(-) create mode 100644 private/bufpkg/bufimage/bufimagemodify/modify.go diff --git a/private/buf/bufgen/bufgen.go b/private/buf/bufgen/bufgen.go index c17157a900..b0d956e53d 100644 --- a/private/buf/bufgen/bufgen.go +++ b/private/buf/bufgen/bufgen.go @@ -23,6 +23,7 @@ import ( "fmt" "strconv" + "github.com/bufbuild/buf/private/bufnew/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" @@ -106,7 +107,7 @@ type Generator interface { Generate( ctx context.Context, container app.EnvStdioContainer, - config *Config, + config bufconfig.GenerateConfig, image bufimage.Image, options ...GenerateOption, ) error diff --git a/private/buf/bufgen/features.go b/private/buf/bufgen/features.go index 4f6aa6dd04..8ffdbaec3f 100644 --- a/private/buf/bufgen/features.go +++ b/private/buf/bufgen/features.go @@ -19,6 +19,7 @@ import ( "sort" "strings" + "github.com/bufbuild/buf/private/bufnew/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/pkg/app" "google.golang.org/protobuf/types/descriptorpb" @@ -60,7 +61,7 @@ func checkRequiredFeatures( container app.StderrContainer, required requiredFeatures, responses []*pluginpb.CodeGeneratorResponse, - configs []*PluginConfig, + configs []bufconfig.GeneratePluginConfig, ) { for responseIndex, response := range responses { if response == nil || response.GetError() != "" { @@ -81,7 +82,7 @@ func checkRequiredFeatures( if len(failed) > 0 { // TODO: plugin config to turn this into an error _, _ = fmt.Fprintf(container.Stderr(), "Warning: plugin %q does not support required features.\n", - configs[responseIndex].PluginName()) + configs[responseIndex].Name()) sort.Slice(failedFeatures, func(i, j int) bool { return failedFeatures[i].Number() < failedFeatures[j].Number() }) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 8491ee4be0..3e0d8c36b2 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -21,9 +21,8 @@ import ( "path/filepath" connect "connectrpc.com/connect" + "github.com/bufbuild/buf/private/bufnew/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify" - "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref" "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" "github.com/bufbuild/buf/private/bufpkg/bufpluginexec" @@ -84,7 +83,7 @@ func newGenerator( func (g *generator) Generate( ctx context.Context, container app.EnvStdioContainer, - config *Config, + config bufconfig.GenerateConfig, image bufimage.Image, options ...GenerateOption, ) error { @@ -107,20 +106,20 @@ func (g *generator) Generate( func (g *generator) generate( ctx context.Context, container app.EnvStdioContainer, - config *Config, + config bufconfig.GenerateConfig, image bufimage.Image, baseOutDirPath string, includeImports bool, includeWellKnownTypes bool, wasmEnabled bool, ) error { - if err := modifyImage(ctx, g.logger, config, image); err != nil { + if err := modifyImage(ctx, g.logger, config.GenerateManagedConfig(), image); err != nil { return err } responses, err := g.execPlugins( ctx, container, - config, + config.GeneratePluginConfigs(), image, includeImports, includeWellKnownTypes, @@ -135,21 +134,21 @@ func (g *generator) generate( g.storageosProvider, appprotoos.ResponseWriterWithCreateOutDirIfNotExists(), ) - for i, pluginConfig := range config.PluginConfigs { - out := pluginConfig.Out + for i, pluginConfig := range config.GeneratePluginConfigs() { + out := pluginConfig.Out() if baseOutDirPath != "" && baseOutDirPath != "." { out = filepath.Join(baseOutDirPath, out) } response := responses[i] if response == nil { - return fmt.Errorf("failed to get plugin response for %s", pluginConfig.PluginName()) + return fmt.Errorf("failed to get plugin response for %s", pluginConfig.Name()) } if err := responseWriter.AddResponse( ctx, response, out, ); err != nil { - return fmt.Errorf("plugin %s: %v", pluginConfig.PluginName(), err) + return fmt.Errorf("plugin %s: %v", pluginConfig.Name(), err) } } if err := responseWriter.Close(); err != nil { @@ -161,7 +160,7 @@ func (g *generator) generate( func (g *generator) execPlugins( ctx context.Context, container app.EnvStdioContainer, - config *Config, + pluginConfigs []bufconfig.GeneratePluginConfig, image bufimage.Image, includeImports bool, includeWellKnownTypes bool, @@ -169,14 +168,14 @@ func (g *generator) execPlugins( ) ([]*pluginpb.CodeGeneratorResponse, error) { imageProvider := newImageProvider(image) // Collect all of the plugin jobs so that they can be executed in parallel. - jobs := make([]func(context.Context) error, 0, len(config.PluginConfigs)) - responses := make([]*pluginpb.CodeGeneratorResponse, len(config.PluginConfigs)) + jobs := make([]func(context.Context) error, 0, len(pluginConfigs)) + responses := make([]*pluginpb.CodeGeneratorResponse, len(pluginConfigs)) requiredFeatures := computeRequiredFeatures(image) - remotePluginConfigTable := make(map[string][]*remotePluginExecArgs, len(config.PluginConfigs)) - for i, pluginConfig := range config.PluginConfigs { + remotePluginConfigTable := make(map[string][]*remotePluginExecArgs, len(pluginConfigs)) + for i, pluginConfig := range pluginConfigs { index := i currentPluginConfig := pluginConfig - remote := currentPluginConfig.GetRemoteHostname() + remote := currentPluginConfig.RemoteHost() if remote != "" { remotePluginConfigTable[remote] = append( remotePluginConfigTable[remote], @@ -210,9 +209,10 @@ func (g *generator) execPlugins( indexedPluginConfigs := indexedPluginConfigs v2Args := make([]*remotePluginExecArgs, 0, len(indexedPluginConfigs)) for _, param := range indexedPluginConfigs { - if param.PluginConfig.Remote != "" { - return nil, fmt.Errorf("invalid plugin reference: %s", param.PluginConfig.Remote) - } + // TODO: check that it's ok to skip it. It's fine because remote is not a thing anymore. + // if param.PluginConfig.Remote != "" { + // return nil, fmt.Errorf("invalid plugin reference: %s", param.PluginConfig.Remote) + // } v2Args = append(v2Args, param) } if len(v2Args) > 0 { @@ -260,10 +260,10 @@ func (g *generator) execPlugins( } return nil, err } - if err := validateResponses(responses, config.PluginConfigs); err != nil { + if err := validateResponses(responses, pluginConfigs); err != nil { return nil, err } - checkRequiredFeatures(container, requiredFeatures, responses, config.PluginConfigs) + checkRequiredFeatures(container, requiredFeatures, responses, pluginConfigs) return responses, nil } @@ -271,18 +271,18 @@ func (g *generator) execLocalPlugin( ctx context.Context, container app.EnvStdioContainer, imageProvider *imageProvider, - pluginConfig *PluginConfig, + pluginConfig bufconfig.GeneratePluginConfig, includeImports bool, includeWellKnownTypes bool, wasmEnabled bool, ) (*pluginpb.CodeGeneratorResponse, error) { - pluginImages, err := imageProvider.GetImages(pluginConfig.Strategy) + pluginImages, err := imageProvider.GetImages(Strategy(pluginConfig.Strategy())) if err != nil { return nil, err } generateOptions := []bufpluginexec.GenerateOption{ - bufpluginexec.GenerateWithPluginPath(pluginConfig.Path...), - bufpluginexec.GenerateWithProtocPath(pluginConfig.ProtocPath), + bufpluginexec.GenerateWithPluginPath(pluginConfig.Path()...), + bufpluginexec.GenerateWithProtocPath(pluginConfig.ProtocPath()), } if wasmEnabled { generateOptions = append( @@ -293,10 +293,10 @@ func (g *generator) execLocalPlugin( response, err := g.pluginexecGenerator.Generate( ctx, container, - pluginConfig.PluginName(), + pluginConfig.Name(), bufimage.ImagesToCodeGeneratorRequests( pluginImages, - pluginConfig.Opt, + pluginConfig.Opt(), nil, includeImports, includeWellKnownTypes, @@ -304,14 +304,14 @@ func (g *generator) execLocalPlugin( generateOptions..., ) if err != nil { - return nil, fmt.Errorf("plugin %s: %v", pluginConfig.PluginName(), err) + return nil, fmt.Errorf("plugin %s: %v", pluginConfig.Name(), err) } return response, nil } type remotePluginExecArgs struct { Index int - PluginConfig *PluginConfig + PluginConfig bufconfig.GeneratePluginConfig } type remotePluginExecutionResult struct { @@ -370,23 +370,23 @@ func (g *generator) execRemotePluginsV2( } func getPluginGenerationRequest( - pluginConfig *PluginConfig, + pluginConfig bufconfig.GeneratePluginConfig, ) (*registryv1alpha1.PluginGenerationRequest, error) { var curatedPluginReference *registryv1alpha1.CuratedPluginReference - if reference, err := bufpluginref.PluginReferenceForString(pluginConfig.Plugin, pluginConfig.Revision); err == nil { + if reference, err := bufpluginref.PluginReferenceForString(pluginConfig.Name(), pluginConfig.Revision()); err == nil { curatedPluginReference = bufplugin.PluginReferenceToProtoCuratedPluginReference(reference) } else { // Try parsing as a plugin identity (no version information) - identity, err := bufpluginref.PluginIdentityForString(pluginConfig.Plugin) + identity, err := bufpluginref.PluginIdentityForString(pluginConfig.Name()) if err != nil { - return nil, fmt.Errorf("invalid remote plugin %q", pluginConfig.Plugin) + return nil, fmt.Errorf("invalid remote plugin %q", pluginConfig.Name()) } curatedPluginReference = bufplugin.PluginIdentityToProtoCuratedPluginReference(identity) } var options []string - if len(pluginConfig.Opt) > 0 { + if len(pluginConfig.Opt()) > 0 { // Only include parameters if they're not empty. - options = []string{pluginConfig.Opt} + options = []string{pluginConfig.Opt()} } return ®istryv1alpha1.PluginGenerationRequest{ PluginReference: curatedPluginReference, @@ -398,188 +398,11 @@ func getPluginGenerationRequest( func modifyImage( ctx context.Context, logger *zap.Logger, - config *Config, + config bufconfig.GenerateManagedConfig, image bufimage.Image, ) error { - if config.ManagedConfig == nil { - // If the config is nil, it implies that the - // user has not enabled managed mode. - return nil - } - sweeper := bufimagemodify.NewFileOptionSweeper() - modifier, err := newModifier(logger, config.ManagedConfig, sweeper) - if err != nil { - return err - } - modifier = bufimagemodify.Merge(modifier, bufimagemodify.ModifierFunc(sweeper.Sweep)) - return modifier.Modify(ctx, image) -} - -func newModifier( - logger *zap.Logger, - managedConfig *ManagedConfig, - sweeper bufimagemodify.Sweeper, -) (bufimagemodify.Modifier, error) { - modifier := bufimagemodify.NewMultiModifier( - bufimagemodify.JavaOuterClassname( - logger, - sweeper, - managedConfig.Override[bufimagemodify.JavaOuterClassNameID], - false, // preserveExistingValue - ), - bufimagemodify.PhpNamespace(logger, sweeper, managedConfig.Override[bufimagemodify.PhpNamespaceID]), - bufimagemodify.PhpMetadataNamespace(logger, sweeper, managedConfig.Override[bufimagemodify.PhpMetadataNamespaceID]), - ) - javaPackagePrefix := &JavaPackagePrefixConfig{Default: bufimagemodify.DefaultJavaPackagePrefix} - if managedConfig.JavaPackagePrefixConfig != nil { - javaPackagePrefix = managedConfig.JavaPackagePrefixConfig - } - javaPackageModifier, err := bufimagemodify.JavaPackage( - logger, - sweeper, - javaPackagePrefix.Default, - javaPackagePrefix.Except, - javaPackagePrefix.Override, - managedConfig.Override[bufimagemodify.JavaPackageID], - ) - if err != nil { - return nil, fmt.Errorf("failed to construct java_package modifier: %w", err) - } - modifier = bufimagemodify.Merge( - modifier, - javaPackageModifier, - ) - javaMultipleFilesValue := bufimagemodify.DefaultJavaMultipleFilesValue - if managedConfig.JavaMultipleFiles != nil { - javaMultipleFilesValue = *managedConfig.JavaMultipleFiles - } - javaMultipleFilesModifier, err := bufimagemodify.JavaMultipleFiles( - logger, - sweeper, - javaMultipleFilesValue, - managedConfig.Override[bufimagemodify.JavaMultipleFilesID], - false, // preserveExistingValue - ) - if err != nil { - return nil, err - } - modifier = bufimagemodify.Merge(modifier, javaMultipleFilesModifier) - if managedConfig.CcEnableArenas != nil { - ccEnableArenasModifier, err := bufimagemodify.CcEnableArenas( - logger, - sweeper, - *managedConfig.CcEnableArenas, - managedConfig.Override[bufimagemodify.CcEnableArenasID], - ) - if err != nil { - return nil, err - } - modifier = bufimagemodify.Merge(modifier, ccEnableArenasModifier) - } - if managedConfig.JavaStringCheckUtf8 != nil { - javaStringCheckUtf8, err := bufimagemodify.JavaStringCheckUtf8( - logger, - sweeper, - *managedConfig.JavaStringCheckUtf8, - managedConfig.Override[bufimagemodify.JavaStringCheckUtf8ID], - ) - if err != nil { - return nil, err - } - modifier = bufimagemodify.Merge(modifier, javaStringCheckUtf8) - } - var ( - csharpNamespaceExcept []bufmoduleref.ModuleIdentity - csharpNamespaceOverride map[bufmoduleref.ModuleIdentity]string - ) - if csharpNameSpaceConfig := managedConfig.CsharpNameSpaceConfig; csharpNameSpaceConfig != nil { - csharpNamespaceExcept = csharpNameSpaceConfig.Except - csharpNamespaceOverride = csharpNameSpaceConfig.Override - } - csharpNamespaceModifier := bufimagemodify.CsharpNamespace( - logger, - sweeper, - csharpNamespaceExcept, - csharpNamespaceOverride, - managedConfig.Override[bufimagemodify.CsharpNamespaceID], - ) - modifier = bufimagemodify.Merge(modifier, csharpNamespaceModifier) - if managedConfig.OptimizeForConfig != nil { - optimizeFor, err := bufimagemodify.OptimizeFor( - logger, - sweeper, - managedConfig.OptimizeForConfig.Default, - managedConfig.OptimizeForConfig.Except, - managedConfig.OptimizeForConfig.Override, - managedConfig.Override[bufimagemodify.OptimizeForID], - ) - if err != nil { - return nil, err - } - modifier = bufimagemodify.Merge( - modifier, - optimizeFor, - ) - } - if managedConfig.GoPackagePrefixConfig != nil { - goPackageModifier, err := bufimagemodify.GoPackage( - logger, - sweeper, - managedConfig.GoPackagePrefixConfig.Default, - managedConfig.GoPackagePrefixConfig.Except, - managedConfig.GoPackagePrefixConfig.Override, - managedConfig.Override[bufimagemodify.GoPackageID], - ) - if err != nil { - return nil, fmt.Errorf("failed to construct go_package modifier: %w", err) - } - modifier = bufimagemodify.Merge( - modifier, - goPackageModifier, - ) - } - var ( - objcClassPrefixDefault string - objcClassPrefixExcept []bufmoduleref.ModuleIdentity - objcClassPrefixOverride map[bufmoduleref.ModuleIdentity]string - ) - if objcClassPrefixConfig := managedConfig.ObjcClassPrefixConfig; objcClassPrefixConfig != nil { - objcClassPrefixDefault = objcClassPrefixConfig.Default - objcClassPrefixExcept = objcClassPrefixConfig.Except - objcClassPrefixOverride = objcClassPrefixConfig.Override - } - objcClassPrefixModifier := bufimagemodify.ObjcClassPrefix( - logger, - sweeper, - objcClassPrefixDefault, - objcClassPrefixExcept, - objcClassPrefixOverride, - managedConfig.Override[bufimagemodify.ObjcClassPrefixID], - ) - modifier = bufimagemodify.Merge( - modifier, - objcClassPrefixModifier, - ) - var ( - rubyPackageExcept []bufmoduleref.ModuleIdentity - rubyPackageOverrides map[bufmoduleref.ModuleIdentity]string - ) - if rubyPackageConfig := managedConfig.RubyPackageConfig; rubyPackageConfig != nil { - rubyPackageExcept = rubyPackageConfig.Except - rubyPackageOverrides = rubyPackageConfig.Override - } - rubyPackageModifier := bufimagemodify.RubyPackage( - logger, - sweeper, - rubyPackageExcept, - rubyPackageOverrides, - managedConfig.Override[bufimagemodify.RubyPackageID], - ) - modifier = bufimagemodify.Merge( - modifier, - rubyPackageModifier, - ) - return modifier, nil + // TODO + return nil } // validateResponses verifies that a response is set for each of the @@ -587,7 +410,7 @@ func newModifier( // plugin. func validateResponses( responses []*pluginpb.CodeGeneratorResponse, - pluginConfigs []*PluginConfig, + pluginConfigs []bufconfig.GeneratePluginConfig, ) error { if len(responses) != len(pluginConfigs) { return fmt.Errorf("unexpected number of responses: expected %d but got %d", len(pluginConfigs), len(responses)) @@ -596,14 +419,14 @@ func validateResponses( for i, response := range responses { pluginConfig := pluginConfigs[i] if response == nil { - return fmt.Errorf("failed to create a response for %q", pluginConfig.PluginName()) + return fmt.Errorf("failed to create a response for %q", pluginConfig.Name()) } pluginResponses = append( pluginResponses, appproto.NewPluginResponse( response, - pluginConfig.PluginName(), - pluginConfig.Out, + pluginConfig.Name(), + pluginConfig.Out(), ), ) } diff --git a/private/bufpkg/bufimage/bufimagemodify/modify.go b/private/bufpkg/bufimage/bufimagemodify/modify.go new file mode 100644 index 0000000000..d64ea6a361 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/modify.go @@ -0,0 +1,554 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "context" + "fmt" + + "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/normalpath" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" +) + +// TODO: this is a temporary file, although it might stay + +// TODO: testcases: + +func Modify( + ctx context.Context, + image bufimage.Image, + config bufconfig.GenerateManagedConfig, +) error { + sweeper := NewFileOptionSweeper() + for _, imageFile := range image.Files() { + if isWellKnownType(ctx, imageFile) { + continue + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaOuterClassname, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverride { + return stringOverride{value: javaOuterClassnameValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, _ stringOverride) string { + return javaOuterClassnameValue(imageFile) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetJavaOuterClassname() + }, + func(options *descriptorpb.FileOptions, value string) { + options.JavaOuterClassname = proto.String(value) + }, + javaOuterClassnamePath, + ); err != nil { + return err + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaPackage, + bufconfig.FileOptionJavaPackagePrefix, + bufconfig.FileOptionJavaPackageSuffix, + func(bufimage.ImageFile) stringOverride { + return stringOverride{prefix: "com"} + }, + getJavaPackageValue, + func(options *descriptorpb.FileOptions) string { + return options.GetJavaPackage() + }, + func(options *descriptorpb.FileOptions, value string) { + options.JavaPackage = proto.String(value) + }, + javaPackagePath, + ); err != nil { + return err + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionGoPackage, + bufconfig.FileOptionGoPackagePrefix, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverride { + return stringOverride{} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { + return GoPackageImportPathForFile(imageFile, stringOverride.prefix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetGoPackage() + }, + func(options *descriptorpb.FileOptions, value string) { + options.GoPackage = proto.String(value) + }, + goPackagePath, + ); err != nil { + return err + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionObjcClassPrefix, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverride { + return stringOverride{value: objcClassPrefixValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, _ stringOverride) string { + return objcClassPrefixValue(imageFile) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetObjcClassPrefix() + }, + func(options *descriptorpb.FileOptions, value string) { + options.ObjcClassPrefix = proto.String(value) + }, + objcClassPrefixPath, + ); err != nil { + return err + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionCsharpNamespace, + bufconfig.FileOptionCsharpNamespacePrefix, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverride { + return stringOverride{value: csharpNamespaceValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { + return getCsharpNamespaceValue(imageFile, stringOverride.prefix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetCsharpNamespace() + }, + func(options *descriptorpb.FileOptions, value string) { + options.CsharpNamespace = proto.String(value) + }, + csharpNamespacePath, + ); err != nil { + return err + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionPhpNamespace, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverride { + return stringOverride{value: phpNamespaceValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, _ stringOverride) string { + return phpNamespaceValue(imageFile) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetPhpNamespace() + }, + func(options *descriptorpb.FileOptions, value string) { + options.PhpNamespace = proto.String(value) + }, + phpNamespacePath, + ); err != nil { + return err + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionPhpMetadataNamespace, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionPhpMetadataNamespaceSuffix, + func(bufimage.ImageFile) stringOverride { + return stringOverride{value: phpMetadataNamespaceValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { + return getPhpMetadataNamespaceValue(imageFile, stringOverride.suffix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetPhpMetadataNamespace() + }, + func(options *descriptorpb.FileOptions, value string) { + options.PhpMetadataNamespace = proto.String(value) + }, + phpMetadataNamespacePath, + ); err != nil { + return err + } + if err := modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionRubyPackage, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionRubyPackageSuffix, + func(bufimage.ImageFile) stringOverride { + return stringOverride{value: rubyPackageValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { + return getRubyPackageValue(imageFile, stringOverride.suffix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetRubyPackage() + }, + func(options *descriptorpb.FileOptions, value string) { + options.RubyPackage = proto.String(value) + }, + rubyPackagePath, + ); err != nil { + return err + } + if err := modifyOption[bool]( + sweeper, + imageFile, + config, + bufconfig.FileOptionCcEnableArenas, + true, + func(options *descriptorpb.FileOptions) bool { + return options.GetCcEnableArenas() + }, + func(options *descriptorpb.FileOptions, value bool) { + options.CcEnableArenas = proto.Bool(value) + }, + ccEnableArenasPath, + ); err != nil { + return err + } + if err := modifyOption[bool]( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaMultipleFiles, + true, + func(options *descriptorpb.FileOptions) bool { + return options.GetJavaMultipleFiles() + }, + func(options *descriptorpb.FileOptions, value bool) { + options.JavaMultipleFiles = proto.Bool(value) + }, + javaMultipleFilesPath, + ); err != nil { + return err + } + if err := modifyOption[bool]( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaStringCheckUtf8, + false, + func(options *descriptorpb.FileOptions) bool { + return options.GetJavaStringCheckUtf8() + }, + func(options *descriptorpb.FileOptions, value bool) { + options.JavaStringCheckUtf8 = proto.Bool(value) + }, + javaStringCheckUtf8Path, + ); err != nil { + return err + } + if err := modifyOption[descriptorpb.FileOptions_OptimizeMode]( + sweeper, + imageFile, + config, + bufconfig.FileOptionOptimizeFor, + descriptorpb.FileOptions_SPEED, + func(options *descriptorpb.FileOptions) descriptorpb.FileOptions_OptimizeMode { + return options.GetOptimizeFor() + }, + func(options *descriptorpb.FileOptions, value descriptorpb.FileOptions_OptimizeMode) { + options.OptimizeFor = value.Enum() + }, + optimizeForPath, + ); err != nil { + return err + } + } + return nil +} + +func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + fileOption bufconfig.FileOption, + // You can set this value to the same as protobuf default, in order to not modify a value by default. + defaultValue T, + getOptionFunc func(*descriptorpb.FileOptions) T, + setOptionFunc func(*descriptorpb.FileOptions, T), + sourceLocationPath []int32, +) error { + if isFileOptionDisabledForFile( + imageFile, + fileOption, + config, + ) { + return nil + } + value := defaultValue + for _, overrideRule := range config.Overrides() { + if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { + continue + } + if overrideRule.FileOption() != fileOption { + continue + } + var ok bool + value, ok = overrideRule.Value().(T) + if !ok { + // This should never happen, since the override rule has been validated. + return fmt.Errorf("invalid value type for %v override: %T", fileOption, overrideRule.Value()) + } + } + descriptor := imageFile.FileDescriptorProto() + if getOptionFunc(descriptor.Options) == value { + // The option is already set to the same value, don't modify or mark it. + return nil + } + if descriptor.Options == nil { + descriptor.Options = &descriptorpb.FileOptions{} + } + setOptionFunc(descriptor.Options, value) + sweeper.mark(imageFile.Path(), sourceLocationPath) + return nil +} + +func modifyStringOption( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + valueOption bufconfig.FileOption, + prefixOption bufconfig.FileOption, + suffixOption bufconfig.FileOption, + defaultOptionsFunc func(bufimage.ImageFile) stringOverride, + valueFunc func(bufimage.ImageFile, stringOverride) string, + getOptionFunc func(*descriptorpb.FileOptions) string, + setOptionFunc func(*descriptorpb.FileOptions, string), + sourceLocationPath []int32, +) error { + override, ok, err := stringOverrideForFile( + imageFile, + config, + defaultOptionsFunc(imageFile), + valueOption, + prefixOption, + suffixOption, + ) + if err != nil { + return err + } + if !ok { + return nil + } + emptyOverride := stringOverride{} + if override == emptyOverride { + return nil + } + // either value is set or one of prefix and suffix is set. + value := override.value + if value == "" { + value = valueFunc(imageFile, override) + } + if value == "" { + return nil + } + descriptor := imageFile.FileDescriptorProto() + if getOptionFunc(descriptor.Options) == value { + // The option is already set to the same value, don't modify or mark it. + return nil + } + if descriptor.Options == nil { + descriptor.Options = &descriptorpb.FileOptions{} + } + setOptionFunc(descriptor.Options, value) + sweeper.mark(imageFile.Path(), sourceLocationPath) + return nil +} + +// TODO: rename to string override options maybe? +type stringOverride struct { + value string + prefix string + suffix string +} + +// Goal: for all file options, do not modify it if it returns an empty string options +// +// For those with a default prefix / suffix (java_package_prefix: com and php_metadata_namespace_suffix: GPBMetadata), +// an empty string modify options means the prefix or suffix has been disabled. Do not modify in this case. +// +// For those with a default computed value (csharp_namespace, ruby_package, php_namespace, objc_class_prefix, java_outer_classname), +// an empty string options means no override has been additionally specified, so compute it in the default way. +// +// For those without a default value / prefix / suffix (go_package), an empty value means no change. + +// If the second returns whehter it's ok to modify this option. If it returns +// false, it means either the config specifies not to modify this option at all, +// or the function returns with an error. Otherwise, it's up to the caller whether +// or not to modify this option. +func stringOverrideForFile( + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + defaultOverride stringOverride, + valueFileOption bufconfig.FileOption, + prefixFileOption bufconfig.FileOption, + suffixFileOption bufconfig.FileOption, +) (stringOverride, bool, error) { + if isFileOptionDisabledForFile(imageFile, valueFileOption, config) { + return stringOverride{}, false, nil + } + override := stringOverride{ + value: defaultOverride.value, + prefix: defaultOverride.prefix, + suffix: defaultOverride.suffix, + } + ignorePrefix := prefixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, prefixFileOption, config) + ignoreSuffix := suffixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, suffixFileOption, config) + for _, overrideRule := range config.Overrides() { + if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { + continue + } + switch overrideRule.FileOption() { + case valueFileOption: + valueString, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", valueFileOption, overrideRule.Value()) + } + override = stringOverride{value: valueString} + case prefixFileOption: + if ignorePrefix { + continue + } + prefixString, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", prefixFileOption, overrideRule.Value()) + } + override.prefix = prefixString + // Do not clear suffix here, because if the last two overrides are suffix and prefix, both are used. + override.value = "" + case suffixFileOption: + if ignoreSuffix { + continue + } + suffixString, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", suffixFileOption, overrideRule.Value()) + } + override.suffix = suffixString + // Do not clear prefix here, because if the last two overrides are suffix and prefix, both are used. + override.value = "" + } + } + return override, true, nil +} + +func isFileOptionDisabledForFile( + imageFile bufimage.ImageFile, + fileOption bufconfig.FileOption, + config bufconfig.GenerateManagedConfig, +) bool { + for _, disableRule := range config.Disables() { + if disableRule.FileOption() != bufconfig.FileOptionUnspecified && disableRule.FileOption() != fileOption { + continue + } + if !fileMatchConfig(imageFile, disableRule.Path(), disableRule.ModuleFullName()) { + continue + } + return true + } + return false +} + +func getValueForFileOption( + imageFile bufimage.ImageFile, + fileOption bufconfig.FileOption, + config bufconfig.GenerateManagedConfig, +) { +} + +func fileMatchConfig( + imageFile bufimage.ImageFile, + requiredPath string, + requiredModuleFullName string, +) bool { + if requiredPath != "" && !normalpath.EqualsOrContainsPath(requiredPath, imageFile.Path(), normalpath.Relative) { + return false + } + if requiredModuleFullName != "" && (imageFile.ModuleFullName() == nil || imageFile.ModuleFullName().String() != requiredModuleFullName) { + return false + } + return true +} + +// TODO: rename these helpers + +func getJavaPackageValue(imageFile bufimage.ImageFile, stringOverride stringOverride) string { + if pkg := imageFile.FileDescriptorProto().GetPackage(); pkg != "" { + if stringOverride.prefix != "" { + pkg = stringOverride.prefix + "." + pkg + } + if stringOverride.suffix != "" { + pkg = pkg + "." + stringOverride.suffix + } + return pkg + } + return "" +} + +func getCsharpNamespaceValue(imageFile bufimage.ImageFile, prefix string) string { + namespace := csharpNamespaceValue(imageFile) + if namespace == "" { + return "" + } + if prefix == "" { + return namespace + } + return prefix + "." + namespace +} + +func getPhpMetadataNamespaceValue(imageFile bufimage.ImageFile, suffix string) string { + namespace := phpNamespaceValue(imageFile) + if namespace == "" { + return "" + } + if suffix == "" { + return namespace + } + return namespace + `\` + suffix +} + +func getRubyPackageValue(imageFile bufimage.ImageFile, suffix string) string { + rubyPackage := rubyPackageValue(imageFile) + if rubyPackage == "" { + return "" + } + if suffix == "" { + return rubyPackage + } + return rubyPackage + "::" + suffix +} From 278975640d8f0723e1e7edc1116b0ef446f3bebf Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 23 Nov 2023 16:38:35 -0500 Subject: [PATCH 22/66] added TODO --- private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go index 52f8f83c13..6b9b1321b3 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go @@ -21,14 +21,16 @@ import ( "strconv" "strings" - "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufnew/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/gen/data/datawkt" "github.com/bufbuild/buf/private/pkg/protoversion" "go.uber.org/zap" "google.golang.org/protobuf/types/descriptorpb" ) +// TODO: remove code dealing with the old config (maps) + // Modifier modifies Images. type Modifier interface { // Modify modifies the Image. From a3fcd61ddfc84ab7db2b9051e469b48466b326bd Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 23 Nov 2023 16:40:06 -0500 Subject: [PATCH 23/66] rename file to a more temporary name --- .../bufimage/bufimagemodify/{modify.go => new_modify.go} | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) rename private/bufpkg/bufimage/bufimagemodify/{modify.go => new_modify.go} (99%) diff --git a/private/bufpkg/bufimage/bufimagemodify/modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go similarity index 99% rename from private/bufpkg/bufimage/bufimagemodify/modify.go rename to private/bufpkg/bufimage/bufimagemodify/new_modify.go index d64ea6a361..2ec08abd1d 100644 --- a/private/bufpkg/bufimage/bufimagemodify/modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -25,9 +25,7 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) -// TODO: this is a temporary file, although it might stay - -// TODO: testcases: +// TODO: this is a temporary file, although it might stay. Need to rename the file at least. func Modify( ctx context.Context, From 15f08011631dc48add186e3ced0549a297cce245 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 23 Nov 2023 18:27:14 -0500 Subject: [PATCH 24/66] checkpoint; going to move some stuff from bufimagemodify to bufgen --- private/buf/bufgen/generator.go | 8 +- .../bufconfig/generate_managed_config.go | 14 + .../bufimagemodify/bufimagemodify_test.go | 18 + .../bufimage/bufimagemodify/new_modify.go | 267 +------------- .../bufimage/bufimagemodify/new_modify2.go | 326 ++++++++++++++++++ .../bufimagemodify/new_modify_test.go | 47 +++ 6 files changed, 412 insertions(+), 268 deletions(-) create mode 100644 private/bufpkg/bufimage/bufimagemodify/new_modify2.go create mode 100644 private/bufpkg/bufimage/bufimagemodify/new_modify_test.go diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index 3e0d8c36b2..e625a9249c 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -23,6 +23,7 @@ import ( connect "connectrpc.com/connect" "github.com/bufbuild/buf/private/bufnew/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify" "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" "github.com/bufbuild/buf/private/bufpkg/bufpluginexec" @@ -401,8 +402,11 @@ func modifyImage( config bufconfig.GenerateManagedConfig, image bufimage.Image, ) error { - // TODO - return nil + return bufimagemodify.Modify( + ctx, + image, + config, + ) } // validateResponses verifies that a response is set for each of the diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index fead8fa4b6..e219856571 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -25,6 +25,8 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" ) +// TODO: check normalize(path) == path for disable and override paths. + // GenerateManagedConfig is a managed mode configuration. type GenerateManagedConfig interface { // Disables returns the disable rules in the configuration. @@ -35,6 +37,18 @@ type GenerateManagedConfig interface { isGenerateManagedConfig() } +// // TODO/Note: this is exported only for testing. Might want to reconsider this approach. +// // NewGenerateManagedConfig returns a new GenerateManagedConfig. +// func NewGenerateManagedConfig( +// disables []ManagedDisableRule, +// overrides []ManagedOverrideRule, +// ) GenerateManagedConfig { +// return &generateManagedConfig{ +// disables: disables, +// overrides: overrides, +// } +// } + // ManagedDisableRule is a disable rule. A disable rule describes: // // - The options to not modify. If not specified, it means all options (both diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go index 11e58762bc..93a3d802c2 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/bufbuild/buf/private/bufnew/bufconfig" "github.com/bufbuild/buf/private/bufnew/bufmodule/bufmoduletest" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagebuild" @@ -91,3 +92,20 @@ func testGetImage(t *testing.T, dirPath string, includeSourceInfo bool) bufimage require.Empty(t, annotations) return image } + +type testDisable struct { + path string + moduleFullName string + fieldName string + fileOption bufconfig.FileOption + fieldOption bufconfig.FieldOption +} + +type testOverride struct { + path string + moduleFullName string + fieldName string + fileOption bufconfig.FileOption + fieldOption bufconfig.FieldOption + value interface{} +} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go index 2ec08abd1d..6da72add32 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -16,11 +16,9 @@ package bufimagemodify import ( "context" - "fmt" "github.com/bufbuild/buf/private/bufnew/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/normalpath" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/descriptorpb" ) @@ -37,6 +35,7 @@ func Modify( if isWellKnownType(ctx, imageFile) { continue } + // TODO: order them like before or by name or by field number if err := modifyStringOption( sweeper, imageFile, @@ -286,267 +285,3 @@ func Modify( } return nil } - -func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( - sweeper Sweeper, - imageFile bufimage.ImageFile, - config bufconfig.GenerateManagedConfig, - fileOption bufconfig.FileOption, - // You can set this value to the same as protobuf default, in order to not modify a value by default. - defaultValue T, - getOptionFunc func(*descriptorpb.FileOptions) T, - setOptionFunc func(*descriptorpb.FileOptions, T), - sourceLocationPath []int32, -) error { - if isFileOptionDisabledForFile( - imageFile, - fileOption, - config, - ) { - return nil - } - value := defaultValue - for _, overrideRule := range config.Overrides() { - if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { - continue - } - if overrideRule.FileOption() != fileOption { - continue - } - var ok bool - value, ok = overrideRule.Value().(T) - if !ok { - // This should never happen, since the override rule has been validated. - return fmt.Errorf("invalid value type for %v override: %T", fileOption, overrideRule.Value()) - } - } - descriptor := imageFile.FileDescriptorProto() - if getOptionFunc(descriptor.Options) == value { - // The option is already set to the same value, don't modify or mark it. - return nil - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - setOptionFunc(descriptor.Options, value) - sweeper.mark(imageFile.Path(), sourceLocationPath) - return nil -} - -func modifyStringOption( - sweeper Sweeper, - imageFile bufimage.ImageFile, - config bufconfig.GenerateManagedConfig, - valueOption bufconfig.FileOption, - prefixOption bufconfig.FileOption, - suffixOption bufconfig.FileOption, - defaultOptionsFunc func(bufimage.ImageFile) stringOverride, - valueFunc func(bufimage.ImageFile, stringOverride) string, - getOptionFunc func(*descriptorpb.FileOptions) string, - setOptionFunc func(*descriptorpb.FileOptions, string), - sourceLocationPath []int32, -) error { - override, ok, err := stringOverrideForFile( - imageFile, - config, - defaultOptionsFunc(imageFile), - valueOption, - prefixOption, - suffixOption, - ) - if err != nil { - return err - } - if !ok { - return nil - } - emptyOverride := stringOverride{} - if override == emptyOverride { - return nil - } - // either value is set or one of prefix and suffix is set. - value := override.value - if value == "" { - value = valueFunc(imageFile, override) - } - if value == "" { - return nil - } - descriptor := imageFile.FileDescriptorProto() - if getOptionFunc(descriptor.Options) == value { - // The option is already set to the same value, don't modify or mark it. - return nil - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - setOptionFunc(descriptor.Options, value) - sweeper.mark(imageFile.Path(), sourceLocationPath) - return nil -} - -// TODO: rename to string override options maybe? -type stringOverride struct { - value string - prefix string - suffix string -} - -// Goal: for all file options, do not modify it if it returns an empty string options -// -// For those with a default prefix / suffix (java_package_prefix: com and php_metadata_namespace_suffix: GPBMetadata), -// an empty string modify options means the prefix or suffix has been disabled. Do not modify in this case. -// -// For those with a default computed value (csharp_namespace, ruby_package, php_namespace, objc_class_prefix, java_outer_classname), -// an empty string options means no override has been additionally specified, so compute it in the default way. -// -// For those without a default value / prefix / suffix (go_package), an empty value means no change. - -// If the second returns whehter it's ok to modify this option. If it returns -// false, it means either the config specifies not to modify this option at all, -// or the function returns with an error. Otherwise, it's up to the caller whether -// or not to modify this option. -func stringOverrideForFile( - imageFile bufimage.ImageFile, - config bufconfig.GenerateManagedConfig, - defaultOverride stringOverride, - valueFileOption bufconfig.FileOption, - prefixFileOption bufconfig.FileOption, - suffixFileOption bufconfig.FileOption, -) (stringOverride, bool, error) { - if isFileOptionDisabledForFile(imageFile, valueFileOption, config) { - return stringOverride{}, false, nil - } - override := stringOverride{ - value: defaultOverride.value, - prefix: defaultOverride.prefix, - suffix: defaultOverride.suffix, - } - ignorePrefix := prefixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, prefixFileOption, config) - ignoreSuffix := suffixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, suffixFileOption, config) - for _, overrideRule := range config.Overrides() { - if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { - continue - } - switch overrideRule.FileOption() { - case valueFileOption: - valueString, ok := overrideRule.Value().(string) - if !ok { - // This should never happen, since the override rule has been validated. - return stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", valueFileOption, overrideRule.Value()) - } - override = stringOverride{value: valueString} - case prefixFileOption: - if ignorePrefix { - continue - } - prefixString, ok := overrideRule.Value().(string) - if !ok { - // This should never happen, since the override rule has been validated. - return stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", prefixFileOption, overrideRule.Value()) - } - override.prefix = prefixString - // Do not clear suffix here, because if the last two overrides are suffix and prefix, both are used. - override.value = "" - case suffixFileOption: - if ignoreSuffix { - continue - } - suffixString, ok := overrideRule.Value().(string) - if !ok { - // This should never happen, since the override rule has been validated. - return stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", suffixFileOption, overrideRule.Value()) - } - override.suffix = suffixString - // Do not clear prefix here, because if the last two overrides are suffix and prefix, both are used. - override.value = "" - } - } - return override, true, nil -} - -func isFileOptionDisabledForFile( - imageFile bufimage.ImageFile, - fileOption bufconfig.FileOption, - config bufconfig.GenerateManagedConfig, -) bool { - for _, disableRule := range config.Disables() { - if disableRule.FileOption() != bufconfig.FileOptionUnspecified && disableRule.FileOption() != fileOption { - continue - } - if !fileMatchConfig(imageFile, disableRule.Path(), disableRule.ModuleFullName()) { - continue - } - return true - } - return false -} - -func getValueForFileOption( - imageFile bufimage.ImageFile, - fileOption bufconfig.FileOption, - config bufconfig.GenerateManagedConfig, -) { -} - -func fileMatchConfig( - imageFile bufimage.ImageFile, - requiredPath string, - requiredModuleFullName string, -) bool { - if requiredPath != "" && !normalpath.EqualsOrContainsPath(requiredPath, imageFile.Path(), normalpath.Relative) { - return false - } - if requiredModuleFullName != "" && (imageFile.ModuleFullName() == nil || imageFile.ModuleFullName().String() != requiredModuleFullName) { - return false - } - return true -} - -// TODO: rename these helpers - -func getJavaPackageValue(imageFile bufimage.ImageFile, stringOverride stringOverride) string { - if pkg := imageFile.FileDescriptorProto().GetPackage(); pkg != "" { - if stringOverride.prefix != "" { - pkg = stringOverride.prefix + "." + pkg - } - if stringOverride.suffix != "" { - pkg = pkg + "." + stringOverride.suffix - } - return pkg - } - return "" -} - -func getCsharpNamespaceValue(imageFile bufimage.ImageFile, prefix string) string { - namespace := csharpNamespaceValue(imageFile) - if namespace == "" { - return "" - } - if prefix == "" { - return namespace - } - return prefix + "." + namespace -} - -func getPhpMetadataNamespaceValue(imageFile bufimage.ImageFile, suffix string) string { - namespace := phpNamespaceValue(imageFile) - if namespace == "" { - return "" - } - if suffix == "" { - return namespace - } - return namespace + `\` + suffix -} - -func getRubyPackageValue(imageFile bufimage.ImageFile, suffix string) string { - rubyPackage := rubyPackageValue(imageFile) - if rubyPackage == "" { - return "" - } - if suffix == "" { - return rubyPackage - } - return rubyPackage + "::" + suffix -} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go new file mode 100644 index 0000000000..c6c064aa59 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go @@ -0,0 +1,326 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "fmt" + + "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/normalpath" + "google.golang.org/protobuf/types/descriptorpb" +) + +// TODO: these might be useful, keeping them around for now + +// type managedConfig interface { +// Disables() []managedDisableRule +// Overrides() []managedOverrideRule +// } + +// type managedDisableRule interface { +// Path() string +// ModuleFullName() string +// FieldName() string +// FileOption() bufconfig.FileOption +// FieldOption() bufconfig.FieldOption +// } + +// type managedOverrideRule interface { +// Path() string +// ModuleFullName() string +// FieldName() string +// FileOption() bufconfig.FileOption +// FieldOption() bufconfig.FieldOption +// Value() interface{} + +// isManagedOverrideRule() +// } + +func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + fileOption bufconfig.FileOption, + // You can set this value to the same as protobuf default, in order to not modify a value by default. + defaultValue T, + getOptionFunc func(*descriptorpb.FileOptions) T, + setOptionFunc func(*descriptorpb.FileOptions, T), + sourceLocationPath []int32, +) error { + value := defaultValue + override, disable, err := overrideAndDisableFromConfig[T]( + imageFile, + config, + fileOption, + ) + if err != nil { + return err + } + if disable { + return nil + } + if override != nil { + value = *override + } + descriptor := imageFile.FileDescriptorProto() + if getOptionFunc(descriptor.Options) == value { + // The option is already set to the same value, don't modify or mark it. + return nil + } + if descriptor.Options == nil { + descriptor.Options = &descriptorpb.FileOptions{} + } + setOptionFunc(descriptor.Options, value) + sweeper.mark(imageFile.Path(), sourceLocationPath) + return nil +} + +// returns the override value and whether managed mode is DISABLED for this file for this file option. +func overrideAndDisableFromConfig[T bool | descriptorpb.FileOptions_OptimizeMode]( + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + fileOption bufconfig.FileOption, +) (*T, bool, error) { + if isFileOptionDisabledForFile( + imageFile, + fileOption, + config, + ) { + return nil, true, nil + } + var override *T + for _, overrideRule := range config.Overrides() { + if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { + continue + } + if overrideRule.FileOption() != fileOption { + continue + } + value, ok := overrideRule.Value().(T) + if !ok { + // This should never happen, since the override rule has been validated. + return nil, false, fmt.Errorf("invalid value type for %v override: %T", fileOption, overrideRule.Value()) + } + override = &value + } + return override, false, nil +} + +func modifyStringOption( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + valueOption bufconfig.FileOption, + prefixOption bufconfig.FileOption, + suffixOption bufconfig.FileOption, + // todo: options? unify + defaultOptionsFunc func(bufimage.ImageFile) stringOverride, + valueFunc func(bufimage.ImageFile, stringOverride) string, + getOptionFunc func(*descriptorpb.FileOptions) string, + setOptionFunc func(*descriptorpb.FileOptions, string), + sourceLocationPath []int32, +) error { + modifyOptions := defaultOptionsFunc(imageFile) + override, disable, err := stringOverrideAndDisableFromConfig( + imageFile, + config, + valueOption, + prefixOption, + suffixOption, + ) + if err != nil { + return err + } + if disable { + return nil + } + if override != nil { + if override.value != "" { + modifyOptions.value = override.value + } + if override.prefix != "" { + modifyOptions.prefix = override.prefix + } + if override.suffix != "" { + modifyOptions.suffix = override.suffix + } + } + // either value is set or one of prefix and suffix is set. + value := modifyOptions.value + if value == "" { + value = valueFunc(imageFile, modifyOptions) + } + if value == "" { + return nil + } + descriptor := imageFile.FileDescriptorProto() + if getOptionFunc(descriptor.Options) == value { + // The option is already set to the same value, don't modify or mark it. + return nil + } + if descriptor.Options == nil { + descriptor.Options = &descriptorpb.FileOptions{} + } + setOptionFunc(descriptor.Options, value) + sweeper.mark(imageFile.Path(), sourceLocationPath) + return nil +} + +// the first value is nil when no override rule is matched +// returns the override value and whether managed mode is DISABLED for this file for this file option. +func stringOverrideAndDisableFromConfig( + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + valueFileOption bufconfig.FileOption, + prefixFileOption bufconfig.FileOption, + suffixFileOption bufconfig.FileOption, +) (*stringOverride, bool, error) { + if isFileOptionDisabledForFile(imageFile, valueFileOption, config) { + return nil, true, nil + } + var override *stringOverride + ignorePrefix := prefixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, prefixFileOption, config) + ignoreSuffix := suffixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, suffixFileOption, config) + for _, overrideRule := range config.Overrides() { + if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { + continue + } + switch overrideRule.FileOption() { + case valueFileOption: + valueString, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return nil, false, fmt.Errorf("invalid value type for %v override: %T", valueFileOption, overrideRule.Value()) + } + override = &stringOverride{value: valueString} + case prefixFileOption: + if ignorePrefix { + continue + } + prefixString, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return &stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", prefixFileOption, overrideRule.Value()) + } + // Keep the suffix if the last two overrides are suffix and prefix. + override = &stringOverride{ + prefix: prefixString, + suffix: override.suffix, + } + case suffixFileOption: + if ignoreSuffix { + continue + } + suffixString, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return &stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", suffixFileOption, overrideRule.Value()) + } + // Keep the prefix if the last two overrides are suffix and prefix. + override = &stringOverride{ + prefix: override.prefix, + suffix: suffixString, + } + } + } + return override, false, nil +} + +// TODO: rename to string override options maybe? +type stringOverride struct { + value string + prefix string + suffix string +} + +func isFileOptionDisabledForFile( + imageFile bufimage.ImageFile, + fileOption bufconfig.FileOption, + config bufconfig.GenerateManagedConfig, +) bool { + for _, disableRule := range config.Disables() { + if disableRule.FileOption() != bufconfig.FileOptionUnspecified && disableRule.FileOption() != fileOption { + continue + } + if !fileMatchConfig(imageFile, disableRule.Path(), disableRule.ModuleFullName()) { + continue + } + return true + } + return false +} + +func fileMatchConfig( + imageFile bufimage.ImageFile, + requiredPath string, + requiredModuleFullName string, +) bool { + if requiredPath != "" && !normalpath.EqualsOrContainsPath(requiredPath, imageFile.Path(), normalpath.Relative) { + return false + } + if requiredModuleFullName != "" && (imageFile.ModuleFullName() == nil || imageFile.ModuleFullName().String() != requiredModuleFullName) { + return false + } + return true +} + +// TODO: rename these helpers + +func getJavaPackageValue(imageFile bufimage.ImageFile, stringOverride stringOverride) string { + if pkg := imageFile.FileDescriptorProto().GetPackage(); pkg != "" { + if stringOverride.prefix != "" { + pkg = stringOverride.prefix + "." + pkg + } + if stringOverride.suffix != "" { + pkg = pkg + "." + stringOverride.suffix + } + return pkg + } + return "" +} + +func getCsharpNamespaceValue(imageFile bufimage.ImageFile, prefix string) string { + namespace := csharpNamespaceValue(imageFile) + if namespace == "" { + return "" + } + if prefix == "" { + return namespace + } + return prefix + "." + namespace +} + +func getPhpMetadataNamespaceValue(imageFile bufimage.ImageFile, suffix string) string { + namespace := phpNamespaceValue(imageFile) + if namespace == "" { + return "" + } + if suffix == "" { + return namespace + } + return namespace + `\` + suffix +} + +func getRubyPackageValue(imageFile bufimage.ImageFile, suffix string) string { + rubyPackage := rubyPackageValue(imageFile) + if rubyPackage == "" { + return "" + } + if suffix == "" { + return rubyPackage + } + return rubyPackage + "::" + suffix +} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go new file mode 100644 index 0000000000..69ff29c517 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go @@ -0,0 +1,47 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "testing" + + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "google.golang.org/protobuf/types/descriptorpb" +) + +func TestModify(t *testing.T) { + t.Parallel() + testcases := []struct { + description string + moduleNameToDirpath string + config bufconfig.Config + pathToExpectedOptions map[string]*descriptorpb.FileOptions + }{} + for _, testcase := range testcases { + testcase := testcase + t.Run(testcase.description, func(t *testing.T) { + }) + } +} + +func TestSweepFileOption(t *testing.T) { + t.Parallel() + // TODO +} + +func TestSweepFieldOption(t *testing.T) { + t.Parallel() + // TODO in v2 +} From 0e6539f05763b432940891c0f3fec4a653015278 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 23 Nov 2023 19:31:50 -0500 Subject: [PATCH 25/66] make New..Override public --- .../bufconfig/generate_managed_config.go | 251 +++++++++--------- 1 file changed, 132 insertions(+), 119 deletions(-) diff --git a/private/bufnew/bufconfig/generate_managed_config.go b/private/bufnew/bufconfig/generate_managed_config.go index e219856571..8fe52ad5c9 100644 --- a/private/bufnew/bufconfig/generate_managed_config.go +++ b/private/bufnew/bufconfig/generate_managed_config.go @@ -37,17 +37,16 @@ type GenerateManagedConfig interface { isGenerateManagedConfig() } -// // TODO/Note: this is exported only for testing. Might want to reconsider this approach. -// // NewGenerateManagedConfig returns a new GenerateManagedConfig. -// func NewGenerateManagedConfig( -// disables []ManagedDisableRule, -// overrides []ManagedOverrideRule, -// ) GenerateManagedConfig { -// return &generateManagedConfig{ -// disables: disables, -// overrides: overrides, -// } -// } +// NewGenerateManagedConfig returns a new GenerateManagedConfig. +func NewGenerateManagedConfig( + disables []ManagedDisableRule, + overrides []ManagedOverrideRule, +) GenerateManagedConfig { + return &generateManagedConfig{ + disables: disables, + overrides: overrides, + } +} // ManagedDisableRule is a disable rule. A disable rule describes: // @@ -79,6 +78,43 @@ type ManagedDisableRule interface { isManagedDisableRule() } +// NewDisableRule returns a new ManagedDisableRule +func NewDisableRule( + path string, + moduleFullName string, + fieldName string, + fileOption FileOption, + fieldOption FieldOption, +) (ManagedDisableRule, error) { + if path != "" && normalpath.Normalize(path) != path { + // TODO: do we want to show words like 'normalized' to users? + return nil, fmt.Errorf("path must be normalized: %s", path) + } + if path == "" && moduleFullName == "" && fieldName == "" && fileOption == FileOptionUnspecified && fieldOption == FieldOptionUnspecified { + // This should never happen to parsing configs from provided by users. + return nil, errors.New("empty disable rule is not allowed") + } + if fieldName != "" && fileOption != FileOptionUnspecified { + return nil, errors.New("cannot disable a file option for a field") + } + if fileOption != FileOptionUnspecified && fieldOption != FieldOptionUnspecified { + return nil, errors.New("at most one of file_option and field_option can be specified") + } + // TODO: validate path here? Was it validated in v1/main? + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, err + } + } + return &managedDisableRule{ + path: path, + moduleFullName: moduleFullName, + fieldName: fieldName, + fileOption: fileOption, + fieldOption: fieldOption, + }, nil +} + // ManagedOverrideRule is an override rule. An override describes: // // - The options to modify. Exactly one of FileOption and FieldOption is not empty. @@ -107,6 +143,81 @@ type ManagedOverrideRule interface { isManagedOverrideRule() } +// NewFieldOptionOverrideRule returns an OverrideRule for a field option. +func NewFileOptionOverrideRule( + path string, + moduleFullName string, + fileOption FileOption, + value interface{}, +) (*managedOverrideRule, error) { + if path != "" && normalpath.Normalize(path) != path { + // TODO: do we want to show words like 'normalized' to users? + return nil, fmt.Errorf("path must be normalized: %s", path) + } + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, err + } + } + // All valid file options have a parse func. This lookup implicitly validates the option. + parseOverrideValueFunc, ok := fileOptionToParseOverrideValueFunc[fileOption] + if !ok { + return nil, fmt.Errorf("invalid fileOption: %v", fileOption) + } + if value == nil { + return nil, fmt.Errorf("value must be specified for override") + } + parsedValue, err := parseOverrideValueFunc(value) + if err != nil { + return nil, fmt.Errorf("invalid value %v for %v: %w", value, fileOption, err) + } + return &managedOverrideRule{ + path: path, + moduleFullName: moduleFullName, + fileOption: fileOption, + value: parsedValue, + }, nil +} + +// NewFieldOptionOverrideRule returns an OverrideRule for a field option. +func NewFieldOptionOverrideRule( + path string, + moduleFullName string, + fieldName string, + fieldOption FieldOption, + value interface{}, +) (ManagedOverrideRule, error) { + if path != "" && normalpath.Normalize(path) != path { + // TODO: do we want to show words like 'normalized' to users? + return nil, fmt.Errorf("path must be normalized: %s", path) + } + // TODO: validate path here? Was it validated in v1/main? + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, err + } + } + // All valid field options have a parse func. This lookup implicitly validates the option. + parseOverrideValueFunc, ok := fieldOptionToParseOverrideValueFunc[fieldOption] + if !ok { + return nil, fmt.Errorf("invalid fieldOption: %v", fieldOption) + } + if value == nil { + return nil, fmt.Errorf("value must be specified for override") + } + parsedValue, err := parseOverrideValueFunc(value) + if err != nil { + return nil, fmt.Errorf("invalid value %v for %v: %w", value, fieldOption, err) + } + return &managedOverrideRule{ + path: path, + moduleFullName: moduleFullName, + fieldName: fieldName, + fieldOption: fieldOption, + value: parsedValue, + }, nil +} + type generateManagedConfig struct { disables []ManagedDisableRule overrides []ManagedOverrideRule @@ -123,7 +234,7 @@ func newManagedOverrideRuleFromExternalV1( overrides []ManagedOverrideRule ) if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { - override, err := newFileOptionOverrideRule( + override, err := NewFileOptionOverrideRule( "", "", FileOptionCcEnableArenas, @@ -135,7 +246,7 @@ func newManagedOverrideRuleFromExternalV1( overrides = append(overrides, override) } if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { - override, err := newFileOptionOverrideRule( + override, err := NewFileOptionOverrideRule( "", "", FileOptionJavaMultipleFiles, @@ -147,7 +258,7 @@ func newManagedOverrideRuleFromExternalV1( overrides = append(overrides, override) } if externalJavaStringCheckUtf8 := externalConfig.JavaStringCheckUtf8; externalJavaStringCheckUtf8 != nil { - override, err := newFileOptionOverrideRule( + override, err := NewFileOptionOverrideRule( "", "", FileOptionJavaStringCheckUtf8, @@ -164,7 +275,7 @@ func newManagedOverrideRuleFromExternalV1( // "java_package_prefix setting requires a default value" return nil, errors.New("java_package_prefix must have a default value") } - defaultOverride, err := newFileOptionOverrideRule( + defaultOverride, err := NewFileOptionOverrideRule( "", "", FileOptionJavaPackagePrefix, @@ -203,7 +314,7 @@ func newManagedOverrideRuleFromExternalV1( if externalOptimizeFor.Default == "" { return nil, errors.New("optimize_for must have a default value") } - defaultOverride, err := newFileOptionOverrideRule( + defaultOverride, err := NewFileOptionOverrideRule( "", "", FileOptionOptimizeFor, @@ -229,7 +340,7 @@ func newManagedOverrideRuleFromExternalV1( if externalGoPackagePrefix.Default == "" { return nil, errors.New("go_package_prefix must have a default value") } - defaultOverride, err := newFileOptionOverrideRule( + defaultOverride, err := NewFileOptionOverrideRule( "", "", FileOptionGoPackagePrefix, @@ -254,7 +365,7 @@ func newManagedOverrideRuleFromExternalV1( if externalObjcClassPrefix := externalConfig.ObjcClassPrefix; !externalObjcClassPrefix.isEmpty() { if externalObjcClassPrefix.Default != "" { // objc class prefix allows empty default - defaultOverride, err := newFileOptionOverrideRule( + defaultOverride, err := NewFileOptionOverrideRule( "", "", FileOptionObjcClassPrefix, @@ -319,38 +430,6 @@ type managedDisableRule struct { fieldOption FieldOption } -func newDisableRule( - path string, - moduleFullName string, - fieldName string, - fileOption FileOption, - fieldOption FieldOption, -) (*managedDisableRule, error) { - if path == "" && moduleFullName == "" && fieldName == "" && fileOption == FileOptionUnspecified && fieldOption == FieldOptionUnspecified { - // This should never happen to parsing configs from provided by users. - return nil, errors.New("empty disable rule is not allowed") - } - if fileOption != FileOptionUnspecified && fieldOption != FieldOptionUnspecified { - return nil, errors.New("at most one of file_option and field_option can be specified") - } - if fieldName != "" && fileOption != FileOptionUnspecified { - return nil, errors.New("cannot disable a file option for a field") - } - // TODO: validate path here? Was it validated in v1/main? - if moduleFullName != "" { - if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { - return nil, err - } - } - return &managedDisableRule{ - path: path, - moduleFullName: moduleFullName, - fieldName: fieldName, - fileOption: fileOption, - fieldOption: fieldOption, - }, nil -} - func (m *managedDisableRule) Path() string { return m.path } @@ -382,72 +461,6 @@ type managedOverrideRule struct { value interface{} } -func newFileOptionOverrideRule( - path string, - moduleFullName string, - fileOption FileOption, - value interface{}, -) (*managedOverrideRule, error) { - // TODO: validate path here? Was it validated in v1/main? - if moduleFullName != "" { - if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { - return nil, err - } - } - // All valid file options have a parse func. This lookup implicitly validates the option. - parseOverrideValueFunc, ok := fileOptionToParseOverrideValueFunc[fileOption] - if !ok { - return nil, fmt.Errorf("invalid fileOption: %v", fileOption) - } - if value == nil { - return nil, fmt.Errorf("value must be specified for override") - } - parsedValue, err := parseOverrideValueFunc(value) - if err != nil { - return nil, fmt.Errorf("invalid value %v for %v: %w", value, fileOption, err) - } - return &managedOverrideRule{ - path: path, - moduleFullName: moduleFullName, - fileOption: fileOption, - value: parsedValue, - }, nil -} - -func newFieldOptionOverrideRule( - path string, - moduleFullName string, - fieldName string, - fieldOption FieldOption, - value interface{}, -) (*managedOverrideRule, error) { - // TODO: validate path here? Was it validated in v1/main? - if moduleFullName != "" { - if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { - return nil, err - } - } - // All valid field options have a parse func. This lookup implicitly validates the option. - parseOverrideValueFunc, ok := fieldOptionToParseOverrideValueFunc[fieldOption] - if !ok { - return nil, fmt.Errorf("invalid fieldOption: %v", fieldOption) - } - if value == nil { - return nil, fmt.Errorf("value must be specified for override") - } - parsedValue, err := parseOverrideValueFunc(value) - if err != nil { - return nil, fmt.Errorf("invalid value %v for %v: %w", value, fieldOption, err) - } - return &managedOverrideRule{ - path: path, - moduleFullName: moduleFullName, - fieldName: fieldName, - fieldOption: fieldOption, - value: parsedValue, - }, nil -} - func (m *managedOverrideRule) Path() string { return m.path } @@ -493,7 +506,7 @@ func disablesAndOverridesFromExceptAndOverrideV1( return nil, nil, fmt.Errorf("%q is defined multiple times in except", exceptModuleFullName) } seenExceptModuleFullNames[exceptModuleFullName] = struct{}{} - disable, err := newDisableRule( + disable, err := NewDisableRule( "", exceptModuleFullName, "", @@ -514,7 +527,7 @@ func disablesAndOverridesFromExceptAndOverrideV1( if _, ok := seenExceptModuleFullNames[overrideModuleFullName]; ok { return nil, nil, fmt.Errorf("override %q is already defined as an except", overrideModuleFullName) } - override, err := newFileOptionOverrideRule( + override, err := NewFileOptionOverrideRule( "", overrideModuleFullName, overrideFileOption, @@ -567,7 +580,7 @@ func overrideRulesForPerFileOverridesV1( return nil, fmt.Errorf("") } } - overrideRule, err := newFileOptionOverrideRule( + overrideRule, err := NewFileOptionOverrideRule( filePath, "", fileOption, From c7a790698f7273e8b1a7d76f6e0f7b6b48792684 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 27 Nov 2023 13:25:13 -0500 Subject: [PATCH 26/66] commit --- .../bufconfig/bufconfigtest/bufconfigtest.go | 77 ++ .../bufconfig/bufconfigtest/usage.gen.go | 19 + .../bufimagemodify/bufimagemodify_test.go | 47 +- .../bufimage/bufimagemodify/new_modify.go | 579 ++++++++------ .../bufimage/bufimagemodify/new_modify2.go | 140 ++-- .../bufimagemodify/new_modify_test.go | 756 +++++++++++++++++- .../testdata/bar/bar_all/with_package.proto | 20 + .../bar/bar_all/without_package.proto | 18 + .../testdata/bar/bar_empty/with_package.proto | 1 + .../bar/bar_empty/without_package.proto | 0 .../testdata/foo/foo_all/with_package.proto | 20 + .../foo/foo_all/without_package.proto | 18 + .../testdata/foo/foo_empty/with_package.proto | 1 + .../foo/foo_empty/without_package.proto | 0 14 files changed, 1338 insertions(+), 358 deletions(-) create mode 100644 private/bufnew/bufconfig/bufconfigtest/bufconfigtest.go create mode 100644 private/bufnew/bufconfig/bufconfigtest/usage.gen.go create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/with_package.proto create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/without_package.proto create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_empty/with_package.proto create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_empty/without_package.proto create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/with_package.proto create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/without_package.proto create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_empty/with_package.proto create mode 100644 private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_empty/without_package.proto diff --git a/private/bufnew/bufconfig/bufconfigtest/bufconfigtest.go b/private/bufnew/bufconfig/bufconfigtest/bufconfigtest.go new file mode 100644 index 0000000000..d396d65fac --- /dev/null +++ b/private/bufnew/bufconfig/bufconfigtest/bufconfigtest.go @@ -0,0 +1,77 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufconfigtest + +import ( + "testing" + + "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/stretchr/testify/require" +) + +func NewTestManagedDisableRule( + t *testing.T, + path string, + moduleFullName string, + fieldName string, + fileOption bufconfig.FileOption, + fieldOption bufconfig.FieldOption, +) bufconfig.ManagedDisableRule { + disable, err := bufconfig.NewDisableRule( + path, + moduleFullName, + fieldName, + fileOption, + fieldOption, + ) + require.NoError(t, err) + return disable +} + +func NewTestFileOptionOverrideRule( + t *testing.T, + path string, + moduleFullName string, + fileOption bufconfig.FileOption, + value interface{}, +) bufconfig.ManagedOverrideRule { + fileOptionOverride, err := bufconfig.NewFileOptionOverrideRule( + path, + moduleFullName, + fileOption, + value, + ) + require.NoError(t, err) + return fileOptionOverride +} + +func NewTestFieldOptionOverrideRule( + t *testing.T, + path string, + moduleFullName string, + fieldName string, + fieldOption bufconfig.FieldOption, + value interface{}, +) bufconfig.ManagedOverrideRule { + fieldOptionOverrid, err := bufconfig.NewFieldOptionOverrideRule( + path, + moduleFullName, + bufconfig.FileOptionPhpMetadataNamespace.String(), + fieldOption, + value, + ) + require.NoError(t, err) + return fieldOptionOverrid +} diff --git a/private/bufnew/bufconfig/bufconfigtest/usage.gen.go b/private/bufnew/bufconfig/bufconfigtest/usage.gen.go new file mode 100644 index 0000000000..7703e24a1c --- /dev/null +++ b/private/bufnew/bufconfig/bufconfigtest/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufconfigtest + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go index 93a3d802c2..5165d7a4e8 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go @@ -18,7 +18,6 @@ import ( "context" "testing" - "github.com/bufbuild/buf/private/bufnew/bufconfig" "github.com/bufbuild/buf/private/bufnew/bufmodule/bufmoduletest" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagebuild" @@ -93,19 +92,35 @@ func testGetImage(t *testing.T, dirPath string, includeSourceInfo bool) bufimage return image } -type testDisable struct { - path string - moduleFullName string - fieldName string - fileOption bufconfig.FileOption - fieldOption bufconfig.FieldOption -} - -type testOverride struct { - path string - moduleFullName string - fieldName string - fileOption bufconfig.FileOption - fieldOption bufconfig.FieldOption - value interface{} +func testGetImageFromDirs( + t *testing.T, + dirPathToModuleFullName map[string]string, + includeSourceInfo bool, +) bufimage.Image { + moduleDatas := make([]bufmoduletest.ModuleData, 0, len(dirPathToModuleFullName)) + for dirPath, moduleFullName := range dirPathToModuleFullName { + moduleDatas = append( + moduleDatas, + bufmoduletest.ModuleData{ + Name: moduleFullName, + DirPath: dirPath, + }, + ) + } + moduleSet, err := bufmoduletest.NewModuleSet(moduleDatas...) + require.NoError(t, err) + var options []bufimagebuild.BuildOption + if !includeSourceInfo { + options = []bufimagebuild.BuildOption{bufimagebuild.WithExcludeSourceCodeInfo()} + } + image, annotations, err := bufimagebuild.NewBuilder( + zap.NewNop(), + ).Build( + context.Background(), + moduleSet, + options..., + ) + require.NoError(t, err) + require.Empty(t, annotations) + return image } diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go index 6da72add32..12a65353b1 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -23,265 +23,350 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) -// TODO: this is a temporary file, although it might stay. Need to rename the file at least. - +// TODO: this is a temporary file, although it might stay. Need to rename the file at least. (The rest of the package can be deleted, except for ) +// TODO: move this package into bufgen/internal func Modify( ctx context.Context, image bufimage.Image, config bufconfig.GenerateManagedConfig, ) error { + if config == nil { + return nil + } + // TODO: in v2 modify field options as well sweeper := NewFileOptionSweeper() for _, imageFile := range image.Files() { if isWellKnownType(ctx, imageFile) { continue } - // TODO: order them like before or by name or by field number - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionJavaOuterClassname, - bufconfig.FileOptionUnspecified, - bufconfig.FileOptionUnspecified, - func(bufimage.ImageFile) stringOverride { - return stringOverride{value: javaOuterClassnameValue(imageFile)} - }, - func(imageFile bufimage.ImageFile, _ stringOverride) string { - return javaOuterClassnameValue(imageFile) - }, - func(options *descriptorpb.FileOptions) string { - return options.GetJavaOuterClassname() - }, - func(options *descriptorpb.FileOptions, value string) { - options.JavaOuterClassname = proto.String(value) - }, - javaOuterClassnamePath, - ); err != nil { - return err - } - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionJavaPackage, - bufconfig.FileOptionJavaPackagePrefix, - bufconfig.FileOptionJavaPackageSuffix, - func(bufimage.ImageFile) stringOverride { - return stringOverride{prefix: "com"} - }, - getJavaPackageValue, - func(options *descriptorpb.FileOptions) string { - return options.GetJavaPackage() - }, - func(options *descriptorpb.FileOptions, value string) { - options.JavaPackage = proto.String(value) - }, - javaPackagePath, - ); err != nil { - return err - } - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionGoPackage, - bufconfig.FileOptionGoPackagePrefix, - bufconfig.FileOptionUnspecified, - func(bufimage.ImageFile) stringOverride { - return stringOverride{} - }, - func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { - return GoPackageImportPathForFile(imageFile, stringOverride.prefix) - }, - func(options *descriptorpb.FileOptions) string { - return options.GetGoPackage() - }, - func(options *descriptorpb.FileOptions, value string) { - options.GoPackage = proto.String(value) - }, - goPackagePath, - ); err != nil { - return err - } - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionObjcClassPrefix, - bufconfig.FileOptionUnspecified, - bufconfig.FileOptionUnspecified, - func(bufimage.ImageFile) stringOverride { - return stringOverride{value: objcClassPrefixValue(imageFile)} - }, - func(imageFile bufimage.ImageFile, _ stringOverride) string { - return objcClassPrefixValue(imageFile) - }, - func(options *descriptorpb.FileOptions) string { - return options.GetObjcClassPrefix() - }, - func(options *descriptorpb.FileOptions, value string) { - options.ObjcClassPrefix = proto.String(value) - }, - objcClassPrefixPath, - ); err != nil { - return err - } - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionCsharpNamespace, - bufconfig.FileOptionCsharpNamespacePrefix, - bufconfig.FileOptionUnspecified, - func(bufimage.ImageFile) stringOverride { - return stringOverride{value: csharpNamespaceValue(imageFile)} - }, - func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { - return getCsharpNamespaceValue(imageFile, stringOverride.prefix) - }, - func(options *descriptorpb.FileOptions) string { - return options.GetCsharpNamespace() - }, - func(options *descriptorpb.FileOptions, value string) { - options.CsharpNamespace = proto.String(value) - }, - csharpNamespacePath, - ); err != nil { - return err - } - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionPhpNamespace, - bufconfig.FileOptionUnspecified, - bufconfig.FileOptionUnspecified, - func(bufimage.ImageFile) stringOverride { - return stringOverride{value: phpNamespaceValue(imageFile)} - }, - func(imageFile bufimage.ImageFile, _ stringOverride) string { - return phpNamespaceValue(imageFile) - }, - func(options *descriptorpb.FileOptions) string { - return options.GetPhpNamespace() - }, - func(options *descriptorpb.FileOptions, value string) { - options.PhpNamespace = proto.String(value) - }, - phpNamespacePath, - ); err != nil { - return err - } - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionPhpMetadataNamespace, - bufconfig.FileOptionUnspecified, - bufconfig.FileOptionPhpMetadataNamespaceSuffix, - func(bufimage.ImageFile) stringOverride { - return stringOverride{value: phpMetadataNamespaceValue(imageFile)} - }, - func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { - return getPhpMetadataNamespaceValue(imageFile, stringOverride.suffix) - }, - func(options *descriptorpb.FileOptions) string { - return options.GetPhpMetadataNamespace() - }, - func(options *descriptorpb.FileOptions, value string) { - options.PhpMetadataNamespace = proto.String(value) - }, - phpMetadataNamespacePath, - ); err != nil { - return err + modifyFuncs := []func(Sweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error{ + modifyCcEnableArenas, + modifyCsharpNamespace, + modifyGoPackage, + modifyJavaMultipleFiles, + modifyJavaOuterClass, + modifyJavaPackage, + modifyJavaStringCheckUtf8, + modifyObjcClassPrefix, + modifyOptmizeFor, + modifyPhpMetadataNamespace, + modifyPhpNamespace, + modifyRubyPackage, } - if err := modifyStringOption( - sweeper, - imageFile, - config, - bufconfig.FileOptionRubyPackage, - bufconfig.FileOptionUnspecified, - bufconfig.FileOptionRubyPackageSuffix, - func(bufimage.ImageFile) stringOverride { - return stringOverride{value: rubyPackageValue(imageFile)} - }, - func(imageFile bufimage.ImageFile, stringOverride stringOverride) string { - return getRubyPackageValue(imageFile, stringOverride.suffix) - }, - func(options *descriptorpb.FileOptions) string { - return options.GetRubyPackage() - }, - func(options *descriptorpb.FileOptions, value string) { - options.RubyPackage = proto.String(value) - }, - rubyPackagePath, - ); err != nil { - return err - } - if err := modifyOption[bool]( - sweeper, - imageFile, - config, - bufconfig.FileOptionCcEnableArenas, - true, - func(options *descriptorpb.FileOptions) bool { - return options.GetCcEnableArenas() - }, - func(options *descriptorpb.FileOptions, value bool) { - options.CcEnableArenas = proto.Bool(value) - }, - ccEnableArenasPath, - ); err != nil { - return err - } - if err := modifyOption[bool]( - sweeper, - imageFile, - config, - bufconfig.FileOptionJavaMultipleFiles, - true, - func(options *descriptorpb.FileOptions) bool { - return options.GetJavaMultipleFiles() - }, - func(options *descriptorpb.FileOptions, value bool) { - options.JavaMultipleFiles = proto.Bool(value) - }, - javaMultipleFilesPath, - ); err != nil { - return err - } - if err := modifyOption[bool]( - sweeper, - imageFile, - config, - bufconfig.FileOptionJavaStringCheckUtf8, - false, - func(options *descriptorpb.FileOptions) bool { - return options.GetJavaStringCheckUtf8() - }, - func(options *descriptorpb.FileOptions, value bool) { - options.JavaStringCheckUtf8 = proto.Bool(value) - }, - javaStringCheckUtf8Path, - ); err != nil { - return err - } - if err := modifyOption[descriptorpb.FileOptions_OptimizeMode]( - sweeper, - imageFile, - config, - bufconfig.FileOptionOptimizeFor, - descriptorpb.FileOptions_SPEED, - func(options *descriptorpb.FileOptions) descriptorpb.FileOptions_OptimizeMode { - return options.GetOptimizeFor() - }, - func(options *descriptorpb.FileOptions, value descriptorpb.FileOptions_OptimizeMode) { - options.OptimizeFor = value.Enum() - }, - optimizeForPath, - ); err != nil { - return err + for _, modifyFunc := range modifyFuncs { + if err := modifyFunc(sweeper, imageFile, config); err != nil { + return err + } } } return nil } + +func modifyJavaOuterClass( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaOuterClassname, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{value: javaOuterClassnameValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, _ stringOverrideOptions) string { + return javaOuterClassnameValue(imageFile) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetJavaOuterClassname() + }, + func(options *descriptorpb.FileOptions, value string) { + options.JavaOuterClassname = proto.String(value) + }, + javaOuterClassnamePath, + ) +} + +func modifyJavaPackage( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaPackage, + bufconfig.FileOptionJavaPackagePrefix, + bufconfig.FileOptionJavaPackageSuffix, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{prefix: "com"} + }, + getJavaPackageValue, + func(options *descriptorpb.FileOptions) string { + return options.GetJavaPackage() + }, + func(options *descriptorpb.FileOptions, value string) { + options.JavaPackage = proto.String(value) + }, + javaPackagePath, + ) +} + +func modifyGoPackage( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionGoPackage, + bufconfig.FileOptionGoPackagePrefix, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { + if stringOverride.prefix == "" { + return "" + } + return GoPackageImportPathForFile(imageFile, stringOverride.prefix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetGoPackage() + }, + func(options *descriptorpb.FileOptions, value string) { + options.GoPackage = proto.String(value) + }, + goPackagePath, + ) +} + +func modifyObjcClassPrefix( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionObjcClassPrefix, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{value: objcClassPrefixValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, _ stringOverrideOptions) string { + return objcClassPrefixValue(imageFile) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetObjcClassPrefix() + }, + func(options *descriptorpb.FileOptions, value string) { + options.ObjcClassPrefix = proto.String(value) + }, + objcClassPrefixPath, + ) +} + +func modifyCsharpNamespace( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionCsharpNamespace, + bufconfig.FileOptionCsharpNamespacePrefix, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{value: csharpNamespaceValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { + return getCsharpNamespaceValue(imageFile, stringOverride.prefix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetCsharpNamespace() + }, + func(options *descriptorpb.FileOptions, value string) { + options.CsharpNamespace = proto.String(value) + }, + csharpNamespacePath, + ) +} + +func modifyPhpNamespace( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionPhpNamespace, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionUnspecified, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{value: phpNamespaceValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, _ stringOverrideOptions) string { + return phpNamespaceValue(imageFile) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetPhpNamespace() + }, + func(options *descriptorpb.FileOptions, value string) { + options.PhpNamespace = proto.String(value) + }, + phpNamespacePath, + ) +} + +func modifyPhpMetadataNamespace( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionPhpMetadataNamespace, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionPhpMetadataNamespaceSuffix, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{value: phpMetadataNamespaceValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { + return getPhpMetadataNamespaceValue(imageFile, stringOverride.suffix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetPhpMetadataNamespace() + }, + func(options *descriptorpb.FileOptions, value string) { + options.PhpMetadataNamespace = proto.String(value) + }, + phpMetadataNamespacePath, + ) +} + +func modifyRubyPackage( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyStringOption( + sweeper, + imageFile, + config, + bufconfig.FileOptionRubyPackage, + bufconfig.FileOptionUnspecified, + bufconfig.FileOptionRubyPackageSuffix, + func(bufimage.ImageFile) stringOverrideOptions { + return stringOverrideOptions{value: rubyPackageValue(imageFile)} + }, + func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { + return getRubyPackageValue(imageFile, stringOverride.suffix) + }, + func(options *descriptorpb.FileOptions) string { + return options.GetRubyPackage() + }, + func(options *descriptorpb.FileOptions, value string) { + options.RubyPackage = proto.String(value) + }, + rubyPackagePath, + ) +} + +func modifyCcEnableArenas( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyOption[bool]( + sweeper, + imageFile, + config, + bufconfig.FileOptionCcEnableArenas, + true, + func(options *descriptorpb.FileOptions) bool { + return options.GetCcEnableArenas() + }, + func(options *descriptorpb.FileOptions, value bool) { + options.CcEnableArenas = proto.Bool(value) + }, + ccEnableArenasPath, + ) +} + +func modifyJavaMultipleFiles( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyOption[bool]( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaMultipleFiles, + true, + func(options *descriptorpb.FileOptions) bool { + return options.GetJavaMultipleFiles() + }, + func(options *descriptorpb.FileOptions, value bool) { + options.JavaMultipleFiles = proto.Bool(value) + }, + javaMultipleFilesPath, + ) +} + +func modifyJavaStringCheckUtf8( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyOption[bool]( + sweeper, + imageFile, + config, + bufconfig.FileOptionJavaStringCheckUtf8, + false, + func(options *descriptorpb.FileOptions) bool { + return options.GetJavaStringCheckUtf8() + }, + func(options *descriptorpb.FileOptions, value bool) { + options.JavaStringCheckUtf8 = proto.Bool(value) + }, + javaStringCheckUtf8Path, + ) +} + +func modifyOptmizeFor( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + return modifyOption[descriptorpb.FileOptions_OptimizeMode]( + sweeper, + imageFile, + config, + bufconfig.FileOptionOptimizeFor, + descriptorpb.FileOptions_SPEED, + func(options *descriptorpb.FileOptions) descriptorpb.FileOptions_OptimizeMode { + return options.GetOptimizeFor() + }, + func(options *descriptorpb.FileOptions, value descriptorpb.FileOptions_OptimizeMode) { + options.OptimizeFor = value.Enum() + }, + optimizeForPath, + ) +} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go index c6c064aa59..4f192f255b 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go @@ -23,32 +23,6 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) -// TODO: these might be useful, keeping them around for now - -// type managedConfig interface { -// Disables() []managedDisableRule -// Overrides() []managedOverrideRule -// } - -// type managedDisableRule interface { -// Path() string -// ModuleFullName() string -// FieldName() string -// FileOption() bufconfig.FileOption -// FieldOption() bufconfig.FieldOption -// } - -// type managedOverrideRule interface { -// Path() string -// ModuleFullName() string -// FieldName() string -// FileOption() bufconfig.FileOption -// FieldOption() bufconfig.FieldOption -// Value() interface{} - -// isManagedOverrideRule() -// } - func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( sweeper Sweeper, imageFile bufimage.ImageFile, @@ -61,7 +35,14 @@ func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( sourceLocationPath []int32, ) error { value := defaultValue - override, disable, err := overrideAndDisableFromConfig[T]( + if isFileOptionDisabledForFile( + imageFile, + fileOption, + config, + ) { + return nil + } + override, err := overrideFromConfig[T]( imageFile, config, fileOption, @@ -69,9 +50,6 @@ func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( if err != nil { return err } - if disable { - return nil - } if override != nil { value = *override } @@ -89,18 +67,11 @@ func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( } // returns the override value and whether managed mode is DISABLED for this file for this file option. -func overrideAndDisableFromConfig[T bool | descriptorpb.FileOptions_OptimizeMode]( +func overrideFromConfig[T bool | descriptorpb.FileOptions_OptimizeMode]( imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, fileOption bufconfig.FileOption, -) (*T, bool, error) { - if isFileOptionDisabledForFile( - imageFile, - fileOption, - config, - ) { - return nil, true, nil - } +) (*T, error) { var override *T for _, overrideRule := range config.Overrides() { if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { @@ -112,11 +83,11 @@ func overrideAndDisableFromConfig[T bool | descriptorpb.FileOptions_OptimizeMode value, ok := overrideRule.Value().(T) if !ok { // This should never happen, since the override rule has been validated. - return nil, false, fmt.Errorf("invalid value type for %v override: %T", fileOption, overrideRule.Value()) + return nil, fmt.Errorf("invalid value type for %v override: %T", fileOption, overrideRule.Value()) } override = &value } - return override, false, nil + return override, nil } func modifyStringOption( @@ -127,16 +98,16 @@ func modifyStringOption( prefixOption bufconfig.FileOption, suffixOption bufconfig.FileOption, // todo: options? unify - defaultOptionsFunc func(bufimage.ImageFile) stringOverride, - valueFunc func(bufimage.ImageFile, stringOverride) string, + defaultOptionsFunc func(bufimage.ImageFile) stringOverrideOptions, + valueFunc func(bufimage.ImageFile, stringOverrideOptions) string, getOptionFunc func(*descriptorpb.FileOptions) string, setOptionFunc func(*descriptorpb.FileOptions, string), sourceLocationPath []int32, ) error { - modifyOptions := defaultOptionsFunc(imageFile) - override, disable, err := stringOverrideAndDisableFromConfig( + overrideOptions, err := stringOverrideFromConfig( imageFile, config, + defaultOptionsFunc(imageFile), valueOption, prefixOption, suffixOption, @@ -144,24 +115,16 @@ func modifyStringOption( if err != nil { return err } - if disable { + var emptyOverrideOptions stringOverrideOptions + // This means the options are all disabled. + if overrideOptions == emptyOverrideOptions { return nil } - if override != nil { - if override.value != "" { - modifyOptions.value = override.value - } - if override.prefix != "" { - modifyOptions.prefix = override.prefix - } - if override.suffix != "" { - modifyOptions.suffix = override.suffix - } - } - // either value is set or one of prefix and suffix is set. - value := modifyOptions.value + // Now either value is set or prefix and/or suffix is set. + value := overrideOptions.value if value == "" { - value = valueFunc(imageFile, modifyOptions) + // TODO: pass in prefix and suffix, instead of just override options + value = valueFunc(imageFile, overrideOptions) } if value == "" { return nil @@ -179,21 +142,32 @@ func modifyStringOption( return nil } +// TODO: see if this still needs to return a pointer as opposed to a struct // the first value is nil when no override rule is matched -// returns the override value and whether managed mode is DISABLED for this file for this file option. -func stringOverrideAndDisableFromConfig( +func stringOverrideFromConfig( imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, + defaultOverrideOptions stringOverrideOptions, valueFileOption bufconfig.FileOption, prefixFileOption bufconfig.FileOption, suffixFileOption bufconfig.FileOption, -) (*stringOverride, bool, error) { - if isFileOptionDisabledForFile(imageFile, valueFileOption, config) { - return nil, true, nil +) (stringOverrideOptions, error) { + if isFileOptionDisabledForFile( + imageFile, + valueFileOption, + config, + ) { + return stringOverrideOptions{}, nil } - var override *stringOverride + overrideOptions := defaultOverrideOptions ignorePrefix := prefixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, prefixFileOption, config) ignoreSuffix := suffixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, suffixFileOption, config) + if ignorePrefix { + overrideOptions.prefix = "" + } + if ignoreSuffix { + overrideOptions.suffix = "" + } for _, overrideRule := range config.Overrides() { if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { continue @@ -203,44 +177,45 @@ func stringOverrideAndDisableFromConfig( valueString, ok := overrideRule.Value().(string) if !ok { // This should never happen, since the override rule has been validated. - return nil, false, fmt.Errorf("invalid value type for %v override: %T", valueFileOption, overrideRule.Value()) + return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", valueFileOption, overrideRule.Value()) } - override = &stringOverride{value: valueString} + // If the latest override matched is a value override (java_package as opposed to java_package_prefix), use the value. + overrideOptions = stringOverrideOptions{value: valueString} case prefixFileOption: if ignorePrefix { continue } - prefixString, ok := overrideRule.Value().(string) + prefix, ok := overrideRule.Value().(string) if !ok { // This should never happen, since the override rule has been validated. - return &stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", prefixFileOption, overrideRule.Value()) + return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", prefixFileOption, overrideRule.Value()) } // Keep the suffix if the last two overrides are suffix and prefix. - override = &stringOverride{ - prefix: prefixString, - suffix: override.suffix, + overrideOptions = stringOverrideOptions{ + prefix: prefix, + suffix: overrideOptions.suffix, } case suffixFileOption: if ignoreSuffix { continue } - suffixString, ok := overrideRule.Value().(string) + suffix, ok := overrideRule.Value().(string) if !ok { // This should never happen, since the override rule has been validated. - return &stringOverride{}, false, fmt.Errorf("invalid value type for %v override: %T", suffixFileOption, overrideRule.Value()) + return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", suffixFileOption, overrideRule.Value()) } // Keep the prefix if the last two overrides are suffix and prefix. - override = &stringOverride{ - prefix: override.prefix, - suffix: suffixString, + overrideOptions = stringOverrideOptions{ + prefix: overrideOptions.prefix, + suffix: suffix, } } } - return override, false, nil + return overrideOptions, nil } // TODO: rename to string override options maybe? -type stringOverride struct { +type stringOverrideOptions struct { value string prefix string suffix string @@ -277,9 +252,8 @@ func fileMatchConfig( return true } -// TODO: rename these helpers - -func getJavaPackageValue(imageFile bufimage.ImageFile, stringOverride stringOverride) string { +// TODO: rename/clean up these helpers (and merge with the other original ones as well) +func getJavaPackageValue(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { if pkg := imageFile.FileDescriptorProto().GetPackage(); pkg != "" { if stringOverride.prefix != "" { pkg = stringOverride.prefix + "." + pkg diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go index 69ff29c517..fb6b85f024 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go @@ -15,33 +15,765 @@ package bufimagemodify import ( + "context" + "path/filepath" "testing" - "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/bufnew/bufconfig/bufconfigtest" + "github.com/bufbuild/buf/private/bufnew/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagetesting" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/descriptorpb" ) -func TestModify(t *testing.T) { +func TestModifyImage(t *testing.T) { t.Parallel() testcases := []struct { - description string - moduleNameToDirpath string - config bufconfig.Config - pathToExpectedOptions map[string]*descriptorpb.FileOptions - }{} + description string + dirPathToModuleFullName map[string]string + config bufconfig.GenerateManagedConfig + filePathToExpectedOptions map[string]*descriptorpb.FileOptions + }{ + { + description: "nil_config", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: nil, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": nil, + "bar_all/with_package.proto": { + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaPackage: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + }, + }, + { + description: "empty_config", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{}, + ), + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": { + // CcEnableArena's default value is true + CsharpNamespace: proto.String("Foo.Empty"), + // GoPackage is not modified by default + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithPackageProto"), + JavaPackage: proto.String("com.foo.empty"), + // JavaStringCheckUtf8 is not modified by default + ObjcClassPrefix: proto.String("FEX"), + // OptimizeFor tries to modifiy this value to SPEED, which is already the default + // Empty is a keyword in php + PhpMetadataNamespace: proto.String(`Foo\Empty_\GPBMetadata`), + PhpNamespace: proto.String(`Foo\Empty_`), + RubyPackage: proto.String("Foo::Empty"), + }, + "foo_empty/without_package.proto": { + // CcEnableArena's default value is true + // GoPackage is not modified by default + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithoutPackageProto"), + // JavaStringCheckUtf8 is not modified by default + // OptimizeFor tries to modifiy this value to SPEED, which is already the default + }, + "bar_all/with_package.proto": { + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("Bar.All"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithPackageProto"), + JavaPackage: proto.String("com.bar.all"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("BAX"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String(`Bar\All\GPBMetadata`), + PhpNamespace: proto.String(`Bar\All`), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("Bar::All"), + SwiftPrefix: proto.String("bar"), + }, + "bar_all/without_package.proto": { + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithoutPackageProto"), + JavaPackage: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String(`bar`), + PhpNamespace: proto.String(`bar`), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + }, + }, + } for _, testcase := range testcases { testcase := testcase - t.Run(testcase.description, func(t *testing.T) { - }) + for _, includeSourceInfo := range []bool{true, false} { + includeSourceInfo := includeSourceInfo + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + image := testGetImageFromDirs(t, testcase.dirPathToModuleFullName, includeSourceInfo) + err := Modify( + context.Background(), + image, + testcase.config, + ) + require.NoError(t, err) + for filePath, expectedOptions := range testcase.filePathToExpectedOptions { + imageFile := image.GetFile(filePath) + require.NotNil(t, imageFile) + require.Empty( + t, + cmp.Diff(expectedOptions, imageFile.FileDescriptorProto().GetOptions(), protocmp.Transform()), + imageFile.FileDescriptorProto().GetOptions(), + ) + } + }) + } + } +} + +func TestModifyImageFile( + t *testing.T, +) { + t.Parallel() + testcases := []struct { + description string + dirPathToModuleFullName map[string]string + config bufconfig.GenerateManagedConfig + modifyFunc func(Sweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error + filePathToExpectedOptions map[string]*descriptorpb.FileOptions + filePathToExpectedMarkedLocationPaths map[string][][]int32 + }{ + { + description: "cc_enable_arena", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionCcEnableArenas, false), + }, + ), + modifyFunc: modifyCcEnableArenas, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/without_package.proto": nil, + "bar_empty/without_package.proto": { + CcEnableArenas: proto.Bool(false), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/without_package.proto": {ccEnableArenasPath}, + }, + }, + { + description: "csharp_namespace", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionCsharpNamespacePrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespacePrefix, "BarPrefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespace, "BarValue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespace, "FooValue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespacePrefix, "FooPrefix"), + }, + ), + modifyFunc: modifyCsharpNamespace, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "bar_empty/with_package.proto": { + CsharpNamespace: proto.String("BarPrefix.Bar.Empty"), + }, + "bar_empty/without_package.proto": { + CsharpNamespace: proto.String("BarValue"), + }, + "foo_empty/with_package.proto": { + CsharpNamespace: proto.String("FooValue"), + }, + "foo_empty/without_package.proto": nil, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/with_package.proto": {csharpNamespacePath}, + "bar_empty/without_package.proto": {csharpNamespacePath}, + "foo_empty/with_package.proto": {csharpNamespacePath}, + }, + }, + { + description: "go_package", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionGoPackagePrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionGoPackagePrefix, "barprefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionGoPackage, "barvalue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty/with_package.proto", "buf.build/acme/foo", bufconfig.FileOptionGoPackage, "foovalue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionGoPackagePrefix, "fooprefix"), + }, + ), + modifyFunc: modifyGoPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "bar_empty/with_package.proto": { + GoPackage: proto.String("barprefix/bar_empty"), + }, + "bar_empty/without_package.proto": { + GoPackage: proto.String("barvalue"), + }, + "foo_empty/with_package.proto": { + GoPackage: proto.String("foovalue"), + }, + "foo_empty/without_package.proto": { + GoPackage: proto.String("fooprefix/foo_empty"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/with_package.proto": {goPackagePath}, + "bar_empty/without_package.proto": {goPackagePath}, + "foo_empty/with_package.proto": {goPackagePath}, + "foo_empty/without_package.proto": {goPackagePath}, + }, + }, + { + description: "java_package_prefix", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackagePrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackagePrefix, "barprefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackageSuffix, "foosuffix"), + }, + ), + modifyFunc: modifyJavaPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": { + // default prefix and override suffix + JavaPackage: proto.String("com.foo.empty.foosuffix"), + }, + // prefix is disabled + "bar_empty/with_package.proto": nil, + // prefix is overridden + "bar_all/with_package.proto": { + JavaPackage: proto.String("barprefix.bar.all"), + // below this point are the values from the file + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + // not modified because it doesn't have a package + "foo_empty/without_package.proto": nil, + "bar_empty/without_package.proto": nil, + "foo_all/without_package.proto": { + // values are from the file + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(true), + CsharpNamespace: proto.String("foo"), + GoPackage: proto.String("foo"), + JavaGenericServices: proto.Bool(true), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("foo"), + JavaPackage: proto.String("foo"), + JavaStringCheckUtf8: proto.Bool(true), + ObjcClassPrefix: proto.String("foo"), + OptimizeFor: descriptorpb.FileOptions_CODE_SIZE.Enum(), + PhpClassPrefix: proto.String("foo"), + PhpGenericServices: proto.Bool(true), + PhpMetadataNamespace: proto.String("foo"), + PhpNamespace: proto.String("foo"), + PyGenericServices: proto.Bool(true), + RubyPackage: proto.String("foo"), + SwiftPrefix: proto.String("foo"), + }, + "bar_all/without_package.proto": { + // values are from the file + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaPackage: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "foo_empty/with_package.proto": {javaPackagePath}, + "bar_all/with_package.proto": {javaPackagePath}, + }, + }, + { + description: "java_package_suffix", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackageSuffix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + modifyFunc: modifyJavaPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": { + // only suffix matches, but apply both prefix and suffix + JavaPackage: proto.String("com.foo.empty.suffix"), + }, + "bar_empty/with_package.proto": { + // only prefix because suffix is disabled + JavaPackage: proto.String("com.bar.empty"), + }, + "bar_all/with_package.proto": { + JavaPackage: proto.String("com.bar.all.suffix"), + // below this point are the values from the file + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + // not modified + "foo_empty/without_package.proto": nil, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "foo_empty/with_package.proto": {javaPackagePath}, + "bar_empty/with_package.proto": {javaPackagePath}, + "bar_all/with_package.proto": {javaPackagePath}, + }, + }, + { + description: "java_package", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackage, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackage, "bar.value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackage, "foo.value"), + }, + ), + modifyFunc: modifyJavaPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + // bar_empty disabled + "bar_empty/with_package.proto": nil, + "bar_empty/without_package.proto": nil, + "bar_all/with_package.proto": { + JavaPackage: proto.String("bar.value"), + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + "foo_empty/with_package.proto": { + JavaPackage: proto.String("foo.value"), + }, + "foo_empty/without_package.proto": { + JavaPackage: proto.String("foo.value"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "foo_empty/with_package.proto": {javaPackagePath}, + "foo_empty/without_package.proto": {javaPackagePath}, + "bar_all/with_package.proto": {javaPackagePath}, + }, + }, + { + description: "objc_class_prefix", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionObjcClassPrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionObjcClassPrefix, "BAR"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOO"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_all", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOOALL"), + }, + ), + modifyFunc: modifyObjcClassPrefix, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "bar_empty/with_package.proto": { + ObjcClassPrefix: proto.String("BAR"), + }, + "bar_empty/without_package.proto": { + ObjcClassPrefix: proto.String("BAR"), + }, + // disabled + "foo_empty/with_package.proto": nil, + // no package + "foo_empty/without_package.proto": { + ObjcClassPrefix: proto.String("FOO"), + }, + "foo_all/with_package.proto": { + ObjcClassPrefix: proto.String("FOOALL"), + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(true), + CsharpNamespace: proto.String("foo"), + GoPackage: proto.String("foo"), + JavaGenericServices: proto.Bool(true), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("foo"), + JavaPackage: proto.String("foo"), + JavaStringCheckUtf8: proto.Bool(true), + OptimizeFor: descriptorpb.FileOptions_CODE_SIZE.Enum(), + PhpClassPrefix: proto.String("foo"), + PhpGenericServices: proto.Bool(true), + PhpMetadataNamespace: proto.String("foo"), + PhpNamespace: proto.String("foo"), + PyGenericServices: proto.Bool(true), + RubyPackage: proto.String("foo"), + SwiftPrefix: proto.String("foo"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/with_package.proto": {objcClassPrefixPath}, + "bar_empty/without_package.proto": {objcClassPrefixPath}, + "foo_empty/without_package.proto": {objcClassPrefixPath}, + "foo_all/without_package.proto": {objcClassPrefixPath}, + "foo_all/with_package.proto": {objcClassPrefixPath}, + }, + }, + } + for _, testcase := range testcases { + testcase := testcase + for _, includeSourceInfo := range []bool{true, false} { + // TODO: we are only testing sweep here, no need to test both include and exclude source info + includeSourceInfo := includeSourceInfo + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + image := testGetImageFromDirs(t, testcase.dirPathToModuleFullName, includeSourceInfo) + sweeper := newFileOptionSweeper() + // TODO: check include source code info + for filePath, expectedOptions := range testcase.filePathToExpectedOptions { + imageFile := image.GetFile(filePath) + testcase.modifyFunc( + sweeper, + imageFile, + testcase.config, + ) + require.NotNil(t, imageFile) + require.Empty( + t, + cmp.Diff(expectedOptions, imageFile.FileDescriptorProto().GetOptions(), protocmp.Transform()), + "incorrect options result for %s", + filePath, + ) + // TODO: full map compare + require.Equal( + t, + slicesext.Map(testcase.filePathToExpectedMarkedLocationPaths[filePath], getPathKey), + slicesext.MapKeysToSortedSlice(sweeper.sourceCodeInfoPaths[filePath]), + ) + } + }) + } } } -func TestSweepFileOption(t *testing.T) { +// TODO: add default values +func TestGetStringOverrideFromConfig(t *testing.T) { t.Parallel() - // TODO + testcases := []struct { + description string + config bufconfig.GenerateManagedConfig + imageFile bufimage.ImageFile + defaultOverrideOptions stringOverrideOptions + expectedOverride stringOverrideOptions + expectedDisable bool + }{ + { + description: "only_value", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{value: "value"}, + }, + { + description: "only_prefix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{prefix: "prefix"}, + }, + { + description: "only_suffix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{suffix: "suffix"}, + }, + { + description: "prefix_then_value", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{value: "value"}, + }, + { + description: "value_then_prefix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{prefix: "prefix"}, + }, + { + description: "prefix_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{ + prefix: "prefix", + suffix: "suffix", + }, + }, + { + description: "value_prefix_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{ + prefix: "prefix", + suffix: "suffix", + }, + }, + { + description: "prefix_value_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{suffix: "suffix"}, + }, + { + description: "prefix_then_prefix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix2"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{prefix: "prefix2"}, + }, + { + description: "suffix_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix2"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{suffix: "suffix2"}, + }, + { + description: "value_then_value", + config: bufconfig.NewGenerateManagedConfig( + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value2"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{value: "value2"}, + }, + } + for _, testcase := range testcases { + testcase := testcase + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + override, err := stringOverrideFromConfig( + testcase.imageFile, + testcase.config, + testcase.defaultOverrideOptions, + bufconfig.FileOptionJavaPackage, + bufconfig.FileOptionJavaPackagePrefix, + bufconfig.FileOptionJavaPackageSuffix, + ) + require.NoError(t, err) + require.Equal(t, testcase.expectedOverride, override) + }) + } } -func TestSweepFieldOption(t *testing.T) { +func TestModifyFieldOption(t *testing.T) { t.Parallel() // TODO in v2 } + +func testGetImageFile( + t *testing.T, + path string, + moduleFullName string, +) bufimage.ImageFile { + parsedModuleFullName, err := bufmodule.ParseModuleFullName(moduleFullName) + require.NoError(t, err) + return bufimagetesting.NewImageFile( + t, + &descriptorpb.FileDescriptorProto{ + Name: proto.String(path), + Syntax: proto.String("proto3"), + }, + parsedModuleFullName, + "", + path, + false, + false, + nil, + ) +} diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/with_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/with_package.proto new file mode 100644 index 0000000000..40912a91a5 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/with_package.proto @@ -0,0 +1,20 @@ +package bar.all; + +option cc_enable_arenas = false; +option cc_generic_services = false; +option csharp_namespace = "bar"; +option go_package = "bar"; +option java_generic_services = false; +option java_multiple_files = false; +option java_outer_classname = "bar"; +option java_package = "bar"; +option java_string_check_utf8 = false; +option objc_class_prefix = "bar"; +option optimize_for = SPEED; +option php_class_prefix = "bar"; +option php_generic_services = false; +option php_metadata_namespace = "bar"; +option php_namespace = "bar"; +option py_generic_services = false; +option ruby_package = "bar"; +option swift_prefix = "bar"; diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/without_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/without_package.proto new file mode 100644 index 0000000000..76f2451093 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_all/without_package.proto @@ -0,0 +1,18 @@ +option cc_enable_arenas = false; +option cc_generic_services = false; +option csharp_namespace = "bar"; +option go_package = "bar"; +option java_generic_services = false; +option java_multiple_files = false; +option java_outer_classname = "bar"; +option java_package = "bar"; +option java_string_check_utf8 = false; +option objc_class_prefix = "bar"; +option optimize_for = SPEED; +option php_class_prefix = "bar"; +option php_generic_services = false; +option php_metadata_namespace = "bar"; +option php_namespace = "bar"; +option py_generic_services = false; +option ruby_package = "bar"; +option swift_prefix = "bar"; diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_empty/with_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_empty/with_package.proto new file mode 100644 index 0000000000..cf4a9e547d --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_empty/with_package.proto @@ -0,0 +1 @@ +package bar.empty; diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_empty/without_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/bar/bar_empty/without_package.proto new file mode 100644 index 0000000000..e69de29bb2 diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/with_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/with_package.proto new file mode 100644 index 0000000000..dec813d807 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/with_package.proto @@ -0,0 +1,20 @@ +package foo.all; + +option cc_enable_arenas = true; +option cc_generic_services = true; +option csharp_namespace = "foo"; +option go_package = "foo"; +option java_generic_services = true; +option java_multiple_files = true; +option java_outer_classname = "foo"; +option java_package = "foo"; +option java_string_check_utf8 = true; +option objc_class_prefix = "foo"; +option optimize_for = CODE_SIZE; +option php_class_prefix = "foo"; +option php_generic_services = true; +option php_metadata_namespace = "foo"; +option php_namespace = "foo"; +option py_generic_services = true; +option ruby_package = "foo"; +option swift_prefix = "foo"; diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/without_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/without_package.proto new file mode 100644 index 0000000000..aab870f526 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_all/without_package.proto @@ -0,0 +1,18 @@ +option cc_enable_arenas = true; +option cc_generic_services = true; +option csharp_namespace = "foo"; +option go_package = "foo"; +option java_generic_services = true; +option java_multiple_files = true; +option java_outer_classname = "foo"; +option java_package = "foo"; +option java_string_check_utf8 = true; +option objc_class_prefix = "foo"; +option optimize_for = CODE_SIZE; +option php_class_prefix = "foo"; +option php_generic_services = true; +option php_metadata_namespace = "foo"; +option php_namespace = "foo"; +option py_generic_services = true; +option ruby_package = "foo"; +option swift_prefix = "foo"; diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_empty/with_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_empty/with_package.proto new file mode 100644 index 0000000000..af978aa59f --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_empty/with_package.proto @@ -0,0 +1 @@ +package foo.empty; diff --git a/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_empty/without_package.proto b/private/bufpkg/bufimage/bufimagemodify/testdata/foo/foo_empty/without_package.proto new file mode 100644 index 0000000000..e69de29bb2 From e87788ac7786c9663eb1a5453e084a80fd0404f8 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 27 Nov 2023 14:37:19 -0500 Subject: [PATCH 27/66] update generate input config interface --- .../bufpkg/bufconfig/generate_input_config.go | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_input_config.go b/private/bufpkg/bufconfig/generate_input_config.go index b504fa0f2b..6414ef75a0 100644 --- a/private/bufpkg/bufconfig/generate_input_config.go +++ b/private/bufpkg/bufconfig/generate_input_config.go @@ -15,12 +15,52 @@ package bufconfig -import "github.com/bufbuild/buf/private/buf/buffetch" - // GenerateInputConfig is an input configuration for code generation. type GenerateInputConfig interface { - // Ref returns the input ref. - Ref() buffetch.Ref + // Exactly one of Module, Directory, ProtoFile, Tarball, ZipArchive, BinaryImage, + // JSONImage or GitRepo is not empty, indicating the input format. + + // Module returns the module name. + Module() string + // Direcotry returns the path to the directory, relative or absolute. + Directory() string + // ProtoFile returns the path to the proto file, relative or absolute. + ProtoFile() string + // Tarball returns the path to the tarball, relative or absolute. + Tarball() string + // ZipArchive returns the path to the zip archive, relative or absolute. + ZipArchive() string + // BinaryImage returns the path to the binary buf image, relative or absolute. + BinaryImage() string + // JSONImage returns the path to the JSON buf image, relative or absolute. + JSONImage() string + // TextImage returns the path to the text buf image, relative or absolute. + TextImage() string + // GitRepo returns the git repository address in https, ssh or local format. + GitRepo() string + // Compression returns the compression scheme, not empty only if format is + // one of tarball, binary image, json image or text image. + Compression() string + // StripComponent returns the number of directories to strip for tar or zip + // inputs, not empty only if format is tarball or zip archive. + StripComponent() uint32 + // Subdir returns the subdirectory to use, not empty only if format is one + // git repo, tarball and zip archive. + Subdir() string + // Branch returns the git branch to checkout out, not empty only if format is git. + Branch() string + // Tag returns the git tag to checkout, not empty only if format is git. + Tag() string + // Ref returns the git ref to checkout, not empty only if format is git. + Ref() string + // Ref returns the depth to clone the git repo with, not empty only if format is git. + Depth() string + // RecurseSubmodules returns whether to clone submodules recursively. Not empty + // only if input if git. + RecurseSubmodules() bool + // IncludePackageFiles returns other files in the same package as the proto file, + // not empty only if format is proto file. + IncludePackageFiles() bool // Types returns the types to generate. If GenerateConfig.GenerateTypeConfig() // returns a non-empty list of types. Types() []string From 82787fbb0f42927f1059b65404db605de60a9996 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 27 Nov 2023 14:44:36 -0500 Subject: [PATCH 28/66] fix import paths --- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 1 + .../bufpkg/bufconfig/bufconfigtest/bufconfigtest.go | 2 +- private/bufpkg/bufconfig/generate_managed_config.go | 2 +- .../bufimage/bufimagemodify/bufimagemodify_test.go | 10 ++++------ private/bufpkg/bufimage/bufimagemodify/new_modify.go | 2 +- private/bufpkg/bufimage/bufimagemodify/new_modify2.go | 2 +- .../bufpkg/bufimage/bufimagemodify/new_modify_test.go | 6 +++--- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 3842144aef..f85e50fe7c 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -17,6 +17,7 @@ package bufconfig import ( "context" "errors" + "fmt" "io" "github.com/bufbuild/buf/private/pkg/storage" diff --git a/private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go b/private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go index d396d65fac..d4ad333115 100644 --- a/private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go +++ b/private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go @@ -17,7 +17,7 @@ package bufconfigtest import ( "testing" - "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/stretchr/testify/require" ) diff --git a/private/bufpkg/bufconfig/generate_managed_config.go b/private/bufpkg/bufconfig/generate_managed_config.go index 8fe52ad5c9..3a9b077a8d 100644 --- a/private/bufpkg/bufconfig/generate_managed_config.go +++ b/private/bufpkg/bufconfig/generate_managed_config.go @@ -20,7 +20,7 @@ import ( "strconv" "strings" - "github.com/bufbuild/buf/private/bufnew/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/pkg/normalpath" "github.com/bufbuild/buf/private/pkg/slicesext" ) diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go index 057deda14d..2d8a10f7a5 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go @@ -106,15 +106,13 @@ func testGetImageFromDirs( } moduleSet, err := bufmoduletest.NewModuleSet(moduleDatas...) require.NoError(t, err) - var options []bufimagebuild.BuildOption + var options []bufimage.BuildImageOption if !includeSourceInfo { - options = []bufimagebuild.BuildOption{bufimagebuild.WithExcludeSourceCodeInfo()} + options = []bufimage.BuildImageOption{bufimage.WithExcludeSourceCodeInfo()} } - image, annotations, err := bufimagebuild.NewBuilder( - zap.NewNop(), - ).Build( + image, annotations, err := bufimage.BuildImage( context.Background(), - moduleSet, + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), options..., ) require.NoError(t, err) diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go index 12a65353b1..e333aaaa20 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -17,7 +17,7 @@ package bufimagemodify import ( "context" - "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/descriptorpb" diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go index 4f192f255b..9d74e75d4c 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go @@ -17,7 +17,7 @@ package bufimagemodify import ( "fmt" - "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/pkg/normalpath" "google.golang.org/protobuf/types/descriptorpb" diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go index fb6b85f024..85e9931704 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go @@ -19,11 +19,11 @@ import ( "path/filepath" "testing" - "github.com/bufbuild/buf/private/bufnew/bufconfig" - "github.com/bufbuild/buf/private/bufnew/bufconfig/bufconfigtest" - "github.com/bufbuild/buf/private/bufnew/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufconfig/bufconfigtest" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagetesting" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" From 57b0328a4d15042d6d1dba149d388bd5ef0fdf8a Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 27 Nov 2023 18:20:16 -0500 Subject: [PATCH 29/66] geneate.go; there's still work to do --- private/buf/buffetch/buffetch.go | 7 + private/buf/bufgen/bufgen.go | 43 ++- private/buf/bufgen/config_test.go | 4 +- private/buf/bufgen/features.go | 2 +- private/buf/bufgen/generator.go | 272 ++++++++++++++---- .../buf/cmd/buf/command/generate/generate.go | 107 +++---- .../bufpkg/bufconfig/generate_input_config.go | 4 + 7 files changed, 305 insertions(+), 134 deletions(-) diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index e531e67f40..ebdfcfcd41 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -41,8 +41,15 @@ const ( useProtoNamesKey = "use_proto_names" useEnumNumbersKey = "use_enum_numbers" + + // TODO: + // SubDirKey Key = "subdir" ) +// TODO: +// type Key struct +// GetModuleRef(context.Context, string, map[Key]string) + var ( // MessageFormatsString is the string representation of all message formats. // diff --git a/private/buf/bufgen/bufgen.go b/private/buf/bufgen/bufgen.go index af75689a45..a98df14cde 100644 --- a/private/buf/bufgen/bufgen.go +++ b/private/buf/bufgen/bufgen.go @@ -23,8 +23,8 @@ import ( "fmt" "strconv" - "github.com/bufbuild/buf/private/bufnew/bufconfig" - "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/buf/bufctl" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" @@ -108,7 +108,6 @@ type Generator interface { ctx context.Context, container app.EnvStdioContainer, config bufconfig.GenerateConfig, - image bufimage.Image, options ...GenerateOption, ) error } @@ -116,6 +115,7 @@ type Generator interface { // NewGenerator returns a new Generator. func NewGenerator( logger *zap.Logger, + controller bufctl.Controller, storageosProvider storageos.Provider, runner command.Runner, wasmPluginExecutor bufwasm.PluginExecutor, @@ -123,6 +123,7 @@ func NewGenerator( ) Generator { return newGenerator( logger, + controller, storageosProvider, runner, wasmPluginExecutor, @@ -169,6 +170,42 @@ func GenerateWithWASMEnabled() GenerateOption { } } +// GenerateWithInputOverride says to generate for this input only, ignoring inputs +// from the config. +func GenerateWithInputOverride(input string) GenerateOption { + return func(generateOptions *generateOptions) { + generateOptions.input = input + } +} + +// GenerateWithModuleConfigPath says to build the image with a module config at this path. +func GenerateWithModuleConfigPath(moduleConfigPath string) GenerateOption { + return func(generateOptions *generateOptions) { + generateOptions.moduleConfigPath = moduleConfigPath + } +} + +// GenerateWithIncludePaths says to only generate code for these paths. +func GenerateWithIncludePaths(includePaths []string) GenerateOption { + return func(generateOptions *generateOptions) { + generateOptions.includePaths = includePaths + } +} + +// GenerateWithExcludePaths says to not generate code for these paths. +func GenerateWithExcludePaths(excludePaths []string) GenerateOption { + return func(generateOptions *generateOptions) { + generateOptions.excludePaths = excludePaths + } +} + +// GenerateWithIncludeTypes says to only generate code for these types. +func GenerateWithIncludeTypes(includeTypes []string) GenerateOption { + return func(generateOptions *generateOptions) { + generateOptions.includeTypes = includeTypes + } +} + // Config is a configuration. type Config struct { // Required diff --git a/private/buf/bufgen/config_test.go b/private/buf/bufgen/config_test.go index 99eb7e0a78..1e1dc541c0 100644 --- a/private/buf/bufgen/config_test.go +++ b/private/buf/bufgen/config_test.go @@ -737,10 +737,10 @@ func assertEqualModuleFullNameKeyedMaps[V any](t *testing.T, m1 map[bufmodule.Mo keyedM1 := make(map[string]V, len(m1)) keyedM2 := make(map[string]V, len(m2)) for k, v := range m1 { - keyedM1[k.IdentityString()] = v + keyedM1[k.String()] = v } for k, v := range m2 { - keyedM2[k.IdentityString()] = v + keyedM2[k.String()] = v } require.Equal(t, keyedM1, keyedM2) } diff --git a/private/buf/bufgen/features.go b/private/buf/bufgen/features.go index 8ffdbaec3f..cd2b434cac 100644 --- a/private/buf/bufgen/features.go +++ b/private/buf/bufgen/features.go @@ -19,7 +19,7 @@ import ( "sort" "strings" - "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/pkg/app" "google.golang.org/protobuf/types/descriptorpb" diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index e625a9249c..b30499ca4b 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -19,9 +19,11 @@ import ( "errors" "fmt" "path/filepath" + "strconv" connect "connectrpc.com/connect" - "github.com/bufbuild/buf/private/bufnew/bufconfig" + "github.com/bufbuild/buf/private/buf/bufctl" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify" "github.com/bufbuild/buf/private/bufpkg/bufplugin" @@ -42,8 +44,11 @@ import ( "google.golang.org/protobuf/types/pluginpb" ) +const defaultInput = "." + type generator struct { logger *zap.Logger + controller bufctl.Controller storageosProvider storageos.Provider pluginexecGenerator bufpluginexec.Generator clientConfig *connectclient.Config @@ -51,6 +56,7 @@ type generator struct { func newGenerator( logger *zap.Logger, + controller bufctl.Controller, storageosProvider storageos.Provider, runner command.Runner, wasmPluginExecutor bufwasm.PluginExecutor, @@ -58,6 +64,7 @@ func newGenerator( ) *generator { return &generator{ logger: logger, + controller: controller, storageosProvider: storageosProvider, pluginexecGenerator: bufpluginexec.NewGenerator(logger, storageosProvider, runner, wasmPluginExecutor), clientConfig: clientConfig, @@ -85,46 +92,62 @@ func (g *generator) Generate( ctx context.Context, container app.EnvStdioContainer, config bufconfig.GenerateConfig, - image bufimage.Image, options ...GenerateOption, ) error { generateOptions := newGenerateOptions() for _, option := range options { option(generateOptions) } - return g.generate( + inputImages, err := getInputImages( ctx, - container, + g.logger, + g.controller, + generateOptions.input, config, - image, - generateOptions.baseOutDirPath, - generateOptions.includeImports, - generateOptions.includeWellKnownTypes, - generateOptions.wasmEnabled, + generateOptions.moduleConfigPath, + generateOptions.includePaths, + generateOptions.excludePaths, + generateOptions.includeTypes, ) + if err != nil { + return err + } + for _, inputImage := range inputImages { + if err := bufimagemodify.Modify(ctx, inputImage, config.GenerateManagedConfig()); err != nil { + return err + } + if err := g.generateCode( + ctx, + container, + inputImage, + generateOptions.baseOutDirPath, + config.GeneratePluginConfigs(), + generateOptions.includeImports, + generateOptions.includeWellKnownTypes, + ); err != nil { + return err + } + } + return nil } -func (g *generator) generate( +func (g *generator) generateCode( ctx context.Context, container app.EnvStdioContainer, - config bufconfig.GenerateConfig, - image bufimage.Image, - baseOutDirPath string, - includeImports bool, - includeWellKnownTypes bool, - wasmEnabled bool, + inputImage bufimage.Image, + baseOutDir string, + pluginConfigs []bufconfig.GeneratePluginConfig, + alwaysIncludeImports bool, + alwaysIncludeWKT bool, ) error { - if err := modifyImage(ctx, g.logger, config.GenerateManagedConfig(), image); err != nil { - return err - } responses, err := g.execPlugins( ctx, container, - config.GeneratePluginConfigs(), - image, - includeImports, - includeWellKnownTypes, - wasmEnabled, + pluginConfigs, + inputImage, + alwaysIncludeImports, + alwaysIncludeWKT, + false, // wasm enabled is false ) if err != nil { return err @@ -135,10 +158,10 @@ func (g *generator) generate( g.storageosProvider, appprotoos.ResponseWriterWithCreateOutDirIfNotExists(), ) - for i, pluginConfig := range config.GeneratePluginConfigs() { + for i, pluginConfig := range pluginConfigs { out := pluginConfig.Out() - if baseOutDirPath != "" && baseOutDirPath != "." { - out = filepath.Join(baseOutDirPath, out) + if baseOutDir != "" && baseOutDir != "." { + out = filepath.Join(baseOutDir, out) } response := responses[i] if response == nil { @@ -158,13 +181,151 @@ func (g *generator) generate( return nil } +// TODO: this is a very temporary solution, although it would be nice if buffetch exposes function that parses ref from a map +func refStringForInputConfig( + ctx context.Context, + logger *zap.Logger, + inputConfig bufconfig.GenerateInputConfig, +) string { + var refString string + var refOptionKeyToValue map[string]string + switch { + case inputConfig.Module() != "": + refString = inputConfig.Module() + case inputConfig.Directory() != "": + refString = inputConfig.Directory() + case inputConfig.ProtoFile() != "": + refString = inputConfig.ProtoFile() + case inputConfig.Tarball() != "": + refString = inputConfig.Tarball() + case inputConfig.ZipArchive() != "": + refString = inputConfig.ZipArchive() + case inputConfig.BinaryImage() != "": + refString = inputConfig.BinaryImage() + case inputConfig.JSONImage() != "": + refString = inputConfig.JSONImage() + case inputConfig.TextImage() != "": + refString = inputConfig.TextImage() + case inputConfig.GitRepo() != "": + refString = inputConfig.GitRepo() + } + if inputConfig.Compression() != "" { + refOptionKeyToValue["compression"] = inputConfig.Compression() + } + if inputConfig.StripComponent() != 0 { + refOptionKeyToValue["strip_components"] = strconv.FormatUint(uint64(inputConfig.StripComponent()), 10) + } + if inputConfig.Subdir() != "" { + refOptionKeyToValue["subdir"] = inputConfig.Subdir() + } + if inputConfig.Branch() != "" { + refOptionKeyToValue["branch"] = inputConfig.Branch() + } + if inputConfig.Tag() != "" { + refOptionKeyToValue["tag"] = inputConfig.Tag() + } + if inputConfig.Ref() != "" { + refOptionKeyToValue["ref"] = inputConfig.Ref() + } + // TODO: != 0 + if inputConfig.Depth() != "" { + refOptionKeyToValue["depth"] = inputConfig.Depth() + } + if inputConfig.RecurseSubmodules() { + refOptionKeyToValue["recurse_submodules"] = "true" + } + if inputConfig.IncludePackageFiles() { + refOptionKeyToValue["include_package_files"] = "true" + } + if len(refOptionKeyToValue) == 0 { + return refString + } + refString += "#" + for key, value := range refOptionKeyToValue { + refString += key + "=" + value + } + return refString +} + +func getInputImages( + ctx context.Context, + logger *zap.Logger, + controller bufctl.Controller, + inputSpecified string, + config bufconfig.GenerateConfig, + moduleConfigOverride string, + includePathsOverride []string, + excludePathsOverride []string, + includeTypesOverride []string, +) ([]bufimage.Image, error) { + var inputImages []bufimage.Image + // If input is specified on the command line, we use that. If input is not + // specified on the command line, but the config has no inputs, use the default input. + if inputSpecified != "" || len(config.GenerateInputConfigs()) == 0 { + input := defaultInput + if inputSpecified != "" { + input = inputSpecified + } + var includeTypes []string + if typesConfig := config.GenerateTypeConfig().IncludeTypes(); typesConfig != nil { + includeTypes = typesConfig + } + if len(includeTypesOverride) > 0 { + includeTypes = includeTypesOverride + } + inputImage, err := controller.GetImage( + ctx, + input, + bufctl.WithConfigOverride(moduleConfigOverride), + bufctl.WithTargetPaths(includePathsOverride, excludePathsOverride), + bufctl.WithImageTypes(includeTypes), + ) + if err != nil { + return nil, err + } + inputImages = []bufimage.Image{inputImage} + } else { + for _, inputConfig := range config.GenerateInputConfigs() { + includePaths := inputConfig.IncludePaths() + if len(includePathsOverride) > 0 { + includePaths = includePathsOverride + } + excludePaths := inputConfig.ExcludePaths() + if len(excludePathsOverride) > 0 { + excludePaths = excludePathsOverride + } + // In V2 we do not need to look at inputConfig.GenerateTypeConfig().IncludeTypes() + // because inputConfig.GenerateTypeConfig() is always nil. + // TODO: document the above in godoc + includeTypes := inputConfig.Types() + if len(includeTypesOverride) > 0 { + includeTypes = includeTypesOverride + } + input := refStringForInputConfig(ctx, logger, inputConfig) + inputImage, err := controller.GetImage( + ctx, + input, + bufctl.WithConfigOverride(moduleConfigOverride), + bufctl.WithTargetPaths(includePaths, excludePaths), + bufctl.WithImageTypes(includeTypes), + ) + if err != nil { + return nil, err + } + inputImages = append(inputImages, inputImage) + } + } + return inputImages, nil +} + func (g *generator) execPlugins( ctx context.Context, container app.EnvStdioContainer, pluginConfigs []bufconfig.GeneratePluginConfig, image bufimage.Image, - includeImports bool, - includeWellKnownTypes bool, + alwaysIncludeImports bool, + alwaysIncludeWellKnownTypes bool, + // TODO: perhaps clean this up wasmEnabled bool, ) ([]*pluginpb.CodeGeneratorResponse, error) { imageProvider := newImageProvider(image) @@ -192,8 +353,9 @@ func (g *generator) execPlugins( container, imageProvider, currentPluginConfig, - includeImports, - includeWellKnownTypes, + // TODO: can the user override this to false on the command line? i.e. is `buf generate --include-imports=false` possible? + alwaysIncludeImports || currentPluginConfig.IncludeImports(), + alwaysIncludeWellKnownTypes || currentPluginConfig.IncludeWKT(), wasmEnabled, ) if err != nil { @@ -224,8 +386,8 @@ func (g *generator) execPlugins( image, remote, v2Args, - includeImports, - includeWellKnownTypes, + alwaysIncludeImports, + alwaysIncludeWellKnownTypes, ) if err != nil { return err @@ -326,12 +488,16 @@ func (g *generator) execRemotePluginsV2( image bufimage.Image, remote string, pluginConfigs []*remotePluginExecArgs, - includeImports bool, - includeWellKnownTypes bool, + alwaysIncludeImports bool, + alwaysIncludeWellKnownTypes bool, ) ([]*remotePluginExecutionResult, error) { requests := make([]*registryv1alpha1.PluginGenerationRequest, len(pluginConfigs)) for i, pluginConfig := range pluginConfigs { - request, err := getPluginGenerationRequest(pluginConfig.PluginConfig) + request, err := getPluginGenerationRequest( + pluginConfig.PluginConfig, + alwaysIncludeImports || pluginConfig.PluginConfig.IncludeImports(), + alwaysIncludeWellKnownTypes || pluginConfig.PluginConfig.IncludeWKT(), + ) if err != nil { return nil, err } @@ -344,8 +510,8 @@ func (g *generator) execRemotePluginsV2( ®istryv1alpha1.GenerateCodeRequest{ Image: bufimage.ImageToProtoImage(image), Requests: requests, - IncludeImports: includeImports, - IncludeWellKnownTypes: includeWellKnownTypes, + IncludeImports: alwaysIncludeImports, + IncludeWellKnownTypes: alwaysIncludeWellKnownTypes, }, ), ) @@ -372,6 +538,8 @@ func (g *generator) execRemotePluginsV2( func getPluginGenerationRequest( pluginConfig bufconfig.GeneratePluginConfig, + includeImports bool, + includeWKT bool, ) (*registryv1alpha1.PluginGenerationRequest, error) { var curatedPluginReference *registryv1alpha1.CuratedPluginReference if reference, err := bufpluginref.PluginReferenceForString(pluginConfig.Name(), pluginConfig.Revision()); err == nil { @@ -390,25 +558,13 @@ func getPluginGenerationRequest( options = []string{pluginConfig.Opt()} } return ®istryv1alpha1.PluginGenerationRequest{ - PluginReference: curatedPluginReference, - Options: options, + PluginReference: curatedPluginReference, + Options: options, + IncludeImports: &includeImports, + IncludeWellKnownTypes: &includeWKT, }, nil } -// modifyImage modifies the image according to the given configuration (i.e. managed mode). -func modifyImage( - ctx context.Context, - logger *zap.Logger, - config bufconfig.GenerateManagedConfig, - image bufimage.Image, -) error { - return bufimagemodify.Modify( - ctx, - image, - config, - ) -} - // validateResponses verifies that a response is set for each of the // pluginConfigs, and that each generated file is generated by a single // plugin. @@ -441,10 +597,18 @@ func validateResponses( } type generateOptions struct { + // plugin specific options: baseOutDirPath string includeImports bool includeWellKnownTypes bool wasmEnabled bool + // image/input specific options: + input string + moduleConfigPath string + // TODO: unify naming: includePaths / pathsIncluded / pathSpecified + includePaths []string + excludePaths []string + includeTypes []string } func newGenerateOptions() *generateOptions { diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 9443a62241..c3eb53e530 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -17,15 +17,16 @@ package generate import ( "context" "fmt" + "io" + "os" "path/filepath" + "strings" "github.com/bufbuild/buf/private/buf/bufcli" - "github.com/bufbuild/buf/private/buf/buffetch" - "github.com/bufbuild/buf/private/buf/bufgen" "github.com/bufbuild/buf/private/buf/bufctl" + "github.com/bufbuild/buf/private/buf/bufgen" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimageutil" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufwasm" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appflag" @@ -294,33 +295,26 @@ func run( // in the context of including imports. return appcmd.NewInvalidArgumentErrorf("Cannot set --%s without --%s", includeWKTFlagName, includeImportsFlagName) } - if err := bufcli.ValidateErrorFormatFlag(flags.ErrorFormat, errorFormatFlagName); err != nil { - return err - } - input, err := bufcli.GetInputValue(container, flags.InputHashtag, ".") + input, err := bufcli.GetInputValue(container, flags.InputHashtag, "") if err != nil { return err } - ref, err := buffetch.NewRefParser(container.Logger()).GetRef(ctx, input) - if err != nil { - return err + var storageosProvider storageos.Provider + if flags.DisableSymlinks { + storageosProvider = storageos.NewProvider() + } else { + storageosProvider = storageos.NewProvider(storageos.ProviderWithSymlinks()) } - storageosProvider := bufcli.NewStorageosProvider(flags.DisableSymlinks) - runner := command.NewRunner() - readWriteBucket, err := storageosProvider.NewReadWriteBucket( - ".", - storageos.ReadWriteBucketWithSymlinksIfSupported(), + controller, err := bufcli.NewController( + container, + bufctl.WithDisableSymlinks(flags.DisableSymlinks), + bufctl.WithFileAnnotationErrorFormat(flags.ErrorFormat), ) if err != nil { return err } - genConfig, err := bufgen.ReadConfig( - ctx, - logger, - bufgen.NewProvider(logger), - readWriteBucket, - bufgen.ReadConfigWithOverride(flags.Template), - ) + wasmPluginExecutor, err := bufwasm.NewPluginExecutor( + filepath.Join(container.CacheDirPath(), bufcli.WASMCompilationCacheDir)) if err != nil { return err } @@ -328,44 +322,27 @@ func run( if err != nil { return err } - imageConfigReader, err := bufcli.NewWireImageConfigReader( - container, - storageosProvider, - runner, - clientConfig, - ) - if err != nil { - return err - } - imageConfigs, fileAnnotations, err := imageConfigReader.GetImageConfigs( - ctx, - container, - ref, - flags.Config, - flags.Paths, // we filter on files - flags.ExcludePaths, // we exclude these paths - false, // input files must exist - false, // we must include source info for generation - ) - if err != nil { - return err - } - if len(fileAnnotations) > 0 { - if err := bufanalysis.PrintFileAnnotations(container.Stderr(), fileAnnotations, flags.ErrorFormat); err != nil { + var configReader io.Reader + templatePathExtension := filepath.Ext(flags.Template) + if flags.Template == "" || templatePathExtension == ".yaml" || templatePathExtension == ".yml" || templatePathExtension == ".json" { + configReader, err = os.Open(flags.Template) + if err != nil { return err } - return bufctl.ErrFileAnnotation + } else { + configReader = strings.NewReader(flags.Template) } - images := make([]bufimage.Image, 0, len(imageConfigs)) - for _, imageConfig := range imageConfigs { - images = append(images, imageConfig.Image()) - } - image, err := bufimage.MergeImages(images...) + config, err := bufconfig.ReadBufGenYAMLFile(configReader) if err != nil { return err } generateOptions := []bufgen.GenerateOption{ bufgen.GenerateWithBaseOutDirPath(flags.BaseOutDirPath), + bufgen.GenerateWithIncludePaths(flags.Paths), + bufgen.GenerateWithExcludePaths(flags.ExcludePaths), + bufgen.GenerateWithIncludeTypes(flags.Types), + bufgen.GenerateWithInputOverride(input), + bufgen.GenerateWithModuleConfigPath(flags.Config), } if flags.IncludeImports { generateOptions = append( @@ -389,35 +366,17 @@ func run( bufgen.GenerateWithWASMEnabled(), ) } - var includedTypes []string - if len(flags.Types) > 0 || len(flags.TypesDeprecated) > 0 { - // command-line flags take precedence - includedTypes = append(flags.Types, flags.TypesDeprecated...) - } else if genConfig.TypesConfig != nil { - includedTypes = genConfig.TypesConfig.Include - } - if len(includedTypes) > 0 { - image, err = bufimageutil.ImageFilteredByTypes(image, includedTypes...) - if err != nil { - return err - } - } - wasmPluginExecutor, err := bufwasm.NewPluginExecutor( - filepath.Join(container.CacheDirPath(), bufcli.WASMCompilationCacheDir)) - if err != nil { - return err - } return bufgen.NewGenerator( logger, + controller, storageosProvider, - runner, + command.NewRunner(), wasmPluginExecutor, clientConfig, ).Generate( ctx, container, - genConfig, - image, + config, generateOptions..., ) } diff --git a/private/bufpkg/bufconfig/generate_input_config.go b/private/bufpkg/bufconfig/generate_input_config.go index 6414ef75a0..08b1dc003e 100644 --- a/private/bufpkg/bufconfig/generate_input_config.go +++ b/private/bufpkg/bufconfig/generate_input_config.go @@ -20,6 +20,10 @@ type GenerateInputConfig interface { // Exactly one of Module, Directory, ProtoFile, Tarball, ZipArchive, BinaryImage, // JSONImage or GitRepo is not empty, indicating the input format. + // TODO: + // Type() InputType + // Location() string + // Module returns the module name. Module() string // Direcotry returns the path to the directory, relative or absolute. From c6237c73f233229319b6b7f41097333c486ef2d5 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 27 Nov 2023 18:57:59 -0500 Subject: [PATCH 30/66] update input interface --- .../bufpkg/bufconfig/generate_input_config.go | 49 +++++++++---------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_input_config.go b/private/bufpkg/bufconfig/generate_input_config.go index 6414ef75a0..db4d3b60ed 100644 --- a/private/bufpkg/bufconfig/generate_input_config.go +++ b/private/bufpkg/bufconfig/generate_input_config.go @@ -15,35 +15,32 @@ package bufconfig +type InputConfigType int + +const ( + InputConfigTypeModule InputConfigType = iota + 1 + InputConfigTypeDirectory + InputConfigTypeGitRepo + InputConfigTypeProtoFile + InputConfigTypeTarball + InputConfigTypeZipArchive + InputConfigTypeBinaryImage + InputConfigTypeJSONImage + InputConfigTypeTextImage +) + // GenerateInputConfig is an input configuration for code generation. type GenerateInputConfig interface { - // Exactly one of Module, Directory, ProtoFile, Tarball, ZipArchive, BinaryImage, - // JSONImage or GitRepo is not empty, indicating the input format. - - // Module returns the module name. - Module() string - // Direcotry returns the path to the directory, relative or absolute. - Directory() string - // ProtoFile returns the path to the proto file, relative or absolute. - ProtoFile() string - // Tarball returns the path to the tarball, relative or absolute. - Tarball() string - // ZipArchive returns the path to the zip archive, relative or absolute. - ZipArchive() string - // BinaryImage returns the path to the binary buf image, relative or absolute. - BinaryImage() string - // JSONImage returns the path to the JSON buf image, relative or absolute. - JSONImage() string - // TextImage returns the path to the text buf image, relative or absolute. - TextImage() string - // GitRepo returns the git repository address in https, ssh or local format. - GitRepo() string + // Type returns the input type. + Type() InputConfigType + // Location returns the location for the input. + Location() string // Compression returns the compression scheme, not empty only if format is // one of tarball, binary image, json image or text image. Compression() string - // StripComponent returns the number of directories to strip for tar or zip + // StripComponents returns the number of directories to strip for tar or zip // inputs, not empty only if format is tarball or zip archive. - StripComponent() uint32 + StripComponents() *uint32 // Subdir returns the subdirectory to use, not empty only if format is one // git repo, tarball and zip archive. Subdir() string @@ -54,16 +51,16 @@ type GenerateInputConfig interface { // Ref returns the git ref to checkout, not empty only if format is git. Ref() string // Ref returns the depth to clone the git repo with, not empty only if format is git. - Depth() string + Depth() *uint32 // RecurseSubmodules returns whether to clone submodules recursively. Not empty // only if input if git. RecurseSubmodules() bool // IncludePackageFiles returns other files in the same package as the proto file, // not empty only if format is proto file. IncludePackageFiles() bool - // Types returns the types to generate. If GenerateConfig.GenerateTypeConfig() + // IncludeTypes returns the types to generate. If GenerateConfig.GenerateTypeConfig() // returns a non-empty list of types. - Types() []string + IncludeTypes() []string // ExcludePaths returns paths not to generate for. ExcludePaths() []string // IncludePaths returns paths to generate for. From 657c761cd0b8eb6964f16391381aff5835d89d58 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 28 Nov 2023 11:26:38 -0500 Subject: [PATCH 31/66] commit --- private/buf/bufgen/generator.go | 51 ++++--------------- private/buf/cmd/buf/buf.go | 3 +- .../buf/cmd/buf/command/generate/generate.go | 33 ++++++++---- 3 files changed, 36 insertions(+), 51 deletions(-) diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index b30499ca4b..f3fe436a05 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -187,33 +187,13 @@ func refStringForInputConfig( logger *zap.Logger, inputConfig bufconfig.GenerateInputConfig, ) string { - var refString string - var refOptionKeyToValue map[string]string - switch { - case inputConfig.Module() != "": - refString = inputConfig.Module() - case inputConfig.Directory() != "": - refString = inputConfig.Directory() - case inputConfig.ProtoFile() != "": - refString = inputConfig.ProtoFile() - case inputConfig.Tarball() != "": - refString = inputConfig.Tarball() - case inputConfig.ZipArchive() != "": - refString = inputConfig.ZipArchive() - case inputConfig.BinaryImage() != "": - refString = inputConfig.BinaryImage() - case inputConfig.JSONImage() != "": - refString = inputConfig.JSONImage() - case inputConfig.TextImage() != "": - refString = inputConfig.TextImage() - case inputConfig.GitRepo() != "": - refString = inputConfig.GitRepo() - } + var refString = inputConfig.Location() + refOptionKeyToValue := map[string]string{} if inputConfig.Compression() != "" { refOptionKeyToValue["compression"] = inputConfig.Compression() } - if inputConfig.StripComponent() != 0 { - refOptionKeyToValue["strip_components"] = strconv.FormatUint(uint64(inputConfig.StripComponent()), 10) + if inputConfig.StripComponents() != nil { + refOptionKeyToValue["strip_components"] = strconv.FormatUint(uint64(*inputConfig.StripComponents()), 10) } if inputConfig.Subdir() != "" { refOptionKeyToValue["subdir"] = inputConfig.Subdir() @@ -228,8 +208,8 @@ func refStringForInputConfig( refOptionKeyToValue["ref"] = inputConfig.Ref() } // TODO: != 0 - if inputConfig.Depth() != "" { - refOptionKeyToValue["depth"] = inputConfig.Depth() + if inputConfig.Depth() != nil { + refOptionKeyToValue["depth"] = strconv.FormatUint(uint64(*inputConfig.Depth()), 10) } if inputConfig.RecurseSubmodules() { refOptionKeyToValue["recurse_submodules"] = "true" @@ -267,8 +247,8 @@ func getInputImages( input = inputSpecified } var includeTypes []string - if typesConfig := config.GenerateTypeConfig().IncludeTypes(); typesConfig != nil { - includeTypes = typesConfig + if typesConfig := config.GenerateTypeConfig(); typesConfig != nil { + includeTypes = typesConfig.IncludeTypes() } if len(includeTypesOverride) > 0 { includeTypes = includeTypesOverride @@ -297,7 +277,7 @@ func getInputImages( // In V2 we do not need to look at inputConfig.GenerateTypeConfig().IncludeTypes() // because inputConfig.GenerateTypeConfig() is always nil. // TODO: document the above in godoc - includeTypes := inputConfig.Types() + includeTypes := inputConfig.IncludeTypes() if len(includeTypesOverride) > 0 { includeTypes = includeTypesOverride } @@ -325,7 +305,6 @@ func (g *generator) execPlugins( image bufimage.Image, alwaysIncludeImports bool, alwaysIncludeWellKnownTypes bool, - // TODO: perhaps clean this up wasmEnabled bool, ) ([]*pluginpb.CodeGeneratorResponse, error) { imageProvider := newImageProvider(image) @@ -370,22 +349,14 @@ func (g *generator) execPlugins( for remote, indexedPluginConfigs := range remotePluginConfigTable { remote := remote indexedPluginConfigs := indexedPluginConfigs - v2Args := make([]*remotePluginExecArgs, 0, len(indexedPluginConfigs)) - for _, param := range indexedPluginConfigs { - // TODO: check that it's ok to skip it. It's fine because remote is not a thing anymore. - // if param.PluginConfig.Remote != "" { - // return nil, fmt.Errorf("invalid plugin reference: %s", param.PluginConfig.Remote) - // } - v2Args = append(v2Args, param) - } - if len(v2Args) > 0 { + if len(indexedPluginConfigs) > 0 { jobs = append(jobs, func(ctx context.Context) error { results, err := g.execRemotePluginsV2( ctx, container, image, remote, - v2Args, + indexedPluginConfigs, alwaysIncludeImports, alwaysIncludeWellKnownTypes, ) diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index c8b42d5e2b..d0448015a5 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -60,6 +60,7 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/build" "github.com/bufbuild/buf/private/buf/cmd/buf/command/convert" "github.com/bufbuild/buf/private/buf/cmd/buf/command/curl" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/generate" "github.com/bufbuild/buf/private/buf/cmd/buf/command/lint" "github.com/bufbuild/buf/private/buf/cmd/buf/command/lsfiles" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modclearcache" @@ -104,7 +105,7 @@ func NewRootCommand(name string) *appcmd.Command { //format.NewCommand("format", builder), lint.NewCommand("lint", builder), breaking.NewCommand("breaking", builder), - //generate.NewCommand("generate", builder), + generate.NewCommand("generate", builder), lsfiles.NewCommand("ls-files", builder), //push.NewCommand("push", builder), convert.NewCommand("convert", builder), diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index c3eb53e530..2d9cb6287d 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -17,7 +17,6 @@ package generate import ( "context" "fmt" - "io" "os" "path/filepath" "strings" @@ -322,19 +321,33 @@ func run( if err != nil { return err } - var configReader io.Reader + var config bufconfig.GenerateConfig templatePathExtension := filepath.Ext(flags.Template) - if flags.Template == "" || templatePathExtension == ".yaml" || templatePathExtension == ".yml" || templatePathExtension == ".json" { - configReader, err = os.Open(flags.Template) + switch { + case flags.Template == "": + bucket, err := storageosProvider.NewReadWriteBucket(".", storageos.ReadWriteBucketWithSymlinksIfSupported()) + if err != nil { + return err + } + config, err = bufconfig.GetBufGenYAMLFileForPrefix(ctx, bucket, ".") + if err != nil { + return err + } + case templatePathExtension == ".yaml" || templatePathExtension == ".yml" || templatePathExtension == ".json": + // We should not read from a bucket at "." because this path can jump context. + configReader, err := os.Open(flags.Template) + if err != nil { + return err + } + config, err = bufconfig.ReadBufGenYAMLFile(configReader) + if err != nil { + return err + } + default: + config, err = bufconfig.ReadBufGenYAMLFile(strings.NewReader(flags.Template)) if err != nil { return err } - } else { - configReader = strings.NewReader(flags.Template) - } - config, err := bufconfig.ReadBufGenYAMLFile(configReader) - if err != nil { - return err } generateOptions := []bufgen.GenerateOption{ bufgen.GenerateWithBaseOutDirPath(flags.BaseOutDirPath), From 678ec08fd66ed9fc515a6c2d003a36309fff7d99 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 28 Nov 2023 11:49:49 -0500 Subject: [PATCH 32/66] update placeholder --- private/bufpkg/bufconfig/generate_config.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index b83ba4fa39..d45969573f 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -73,24 +73,20 @@ type generateConfig struct { inputConfigs []GenerateInputConfig } -func newGenerateConfig() *generateConfig { - return &generateConfig{} +func (g *generateConfig) GeneratePluginConfigs() []GeneratePluginConfig { + return g.pluginConfigs } -func (*generateConfig) GeneratePluginConfigs() []GeneratePluginConfig { - return nil +func (g *generateConfig) GenerateManagedConfig() GenerateManagedConfig { + return g.managedConfig } -func (*generateConfig) GenerateManagedConfig() GenerateManagedConfig { - return nil +func (g *generateConfig) GenerateTypeConfig() GenerateTypeConfig { + return g.typeConfig } -func (*generateConfig) GenerateTypeConfig() GenerateTypeConfig { - return nil -} - -func (*generateConfig) GenerateInputConfigs() []GenerateInputConfig { - return nil +func (g *generateConfig) GenerateInputConfigs() []GenerateInputConfig { + return g.inputConfigs } func (*generateConfig) isGenerateConfig() {} From bccd90085e5b2170bf078f241e1038172221459c Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 28 Nov 2023 15:30:24 -0500 Subject: [PATCH 33/66] parse v2 --- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 13 +- private/bufpkg/bufconfig/generate_config.go | 33 ++- .../bufconfig/generate_external_config.go | 113 +++++++- .../bufpkg/bufconfig/generate_input_config.go | 262 +++++++++++++++++- .../bufconfig/generate_managed_config.go | 90 +++++- .../bufconfig/generate_plugin_config.go | 142 ++++++++++ 6 files changed, 638 insertions(+), 15 deletions(-) diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index f85e50fe7c..bdd5fe0548 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -133,7 +133,18 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error GenerateConfig: generateConfig, }, nil case FileVersionV2: - return nil, errors.New("TODO") + var externalGenYAMLFile externalBufGenYAMLFileV2 + if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { + return nil, fmt.Errorf("invalid as version %v: %w", fileVersion, err) + } + generateConfig, err := newGenerateConfigFromExternalFileV2(externalGenYAMLFile) + if err != nil { + return nil, err + } + return &bufGenYAMLFile{ + fileVersion: fileVersion, + GenerateConfig: generateConfig, + }, nil default: // This is a system error since we've already parsed. return nil, syserror.Newf("unknown FileVersion: %v", fileVersion) diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index d45969573f..e53c0938b4 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -38,10 +38,11 @@ type GenerateConfig interface { isGenerateConfig() } +// TODO: check if this is consistent with the rest of bufconfig (esp. its name and whether it should exist) func newGenerateConfigFromExternalFileV1( externalFile externalBufGenYAMLFileV1, ) (GenerateConfig, error) { - managedConfig, err := newManagedOverrideRuleFromExternalV1(externalFile.Managed) + managedConfig, err := newManagedConfigFromExternalV1(externalFile.Managed) if err != nil { return nil, err } @@ -59,8 +60,34 @@ func newGenerateConfigFromExternalFileV1( pluginConfigs: pluginConfigs, managedConfig: managedConfig, typeConfig: newGenerateTypeConfig(externalFile.Types.Include), - // TODO for v2 - inputConfigs: nil, + }, nil +} + +func newGenerateConfigFromExternalFileV2( + externalFile externalBufGenYAMLFileV2, +) (GenerateConfig, error) { + managedConfig, err := newManagedConfigFromExternalV2(externalFile.Managed) + if err != nil { + return nil, err + } + pluginConfigs, err := slicesext.MapError( + externalFile.Plugins, + newPluginConfigFromExternalV2, + ) + if err != nil { + return nil, err + } + inputConfigs, err := slicesext.MapError( + externalFile.Inputs, + newInputConfigFromExternalInputConfigV2, + ) + if err != nil { + return nil, err + } + return &generateConfig{ + managedConfig: managedConfig, + pluginConfigs: pluginConfigs, + inputConfigs: inputConfigs, }, nil } diff --git a/private/bufpkg/bufconfig/generate_external_config.go b/private/bufpkg/bufconfig/generate_external_config.go index 1e5592f882..b5a012fe97 100644 --- a/private/bufpkg/bufconfig/generate_external_config.go +++ b/private/bufpkg/bufconfig/generate_external_config.go @@ -240,13 +240,6 @@ type externalOptionsConfigV1Beta1 struct { OptimizeFor string `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` } -// TODO: remove this -// externalConfigVersion defines the subset of all config -// file versions that is used to determine the configuration version. -type externalConfigVersion struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` -} - // externalTypesConfigV1 is an external types configuration. type externalTypesConfigV1 struct { Include []string `json:"include,omitempty" yaml:"include"` @@ -256,3 +249,109 @@ type externalTypesConfigV1 struct { func (e externalTypesConfigV1) isEmpty() bool { return len(e.Include) == 0 } + +// externalBufGenYAMLFileV2 is an external configuration. +type externalBufGenYAMLFileV2 struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Managed externalGenerateManagedConfigV2 `json:"managed,omitempty" yaml:"managed,omitempty"` + Plugins []externalGeneratePluginConfigV2 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Inputs []externalInputConfigV2 `json:"inputs,omitempty" yaml:"inputs,omitempty"` +} + +// externalGenerateManagedConfigV2 is an external managed mode configuration. +type externalGenerateManagedConfigV2 struct { + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + Disable []externalManagedDisableConfigV2 `json:"disable,omitempty" yaml:"disable,omitempty"` + Override []externalManagedOverrideConfigV2 `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty. +func (m externalGenerateManagedConfigV2) isEmpty() bool { + return !m.Enabled && len(m.Disable) == 0 && len(m.Override) == 0 +} + +// externalManagedDisableConfigV2 is an external configuration that disables file options in +// managed mode. +type externalManagedDisableConfigV2 struct { + // At most one of FileOption and FieldOption can be set + // Must be validated to be a valid FileOption + FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` + // Must be validated to be a valid FieldOption + FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` + // Must be validated to be a valid module path + Module string `json:"module,omitempty" yaml:"module,omitempty"` + // Must be normalized and validated + Path string `json:"path,omitempty" yaml:"path,omitempty"` + // Must be validated to be a valid to be a valid field name. + Field string `json:"field,omitempty" yaml:"field,omitempty"` +} + +// externalManagedOverrideConfigV2 is an external configuration that overrides file options in +// managed mode. +type externalManagedOverrideConfigV2 struct { + // Must set exactly one of FileOption and FieldOption + // Must be validated to be a valid FileOption + FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` + // Must be validated to be a valid FieldOption + FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` + // Must be validated to be a valid module path + Module string `json:"module,omitempty" yaml:"module,omitempty"` + // Must be normalized and validated + Path string `json:"path,omitempty" yaml:"path,omitempty"` + // Must be validated to be a valid field name + Field string `json:"field,omitempty" yaml:"field,omitempty"` + // Required + Value interface{} `json:"value,omitempty" yaml:"value,omitempty"` +} + +// externalGeneratePluginConfigV2 is an external plugin configuration. +type externalGeneratePluginConfigV2 struct { + // Only one of Remote, Binary, Wasm, ProtocBuiltin can be set + Remote *string `json:"remote,omitempty" yaml:"remote,omitempty"` + // Can be multiple arguments + // All arguments must be strings + Binary interface{} `json:"binary,omitempty" yaml:"binary,omitempty"` + ProtocBuiltin *string `json:"protoc_builtin,omitempty" yaml:"protoc_builtin,omitempty"` + // Only valid with Remote + Revision *int `json:"revision,omitempty" yaml:"revision,omitempty"` + // Only valid with ProtocBuiltin + ProtocPath *string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` + // Required + Out string `json:"out,omitempty" yaml:"out,omitempty"` + // Can be one string or multiple strings + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + IncludeImports bool `json:"include_imports,omitempty" yaml:"include_imports,omitempty"` + IncludeWKT bool `json:"include_wkt,omitempty" yaml:"include_wkt,omitempty"` + // Must be a valid Strategy, only valid with ProtoBuiltin and Binary + Strategy *string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalInputConfigV2 is an external input configuration. +type externalInputConfigV2 struct { + // One and only one of Module, Directory, ProtoFile, Tarball, ZipArchive, BinaryImage, + // JSONImage and GitRepo must be specified as the format. + Module *string `json:"module,omitempty" yaml:"module,omitempty"` + Directory *string `json:"directory,omitempty" yaml:"directory,omitempty"` + ProtoFile *string `json:"proto_file,omitempty" yaml:"proto_file,omitempty"` + Tarball *string `json:"tarball,omitempty" yaml:"tarball,omitempty"` + ZipArchive *string `json:"zip_archive,omitempty" yaml:"zip_archive,omitempty"` + BinaryImage *string `json:"binary_image,omitempty" yaml:"binary_image,omitempty"` + JSONImage *string `json:"json_image,omitempty" yaml:"json_image,omitempty"` + TextImage *string `json:"text_image,omitempty" yaml:"text_image,omitempty"` + GitRepo *string `json:"git_repo,omitempty" yaml:"git_repo,omitempty"` + // Compression, StripComponents, Subdir, Branch, Tag, Ref, Depth, RecurseSubmodules + // and IncludePackageFils are available for only some formats. + Compression *string `json:"compression,omitempty" yaml:"compression,omitempty"` + StripComponents *uint32 `json:"strip_components,omitempty" yaml:"strip_components,omitempty"` + Subdir *string `json:"subdir,omitempty" yaml:"subdir,omitempty"` + Branch *string `json:"branch,omitempty" yaml:"branch,omitempty"` + Tag *string `json:"tag,omitempty" yaml:"tag,omitempty"` + Ref *string `json:"ref,omitempty" yaml:"ref,omitempty"` + Depth *uint32 `json:"depth,omitempty" yaml:"depth,omitempty"` + RecurseSubmodules *bool `json:"recurse_submodules,omitempty" yaml:"recurse_submodules,omitempty"` + IncludePackageFiles *bool `json:"include_package_files,omitempty" yaml:"include_package_files,omitempty"` + // Types, IncludePaths and ExcludePaths are available for all formats. + Types []string `json:"types,omitempty" yaml:"types,omitempty"` + IncludePaths []string `json:"include_paths,omitempty" yaml:"include_paths,omitempty"` + ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` +} diff --git a/private/bufpkg/bufconfig/generate_input_config.go b/private/bufpkg/bufconfig/generate_input_config.go index db4d3b60ed..63a69573d0 100644 --- a/private/bufpkg/bufconfig/generate_input_config.go +++ b/private/bufpkg/bufconfig/generate_input_config.go @@ -15,6 +15,13 @@ package bufconfig +import ( + "errors" + "fmt" + "strconv" +) + +// TODO: InputFormat? type InputConfigType int const ( @@ -29,6 +36,83 @@ const ( InputConfigTypeTextImage ) +// Implements fmt.Stringer +func (i InputConfigType) String() string { + s, ok := inputConfigTypeToString[i] + if !ok { + return strconv.Itoa(int(i)) + } + return s +} + +const ( + // TODO: move string literal to maps + formatGitRepo = "git_repo" + formatModule = "module" + formatDirectory = "directory" + formatProtoFile = "proto_file" + formatBinaryImage = "binary_image" + formatTarball = "tarball" + formatZipArchive = "zip_archive" + formatJSONImage = "json_image" + formatTextImage = "text_image" + optionCompression = "compression" + optionBranch = "branch" + optionTag = "tag" + optionRef = "ref" + optionDepth = "depth" + optionRecurseSubmodules = "recurse_submodules" + optionStripComponents = "strip_components" + optionSubdir = "subdir" + optionIncludePackageFiles = "include_package_files" +) + +var allowedOptionsForFormat = map[InputConfigType](map[string]bool){ + InputConfigTypeGitRepo: { + optionBranch: true, + optionTag: true, + optionRef: true, + optionDepth: true, + optionRecurseSubmodules: true, + optionSubdir: true, + }, + InputConfigTypeModule: {}, + InputConfigTypeDirectory: {}, + InputConfigTypeProtoFile: { + optionIncludePackageFiles: true, + }, + InputConfigTypeTarball: { + optionCompression: true, + optionStripComponents: true, + optionSubdir: true, + }, + InputConfigTypeZipArchive: { + optionStripComponents: true, + optionSubdir: true, + }, + InputConfigTypeBinaryImage: { + optionCompression: true, + }, + InputConfigTypeJSONImage: { + optionCompression: true, + }, + InputConfigTypeTextImage: { + optionCompression: true, + }, +} + +var inputConfigTypeToString = map[InputConfigType]string{ + InputConfigTypeGitRepo: formatGitRepo, + InputConfigTypeModule: formatModule, + InputConfigTypeDirectory: formatDirectory, + InputConfigTypeProtoFile: formatProtoFile, + InputConfigTypeTarball: formatTarball, + InputConfigTypeZipArchive: formatZipArchive, + InputConfigTypeBinaryImage: formatBinaryImage, + InputConfigTypeJSONImage: formatJSONImage, + InputConfigTypeTextImage: formatTextImage, +} + // GenerateInputConfig is an input configuration for code generation. type GenerateInputConfig interface { // Type returns the input type. @@ -58,13 +142,185 @@ type GenerateInputConfig interface { // IncludePackageFiles returns other files in the same package as the proto file, // not empty only if format is proto file. IncludePackageFiles() bool - // IncludeTypes returns the types to generate. If GenerateConfig.GenerateTypeConfig() - // returns a non-empty list of types. - IncludeTypes() []string // ExcludePaths returns paths not to generate for. ExcludePaths() []string // IncludePaths returns paths to generate for. IncludePaths() []string + // IncludeTypes returns the types to generate. If GenerateConfig.GenerateTypeConfig() + // returns a non-empty list of types. + IncludeTypes() []string isGenerateInputConfig() } + +type generateInputConfig struct { + inputType InputConfigType + location string + compression string + stripComponents *uint32 + subdir string + branch string + tag string + ref string + depth *uint32 + recurseSubmodules bool + includePackageFiles bool + includeTypes []string + excludePaths []string + includePaths []string +} + +func (g *generateInputConfig) Type() InputConfigType { + return g.inputType +} + +func (g *generateInputConfig) Location() string { + return g.location +} + +func (g *generateInputConfig) Compression() string { + return g.compression +} + +func (g *generateInputConfig) StripComponents() *uint32 { + return g.stripComponents +} + +func (g *generateInputConfig) Subdir() string { + return g.subdir +} + +func (g *generateInputConfig) Branch() string { + return g.branch +} + +func (g *generateInputConfig) Tag() string { + return g.tag +} + +func (g *generateInputConfig) Ref() string { + return g.ref +} + +func (g *generateInputConfig) Depth() *uint32 { + return g.depth +} + +func (g *generateInputConfig) RecurseSubmodules() bool { + return g.recurseSubmodules +} + +func (g *generateInputConfig) IncludePackageFiles() bool { + return g.includePackageFiles +} + +func (g *generateInputConfig) ExcludePaths() []string { + return g.excludePaths +} + +func (g *generateInputConfig) IncludePaths() []string { + return g.includePaths +} + +func (g *generateInputConfig) IncludeTypes() []string { + return g.includeTypes +} + +func (g *generateInputConfig) isGenerateInputConfig() {} + +func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV2) (GenerateInputConfig, error) { + inputConfig := &generateInputConfig{} + var inputTypes []InputConfigType + var options []string + if externalConfig.Module != nil { + inputTypes = append(inputTypes, InputConfigTypeModule) + inputConfig.location = *externalConfig.Module + } + if externalConfig.Directory != nil { + inputTypes = append(inputTypes, InputConfigTypeDirectory) + inputConfig.location = *externalConfig.Directory + } + if externalConfig.ProtoFile != nil { + inputTypes = append(inputTypes, InputConfigTypeProtoFile) + inputConfig.location = *externalConfig.ProtoFile + } + if externalConfig.BinaryImage != nil { + inputTypes = append(inputTypes, InputConfigTypeBinaryImage) + inputConfig.location = *externalConfig.BinaryImage + } + if externalConfig.Tarball != nil { + inputTypes = append(inputTypes, InputConfigTypeTarball) + inputConfig.location = *externalConfig.Tarball + } + if externalConfig.ZipArchive != nil { + inputTypes = append(inputTypes, InputConfigTypeZipArchive) + inputConfig.location = *externalConfig.ZipArchive + } + if externalConfig.JSONImage != nil { + inputTypes = append(inputTypes, InputConfigTypeJSONImage) + inputConfig.location = *externalConfig.JSONImage + } + if externalConfig.TextImage != nil { + inputTypes = append(inputTypes, InputConfigTypeTextImage) + inputConfig.location = *externalConfig.TextImage + } + if externalConfig.GitRepo != nil { + inputTypes = append(inputTypes, InputConfigTypeGitRepo) + inputConfig.location = *externalConfig.GitRepo + } + if externalConfig.Compression != nil { + options = append(options, optionCompression) + inputConfig.compression = *externalConfig.Compression + } + if externalConfig.StripComponents != nil { + options = append(options, optionStripComponents) + inputConfig.stripComponents = externalConfig.StripComponents + } + if externalConfig.Subdir != nil { + options = append(options, optionSubdir) + inputConfig.subdir = *externalConfig.Subdir + } + if externalConfig.Branch != nil { + options = append(options, optionBranch) + inputConfig.branch = *externalConfig.Branch + } + if externalConfig.Tag != nil { + options = append(options, optionTag) + inputConfig.tag = *externalConfig.Tag + } + if externalConfig.Ref != nil { + options = append(options, optionRef) + inputConfig.ref = *externalConfig.Ref + } + if externalConfig.Depth != nil { + options = append(options, optionDepth) + inputConfig.depth = externalConfig.Depth + } + if externalConfig.RecurseSubmodules != nil { + options = append(options, optionRecurseSubmodules) + inputConfig.recurseSubmodules = *externalConfig.RecurseSubmodules + } + if externalConfig.IncludePackageFiles != nil { + options = append(options, optionIncludePackageFiles) + inputConfig.includePackageFiles = *externalConfig.IncludePackageFiles + } + if len(inputTypes) == 0 { + return nil, errors.New("must specify input type") + } + if len(inputTypes) > 1 { + // TODO: print out all types allowed + return nil, fmt.Errorf("exactly one input type can be specified") + } + format := inputTypes[0] + allowedOptions, ok := allowedOptionsForFormat[format] + if !ok { + // this should not happen + return nil, fmt.Errorf("unable to find allowed options for format %v", format) + } + for _, option := range options { + if !allowedOptions[option] { + return nil, fmt.Errorf("option %s is not allowed for format %v", option, format) + } + } + return inputConfig, nil +} diff --git a/private/bufpkg/bufconfig/generate_managed_config.go b/private/bufpkg/bufconfig/generate_managed_config.go index 3a9b077a8d..12793cdaa4 100644 --- a/private/bufpkg/bufconfig/generate_managed_config.go +++ b/private/bufpkg/bufconfig/generate_managed_config.go @@ -223,7 +223,7 @@ type generateManagedConfig struct { overrides []ManagedOverrideRule } -func newManagedOverrideRuleFromExternalV1( +func newManagedConfigFromExternalV1( externalConfig externalGenerateManagedConfigV1, ) (GenerateManagedConfig, error) { if !externalConfig.Enabled { @@ -412,6 +412,94 @@ func newManagedOverrideRuleFromExternalV1( }, nil } +func newManagedConfigFromExternalV2( + externalConfig externalGenerateManagedConfigV2, +) (GenerateManagedConfig, error) { + if externalConfig.isEmpty() { + return nil, nil + } + // TODO: log warning if disabled but non-empty + var disables []ManagedDisableRule + var overrides []ManagedOverrideRule + for _, externalDisableConfig := range externalConfig.Disable { + var ( + fileOption FileOption + fieldOption FieldOption + err error + ) + if externalDisableConfig.FileOption != "" { + fileOption, err = parseFileOption(externalDisableConfig.FileOption) + if err != nil { + return nil, err + } + } + if externalDisableConfig.FieldOption != "" { + fieldOption, err = parseFieldOption(externalDisableConfig.FieldOption) + if err != nil { + return nil, err + } + } + disable, err := NewDisableRule( + externalDisableConfig.Path, + externalDisableConfig.Module, + externalDisableConfig.Field, + fileOption, + fieldOption, + ) + if err != nil { + return nil, err + } + disables = append(disables, disable) + } + for _, externalOverrideConfig := range externalConfig.Override { + if externalOverrideConfig.FileOption == "" && externalOverrideConfig.FieldOption == "" { + return nil, errors.New("must set file_option or field_option for an override") + } + if externalOverrideConfig.FileOption != "" && externalOverrideConfig.FieldOption != "" { + return nil, errors.New("exactly one of file_option and field_option must be set for an override") + } + if externalOverrideConfig.Value == nil { + return nil, errors.New("must set value for an override") + } + if externalOverrideConfig.FieldOption != "" { + fieldOption, err := parseFieldOption(externalOverrideConfig.FieldOption) + if err != nil { + return nil, err + } + override, err := NewFieldOptionOverrideRule( + externalOverrideConfig.Path, + externalOverrideConfig.Module, + externalOverrideConfig.Field, + fieldOption, + externalOverrideConfig.Value, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + continue + } + fileOption, err := parseFileOption(externalOverrideConfig.FileOption) + if err != nil { + return nil, err + } + override, err := NewFileOptionOverrideRule( + externalOverrideConfig.Path, + externalOverrideConfig.Module, + fileOption, + externalOverrideConfig.Value, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + return &generateManagedConfig{ + disables: disables, + overrides: overrides, + }, nil +} + func (g *generateManagedConfig) Disables() []ManagedDisableRule { return g.disables } diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index e6112825cc..bcf8ae7957 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -17,6 +17,8 @@ package bufconfig import ( "errors" "fmt" + "math" + "strings" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" @@ -271,6 +273,146 @@ func newPluginConfigFromExternalV1( ) } +// TODO: move this up, maybe let v1 use this as well +const ( + typeRemote = "remote" + typeBinary = "binary" + typeProtocBuiltin = "protoc_builtin" +) + +const ( + optionRevision = "revision" + optionProtocPath = "protoc_path" + optionStrategy = "strategy" +) + +var allowedOptionsForType = map[string](map[string]bool){ + typeRemote: { + optionRevision: true, + }, + typeBinary: { + optionStrategy: true, + }, + typeProtocBuiltin: { + optionProtocPath: true, + optionStrategy: true, + }, +} + +func getTypesAndOptions(externalConfig externalGeneratePluginConfigV2) ([]string, []string, error) { + var ( + types []string + options []string + ) + if externalConfig.Remote != nil { + types = append(types, typeRemote) + } + path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Binary) + if err != nil { + return nil, nil, err + } + if len(path) > 0 { + types = append(types, typeBinary) + } + if externalConfig.ProtocBuiltin != nil { + types = append(types, typeProtocBuiltin) + } + if externalConfig.Revision != nil { + options = append(options, optionRevision) + } + if externalConfig.ProtocPath != nil { + options = append(options, optionProtocPath) + } + if externalConfig.Strategy != nil { + options = append(options, optionStrategy) + } + return types, options, nil +} + +func newPluginConfigFromExternalV2( + externalConfig externalGeneratePluginConfigV2, +) (GeneratePluginConfig, error) { + pluginTypes, options, err := getTypesAndOptions(externalConfig) + if err != nil { + return nil, err + } + if len(pluginTypes) == 0 { + return nil, fmt.Errorf("must specify one of %s, %s and %s", typeRemote, typeBinary, typeProtocBuiltin) + } + if len(pluginTypes) > 1 { + return nil, fmt.Errorf("only one of %s, %s and %s is allowed", typeRemote, typeBinary, typeProtocBuiltin) + } + pluginType := pluginTypes[0] + allowedOptions := allowedOptionsForType[pluginType] + for _, option := range options { + if !allowedOptions[option] { + return nil, fmt.Errorf("%s is not allowed for %s plugin", option, pluginType) + } + } + var strategy string + if externalConfig.Strategy != nil { + strategy = *externalConfig.Strategy + } + parsedStrategy, err := parseStrategy(strategy) + if err != nil { + return nil, err + } + opt, err := encoding.InterfaceSliceOrStringToCommaSepString(externalConfig.Opt) + if err != nil { + return nil, err + } + switch pluginType { + case typeRemote: + var revision int + if externalConfig.Revision != nil { + revision = *externalConfig.Revision + } + if revision < 0 || revision > math.MaxInt32 { + return nil, fmt.Errorf("revision %d is out of accepted range %d-%d", revision, 0, math.MaxInt32) + } + return newRemotePluginConfig( + *externalConfig.Remote, + revision, + externalConfig.Out, + opt, + externalConfig.IncludeImports, + externalConfig.IncludeWKT, + ) + case typeBinary: + path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Binary) + if err != nil { + return nil, err + } + return newBinaryPluginConfig( + strings.Join(path, ""), + path, + parsedStrategy, + externalConfig.Out, + opt, + externalConfig.IncludeImports, + externalConfig.IncludeWKT, + ) + case typeProtocBuiltin: + var protocPath string + if externalConfig.ProtocPath != nil { + protocPath = *externalConfig.ProtocPath + } + return newProtocBuiltinPluginConfig( + *externalConfig.ProtocBuiltin, + protocPath, + externalConfig.Out, + opt, + externalConfig.IncludeImports, + externalConfig.IncludeWKT, + parsedStrategy, + ) + default: + // this should not happen + return nil, fmt.Errorf("must specify one of %s, %s and %s", typeRemote, typeBinary, typeProtocBuiltin) + } +} + +// TODO: unify parameter order func newLocalPluginConfig( name string, strategy GenerateStrategy, From ad75d08b15f9a52b511d325273c8e2ef5db68dd2 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 28 Nov 2023 16:27:14 -0500 Subject: [PATCH 34/66] port modify js type (without sweeper) --- .../bufimage/bufimagemodify/bufimagemodify.go | 6 + .../bufimage/bufimagemodify/field_option.go | 122 ++++++++++++++++++ .../bufimage/bufimagemodify/new_modify.go | 1 + 3 files changed, 129 insertions(+) create mode 100644 private/bufpkg/bufimage/bufimagemodify/field_option.go diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go index 59636f1c2a..43a14fc840 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go @@ -308,11 +308,17 @@ func RubyPackage( ) } +// TODO: delete this one // isWellKnownType returns true if the given path is one of the well-known types. func isWellKnownType(ctx context.Context, imageFile bufimage.ImageFile) bool { return datawkt.Exists(imageFile.Path()) } +// TODO: rename this one +func isWellKnownTypeNoCtx(imageFile bufimage.ImageFile) bool { + return datawkt.Exists(imageFile.Path()) +} + // int32SliceIsEqual returns true if x and y contain the same elements. func int32SliceIsEqual(x []int32, y []int32) bool { if len(x) != len(y) { diff --git a/private/bufpkg/bufimage/bufimagemodify/field_option.go b/private/bufpkg/bufimage/bufimagemodify/field_option.go new file mode 100644 index 0000000000..dfa7853546 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/field_option.go @@ -0,0 +1,122 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "fmt" + + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/protocompile/walk" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/descriptorpb" +) + +// JSTypeSubPath is the SourceCodeInfo sub path for the jstype field option. +// https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L567 +var jsTypeSubPath = []int32{8, 6} + +func modifyJsType( + sweeper Sweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, +) error { + overrideRules := slicesext.Filter( + config.Overrides(), + func(override bufconfig.ManagedOverrideRule) bool { + return override.FieldOption() == bufconfig.FieldOptionJSType + }, + ) + // Unless specified, js type is not modified. + if len(overrideRules) == 0 { + return nil + } + for _, disable := range config.Disables() { + // If the entire file is disabled, skip. + if fileMatchConfig(imageFile, disable.Path(), disable.ModuleFullName()) && + disable.FieldName() == "" && + (disable.FieldOption() == bufconfig.FieldOptionUnspecified || disable.FieldOption() == bufconfig.FieldOptionJSType) { + return nil + } + } + // TODO: this check might not be needed + if isWellKnownTypeNoCtx(imageFile) { + return nil + } + return walk.DescriptorProtosWithPath( + imageFile.FileDescriptorProto(), + func( + fullName protoreflect.FullName, + path protoreflect.SourcePath, + message proto.Message, + ) error { + fieldDescriptor, ok := message.(*descriptorpb.FieldDescriptorProto) + if !ok { + return nil + } + for _, disable := range config.Disables() { + // If the entire file is disabled, skip. + if fileMatchConfig(imageFile, disable.Path(), disable.ModuleFullName()) && + (disable.FieldName() == "" || disable.FieldName() == string(fullName)) && + (disable.FieldOption() == bufconfig.FieldOptionUnspecified || disable.FieldOption() == bufconfig.FieldOptionJSType) { + return nil + } + } + var jsType *descriptorpb.FieldOptions_JSType + for _, override := range config.Overrides() { + if fileMatchConfig(imageFile, override.Path(), override.ModuleFullName()) && + (override.FieldName() == "" || override.FieldName() == string(fullName)) && + override.FieldOption() == bufconfig.FieldOptionJSType { + jsTypeValue, ok := override.Value().(descriptorpb.FieldOptions_JSType) + if !ok { + return fmt.Errorf("invalid js_type override value of type %T", override.Value()) + } + jsType = &jsTypeValue + } + } + if jsType == nil { + return nil + } + if fieldDescriptor.Type == nil || !isJsTypePermittedForType(*fieldDescriptor.Type) { + return nil + } + if options := fieldDescriptor.Options; options != nil { + if existingJSTYpe := options.Jstype; existingJSTYpe != nil && *existingJSTYpe == *jsType { + return nil + } + } + if fieldDescriptor.Options == nil { + fieldDescriptor.Options = &descriptorpb.FieldOptions{} + } + fieldDescriptor.Options.Jstype = jsType + if len(path) > 0 { + jsTypeOptionPath := append(path, jsTypeSubPath...) + sweeper.mark(imageFile.Path(), jsTypeOptionPath) + } + return nil + }, + ) +} + +func isJsTypePermittedForType(fieldType descriptorpb.FieldDescriptorProto_Type) bool { + // https://github.com/protocolbuffers/protobuf/blob/d4db41d395dcbb2c79b7fb1f109086fa04afd8aa/src/google/protobuf/descriptor.proto#L622 + return fieldType == descriptorpb.FieldDescriptorProto_TYPE_INT64 || + fieldType == descriptorpb.FieldDescriptorProto_TYPE_UINT64 || + fieldType == descriptorpb.FieldDescriptorProto_TYPE_SINT64 || + fieldType == descriptorpb.FieldDescriptorProto_TYPE_FIXED64 || + fieldType == descriptorpb.FieldDescriptorProto_TYPE_SFIXED64 +} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go index e333aaaa20..35821f7c7e 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -52,6 +52,7 @@ func Modify( modifyPhpMetadataNamespace, modifyPhpNamespace, modifyRubyPackage, + modifyJsType, } for _, modifyFunc := range modifyFuncs { if err := modifyFunc(sweeper, imageFile, config); err != nil { From 048aff3ff2979a604a5db3c7ef92ffd5edc427ff Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Tue, 28 Nov 2023 16:47:01 -0500 Subject: [PATCH 35/66] mark sweeper that supports sweeping field options --- .../bufimage/bufimagemodify/field_option.go | 4 +- .../bufimagemodify/field_options_trie.go | 113 ++++++++++++ .../bufimagemodify/location_path_dfa.go | 101 ++++++++++ .../bufimage/bufimagemodify/marksweeper.go | 174 ++++++++++++++++++ .../bufimage/bufimagemodify/new_modify.go | 28 +-- .../bufimage/bufimagemodify/new_modify2.go | 8 +- .../bufimagemodify/new_modify_test.go | 4 +- 7 files changed, 410 insertions(+), 22 deletions(-) create mode 100644 private/bufpkg/bufimage/bufimagemodify/field_options_trie.go create mode 100644 private/bufpkg/bufimage/bufimagemodify/location_path_dfa.go create mode 100644 private/bufpkg/bufimage/bufimagemodify/marksweeper.go diff --git a/private/bufpkg/bufimage/bufimagemodify/field_option.go b/private/bufpkg/bufimage/bufimagemodify/field_option.go index dfa7853546..4e981ce30b 100644 --- a/private/bufpkg/bufimage/bufimagemodify/field_option.go +++ b/private/bufpkg/bufimage/bufimagemodify/field_option.go @@ -31,7 +31,7 @@ import ( var jsTypeSubPath = []int32{8, 6} func modifyJsType( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -105,7 +105,7 @@ func modifyJsType( fieldDescriptor.Options.Jstype = jsType if len(path) > 0 { jsTypeOptionPath := append(path, jsTypeSubPath...) - sweeper.mark(imageFile.Path(), jsTypeOptionPath) + sweeper.Mark(imageFile, jsTypeOptionPath) } return nil }, diff --git a/private/bufpkg/bufimage/bufimagemodify/field_options_trie.go b/private/bufpkg/bufimage/bufimagemodify/field_options_trie.go new file mode 100644 index 0000000000..db533b7b46 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/field_options_trie.go @@ -0,0 +1,113 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "sort" + + "golang.org/x/exp/slices" +) + +// fieldOptionsTrie stores paths to FieldOptions (tag 8 of a FieldDescriptorProto). +type fieldOptionsTrie []*fieldOptionsTrieNode + +type fieldOptionsTrieNode struct { + value int32 + isPathEnd bool + children fieldOptionsTrie + // locationIndex is the data attached that we are interested in retrieving. + // This field is irrelevant to traversal or the trie structure. + locationIndex int + // registeredDescendantCount records how many times a descendant of this node + // is registered. This field is irrelevant to traversal or the trie structure. + registeredDescendantCount int +} + +// insert inserts a path into the trie. The caller should +// ensure that the path is for a FieldOptions. +func (p *fieldOptionsTrie) insert(path []int32, locationIndex int) { + trie := p + for index, element := range path { + isLastElement := index == len(path)-1 + nodes := *trie + pos, found := sort.Find(len(nodes), func(i int) int { + return int(element - nodes[i].value) + }) + if found { + if isLastElement { + nodes[pos].isPathEnd = true + nodes[pos].locationIndex = locationIndex + return + } + trie = &nodes[pos].children + continue + } + newNode := &fieldOptionsTrieNode{ + value: element, + children: fieldOptionsTrie{}, + } + if isLastElement { + newNode.isPathEnd = true + newNode.locationIndex = locationIndex + } + nodes = slices.Insert(nodes, pos, newNode) + *trie = nodes + trie = &nodes[pos].children + } +} + +// registerDescendant finds if there is an ancestor of the provided +// path and increments this ancestor's counter if it exists. +func (p *fieldOptionsTrie) registerDescendant(descendant []int32) { + trie := p + for i, element := range descendant { + nodes := *trie + pos, found := sort.Find(len(nodes), func(i int) int { + return int(element - nodes[i].value) + }) + if !found { + return + } + ancestor := nodes[pos] + descendantContinues := i != len(descendant)-1 + if ancestor.isPathEnd && descendantContinues { + ancestor.registeredDescendantCount += 1 + return + } + trie = &ancestor.children + } +} + +// indicesWithoutDescendant returns the location indices of +func (p *fieldOptionsTrie) indicesWithoutDescendant() []int { + locationIndices := []int{} + walkTrie(*p, func(node *fieldOptionsTrieNode) { + if node.isPathEnd && node.registeredDescendantCount == 0 { + locationIndices = append(locationIndices, node.locationIndex) + } + }) + return locationIndices +} + +func walkTrie(trie fieldOptionsTrie, enter func(node *fieldOptionsTrieNode)) { + for _, node := range trie { + walkTrieNode(node, enter) + } +} + +func walkTrieNode(node *fieldOptionsTrieNode, enter func(node *fieldOptionsTrieNode)) { + enter(node) + walkTrie(node.children, enter) +} diff --git a/private/bufpkg/bufimage/bufimagemodify/location_path_dfa.go b/private/bufpkg/bufimage/bufimagemodify/location_path_dfa.go new file mode 100644 index 0000000000..8a445b3204 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/location_path_dfa.go @@ -0,0 +1,101 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +const ( + pathTypeNotFieldOption pathType = iota + 1 + pathTypeFieldOptionsRoot + pathTypeFieldOption + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L75 + messageTypeTagInFile int32 = 4 + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L78 + extensionTagInFile int32 = 7 + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L97 + fieldTagInMessage int32 = 2 + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L100 + nestedTypeTagInMessage int32 = 3 + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L98 + extensionTagInMessage int32 = 6 + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L215 + optionsTagInField int32 = 8 +) + +// pathType is the type of a message pointed to by a location path. +type pathType int + +// locationPathDFAState takes an input and returns the next state and the path type +// that ends with the input, which is consistent with the returned next state. The next +// state is nil if the DFA has finished running. +type locationPathDFAState func(int32) (locationPathDFAState, pathType) + +// getPathType returns the type of the path. It only accepts one of: +// empty, messages, message, fields, field, field options and field option. +func getPathType(path []int32) pathType { + pathType := pathTypeNotFieldOption + currentState := start + for _, element := range path { + if currentState == nil { + break + } + currentState, pathType = currentState(element) + } + return pathType +} + +func start(input int32) (locationPathDFAState, pathType) { + switch input { + case messageTypeTagInFile: + return messages, pathTypeNotFieldOption + case extensionTagInFile: + return fields, pathTypeNotFieldOption + default: + return nil, pathTypeNotFieldOption + } +} + +func messages(input int32) (locationPathDFAState, pathType) { + // we are not checking index >= 0, the caller must ensure this + return message, pathTypeNotFieldOption +} + +func message(input int32) (locationPathDFAState, pathType) { + switch input { + case nestedTypeTagInMessage: + return messages, pathTypeNotFieldOption + case fieldTagInMessage, extensionTagInMessage: + return fields, pathTypeNotFieldOption + } + return nil, pathTypeNotFieldOption +} + +func fields(input int32) (locationPathDFAState, pathType) { + // we are not checking index >= 0, the caller must ensure this + return field, pathTypeNotFieldOption +} + +func field(input int32) (locationPathDFAState, pathType) { + switch input { + case optionsTagInField: + return fieldOptions, pathTypeFieldOptionsRoot + default: + return nil, pathTypeNotFieldOption + } +} + +func fieldOptions(input int32) (locationPathDFAState, pathType) { + // We are done here: after this point the path will be for a FieldOption. + // No need for the DFA to keep running. + return nil, pathTypeFieldOption +} diff --git a/private/bufpkg/bufimage/bufimagemodify/marksweeper.go b/private/bufpkg/bufimage/bufimagemodify/marksweeper.go new file mode 100644 index 0000000000..946251a965 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/marksweeper.go @@ -0,0 +1,174 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "fmt" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "google.golang.org/protobuf/types/descriptorpb" +) + +type markSweeper struct { + image bufimage.Image + // Filepath -> SourceCodeInfo_Location.Path keys. + sourceCodeInfoPaths map[string]map[string]struct{} +} + +func newMarkSweeper(image bufimage.Image) *markSweeper { + return &markSweeper{ + image: image, + sourceCodeInfoPaths: make(map[string]map[string]struct{}), + } +} + +func (s *markSweeper) Mark(imageFile bufimage.ImageFile, path []int32) { + paths, ok := s.sourceCodeInfoPaths[imageFile.Path()] + if !ok { + paths = make(map[string]struct{}) + s.sourceCodeInfoPaths[imageFile.Path()] = paths + } + paths[getPathKey(path)] = struct{}{} +} + +func (s *markSweeper) Sweep() error { + for _, imageFile := range s.image.Files() { + descriptor := imageFile.FileDescriptorProto() + if descriptor.SourceCodeInfo == nil { + continue + } + paths, ok := s.sourceCodeInfoPaths[imageFile.Path()] + if !ok { + continue + } + err := removeLocationsFromSourceCodeInfo(descriptor.SourceCodeInfo, paths) + if err != nil { + return err + } + } + return nil +} + +// removeLocationsFromSourceCodeInfo removes paths from the given sourceCodeInfo. +// Each path must be for either a file option or a field option. +func removeLocationsFromSourceCodeInfo( + sourceCodeInfo *descriptorpb.SourceCodeInfo, + pathsToRemove map[string]struct{}, +) error { + // TODO: in v1 there is no need to check for field options, maybe v1 and v2 + // don't need to share this function. + + // We can't just match on an exact path match because the target + // file option's parent path elements would remain (i.e [8]), + // or the target field option's parent path has no other child left. + // Instead, we perform an initial pass to validate that the paths + // are structured as expected, and collect all of the indices that + // we need to delete. + indices := make(map[int]struct{}, len(pathsToRemove)*2) + // each path in this trie is for a FieldOptions message (not for a singular option) + var fieldOptionsPaths fieldOptionsTrie + for i, location := range sourceCodeInfo.Location { + path := location.Path + pathType := getPathType(path) + if pathType == pathTypeFieldOptionsRoot { + fieldOptionsPaths.insert(path, i) + } + if _, ok := pathsToRemove[getPathKey(path)]; !ok { + if pathType == pathTypeFieldOption { + // This field option path will not be removed, register it to its + // parent FieldOptions. + fieldOptionsPaths.registerDescendant(path) + } + continue + } + if i == 0 { + return fmt.Errorf("path %v must have a preceding parent path", location.Path) + } + if isPathForFileOption(location.Path) { + if !Int32SliceIsEqual(sourceCodeInfo.Location[i-1].Path, fileOptionPath) { + return fmt.Errorf("file option path %v must have a preceding parent path equal to %v", location.Path, fileOptionPath) + } + // Add the target path and its parent. + indices[i-1] = struct{}{} + indices[i] = struct{}{} + continue + } + if pathType == pathTypeFieldOption { + // Note that there is a difference between the generated file option paths and field options paths. + // For example, for: + // ... + // option java_package = "com.example"; + // option go_package = "github.com/hello/world"; + // ... + // the generated paths are + // [8], [8,1], [8], [8,11] + // where each file option declared has a parent. + // However, for different field options of the same field, they share the same parent. For + // ... + // optional string id2 = 2 [jstype = JS_STRING, ctype = CORD]; + // required fixed64 first = 1 [ + // (foo.bar.baz.aaa).foo = "hello", + // (foo.bar.baz.bbb).a.foo = "hey", + // (foo.bar.baz.ccc) = 123, // ccc is a repeated option + // jstype = JS_STRING + // ]; + // ... + // the generated paths are + // [4,0,2,0,8],[4,0,2,0,8,50000,1],[4,0,2,0,8,50002,1,1],[4,0,2,0,8,50003,0],[4,0,2,0,8,6] + // where two field options share the same parent. + // Therefore, do not remove the parent path yet. + indices[i] = struct{}{} + continue + } + return fmt.Errorf("path %v is neither a file option path nor a field option path", location.Path) + } + for _, emptyFieldOptions := range fieldOptionsPaths.indicesWithoutDescendant() { + indices[emptyFieldOptions] = struct{}{} + } + // Now that we know exactly which indices to exclude, we can + // filter the SourceCodeInfo_Locations as needed. + locations := make( + []*descriptorpb.SourceCodeInfo_Location, + 0, + len(sourceCodeInfo.Location)-len(indices), + ) + for i, location := range sourceCodeInfo.Location { + if _, ok := indices[i]; ok { + continue + } + locations = append(locations, location) + } + sourceCodeInfo.Location = locations + return nil +} + +func isPathForFileOption(path []int32) bool { + // a file option's path is {8, x} + fileOptionPathLen := 2 + return len(path) == fileOptionPathLen && path[0] == fileOptionPath[0] +} + +// Int32SliceIsEqual returns true if x and y contain the same elements. +func Int32SliceIsEqual(x []int32, y []int32) bool { + if len(x) != len(y) { + return false + } + for i, elem := range x { + if elem != y[i] { + return false + } + } + return true +} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go index 35821f7c7e..8bd5ddd871 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -34,12 +34,12 @@ func Modify( return nil } // TODO: in v2 modify field options as well - sweeper := NewFileOptionSweeper() + sweeper := newMarkSweeper(image) for _, imageFile := range image.Files() { if isWellKnownType(ctx, imageFile) { continue } - modifyFuncs := []func(Sweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error{ + modifyFuncs := []func(*markSweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error{ modifyCcEnableArenas, modifyCsharpNamespace, modifyGoPackage, @@ -64,7 +64,7 @@ func Modify( } func modifyJavaOuterClass( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -92,7 +92,7 @@ func modifyJavaOuterClass( } func modifyJavaPackage( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -118,7 +118,7 @@ func modifyJavaPackage( } func modifyGoPackage( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -149,7 +149,7 @@ func modifyGoPackage( } func modifyObjcClassPrefix( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -177,7 +177,7 @@ func modifyObjcClassPrefix( } func modifyCsharpNamespace( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -205,7 +205,7 @@ func modifyCsharpNamespace( } func modifyPhpNamespace( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -233,7 +233,7 @@ func modifyPhpNamespace( } func modifyPhpMetadataNamespace( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -261,7 +261,7 @@ func modifyPhpMetadataNamespace( } func modifyRubyPackage( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -289,7 +289,7 @@ func modifyRubyPackage( } func modifyCcEnableArenas( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -310,7 +310,7 @@ func modifyCcEnableArenas( } func modifyJavaMultipleFiles( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -331,7 +331,7 @@ func modifyJavaMultipleFiles( } func modifyJavaStringCheckUtf8( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -352,7 +352,7 @@ func modifyJavaStringCheckUtf8( } func modifyOptmizeFor( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go index 9d74e75d4c..632acfab19 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go @@ -24,7 +24,7 @@ import ( ) func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, fileOption bufconfig.FileOption, @@ -62,7 +62,7 @@ func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( descriptor.Options = &descriptorpb.FileOptions{} } setOptionFunc(descriptor.Options, value) - sweeper.mark(imageFile.Path(), sourceLocationPath) + sweeper.Mark(imageFile, sourceLocationPath) return nil } @@ -91,7 +91,7 @@ func overrideFromConfig[T bool | descriptorpb.FileOptions_OptimizeMode]( } func modifyStringOption( - sweeper Sweeper, + sweeper *markSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, valueOption bufconfig.FileOption, @@ -138,7 +138,7 @@ func modifyStringOption( descriptor.Options = &descriptorpb.FileOptions{} } setOptionFunc(descriptor.Options, value) - sweeper.mark(imageFile.Path(), sourceLocationPath) + sweeper.Mark(imageFile, sourceLocationPath) return nil } diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go index 85e9931704..8acf6fd864 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go @@ -183,7 +183,7 @@ func TestModifyImageFile( description string dirPathToModuleFullName map[string]string config bufconfig.GenerateManagedConfig - modifyFunc func(Sweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error + modifyFunc func(*markSweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error filePathToExpectedOptions map[string]*descriptorpb.FileOptions filePathToExpectedMarkedLocationPaths map[string][][]int32 }{ @@ -556,7 +556,7 @@ func TestModifyImageFile( t.Run(testcase.description, func(t *testing.T) { t.Parallel() image := testGetImageFromDirs(t, testcase.dirPathToModuleFullName, includeSourceInfo) - sweeper := newFileOptionSweeper() + sweeper := newMarkSweeper(image) // TODO: check include source code info for filePath, expectedOptions := range testcase.filePathToExpectedOptions { imageFile := image.GetFile(filePath) From 44498746e420be65d5a19d8a2a0a33ae4915e27b Mon Sep 17 00:00:00 2001 From: bufdev Date: Tue, 28 Nov 2023 17:58:53 -0500 Subject: [PATCH 36/66] commit --- private/buf/bufctl/controller.go | 85 +++++++---- private/buf/buffetch/internal/internal.go | 1 + private/buf/bufgen/bufgen.go | 7 +- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 5 +- private/bufpkg/bufconfig/generate_config.go | 2 - ...nerate_input_config.go => input_config.go} | 132 +++++++++--------- 6 files changed, 132 insertions(+), 100 deletions(-) rename private/bufpkg/bufconfig/{generate_input_config.go => input_config.go} (89%) diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 08d6a167c7..be0a643120 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -68,6 +68,11 @@ type Controller interface { input string, options ...FunctionOption, ) (bufimage.Image, error) + GetImageForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + options ...FunctionOption, + ) (bufimage.Image, error) GetImageWithConfigs( ctx context.Context, input string, @@ -314,35 +319,23 @@ func (c *controller) GetImage( if err != nil { return nil, err } - switch t := ref.(type) { - case buffetch.ProtoFileRef: - return nil, errors.New("TODO") - case buffetch.SourceRef: - workspace, err := c.getWorkspaceForSourceRef(ctx, t, functionOptions) - if err != nil { - return nil, err - } - return c.buildImage( - ctx, - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), - functionOptions, - ) - case buffetch.ModuleRef: - workspace, err := c.getWorkspaceForModuleRef(ctx, t, functionOptions) - if err != nil { - return nil, err - } - return c.buildImage( - ctx, - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), - functionOptions, - ) - case buffetch.MessageRef: - return c.getImageForMessageRef(ctx, t, functionOptions) - default: - // This is a system error. - return nil, syserror.Newf("invalid Ref: %T", ref) + return c.getImage(ctx, ref, functionOptions) +} + +func (c *controller) GetImageForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + options ...FunctionOption, +) (bufimage.Image, error) { + functionOptions := newFunctionOptions() + for _, option := range options { + option(functionOptions) } + ref, err := c.buffetchRefParser.GetRefForInputConfig(ctx, inputConfig) + if err != nil { + return nil, err + } + return c.getImage(ctx, ref, functionOptions) } func (c *controller) GetImageWithConfigs( @@ -587,6 +580,42 @@ func (c *controller) PutMessage( return multierr.Append(err, writeCloser.Close()) } +func (c *controller) getImage( + ctx context.Context, + ref buffetch.Ref, + functionOptions *functionOptions, +) (bufimage.Image, error) { + switch t := ref.(type) { + case buffetch.ProtoFileRef: + return nil, errors.New("TODO") + case buffetch.SourceRef: + workspace, err := c.getWorkspaceForSourceRef(ctx, t, functionOptions) + if err != nil { + return nil, err + } + return c.buildImage( + ctx, + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), + functionOptions, + ) + case buffetch.ModuleRef: + workspace, err := c.getWorkspaceForModuleRef(ctx, t, functionOptions) + if err != nil { + return nil, err + } + return c.buildImage( + ctx, + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), + functionOptions, + ) + case buffetch.MessageRef: + return c.getImageForMessageRef(ctx, t, functionOptions) + default: + // This is a system error. + return nil, syserror.Newf("invalid Ref: %T", ref) + } +} + func (c *controller) getWorkspaceForSourceRef( ctx context.Context, sourceRef buffetch.SourceRef, diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index b4ffbe6948..75d9134a10 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -361,6 +361,7 @@ type RefParser interface { // // The options should be used to validate that you are getting one of the correct formats. GetParsedRef(ctx context.Context, value string, options ...GetParsedRefOption) (ParsedRef, error) + GetParsedRefForInputConfig(ctx context.Context, bufconfig.InputConfig, options ...GetParsedRefOption) (ParsedRef, error) } // NewRefParser returns a new RefParser. diff --git a/private/buf/bufgen/bufgen.go b/private/buf/bufgen/bufgen.go index a98df14cde..d1be9663c5 100644 --- a/private/buf/bufgen/bufgen.go +++ b/private/buf/bufgen/bufgen.go @@ -23,8 +23,8 @@ import ( "fmt" "strconv" - "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" @@ -108,6 +108,7 @@ type Generator interface { ctx context.Context, container app.EnvStdioContainer, config bufconfig.GenerateConfig, + images []bufimage.Image, options ...GenerateOption, ) error } @@ -115,7 +116,7 @@ type Generator interface { // NewGenerator returns a new Generator. func NewGenerator( logger *zap.Logger, - controller bufctl.Controller, + //controller bufctl.Controller, storageosProvider storageos.Provider, runner command.Runner, wasmPluginExecutor bufwasm.PluginExecutor, @@ -123,7 +124,7 @@ func NewGenerator( ) Generator { return newGenerator( logger, - controller, + //controller, storageosProvider, runner, wasmPluginExecutor, diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index bdd5fe0548..09bc2f90c2 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -34,8 +34,9 @@ var ( // For v2, generation configuration has been merged into BufYAMLFiles. type BufGenYAMLFile interface { File - // Will always have empty GenerateInputConfigs. - GenerateConfig + + GenerateConfig() GenerateConfig + InputConfigs() []InputConfig isBufGenYAMLFile() } diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index e53c0938b4..0086032cb2 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -32,8 +32,6 @@ type GenerateConfig interface { // filters from input configurations, which exist in v2. // This will always be nil in v2 GenerateTypeConfig() GenerateTypeConfig - // GenerateInputConfigs returns the input config. - GenerateInputConfigs() []GenerateInputConfig isGenerateConfig() } diff --git a/private/bufpkg/bufconfig/generate_input_config.go b/private/bufpkg/bufconfig/input_config.go similarity index 89% rename from private/bufpkg/bufconfig/generate_input_config.go rename to private/bufpkg/bufconfig/input_config.go index 63a69573d0..96552da255 100644 --- a/private/bufpkg/bufconfig/generate_input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -1,4 +1,4 @@ -// GenerateInputConfig is an input configuration. +// InputConfig is an input configuration. // Copyright 2020-2023 Buf Technologies, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -113,8 +113,8 @@ var inputConfigTypeToString = map[InputConfigType]string{ InputConfigTypeTextImage: formatTextImage, } -// GenerateInputConfig is an input configuration for code generation. -type GenerateInputConfig interface { +// InputConfig is an input configuration for code generation. +type InputConfig interface { // Type returns the input type. Type() InputConfigType // Location returns the location for the input. @@ -150,10 +150,12 @@ type GenerateInputConfig interface { // returns a non-empty list of types. IncludeTypes() []string - isGenerateInputConfig() + isInputConfig() } -type generateInputConfig struct { +// *** PRIVATE *** + +type inputConfig struct { inputType InputConfigType location string compression string @@ -170,66 +172,8 @@ type generateInputConfig struct { includePaths []string } -func (g *generateInputConfig) Type() InputConfigType { - return g.inputType -} - -func (g *generateInputConfig) Location() string { - return g.location -} - -func (g *generateInputConfig) Compression() string { - return g.compression -} - -func (g *generateInputConfig) StripComponents() *uint32 { - return g.stripComponents -} - -func (g *generateInputConfig) Subdir() string { - return g.subdir -} - -func (g *generateInputConfig) Branch() string { - return g.branch -} - -func (g *generateInputConfig) Tag() string { - return g.tag -} - -func (g *generateInputConfig) Ref() string { - return g.ref -} - -func (g *generateInputConfig) Depth() *uint32 { - return g.depth -} - -func (g *generateInputConfig) RecurseSubmodules() bool { - return g.recurseSubmodules -} - -func (g *generateInputConfig) IncludePackageFiles() bool { - return g.includePackageFiles -} - -func (g *generateInputConfig) ExcludePaths() []string { - return g.excludePaths -} - -func (g *generateInputConfig) IncludePaths() []string { - return g.includePaths -} - -func (g *generateInputConfig) IncludeTypes() []string { - return g.includeTypes -} - -func (g *generateInputConfig) isGenerateInputConfig() {} - -func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV2) (GenerateInputConfig, error) { - inputConfig := &generateInputConfig{} +func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV2) (InputConfig, error) { + inputConfig := &inputConfig{} var inputTypes []InputConfigType var options []string if externalConfig.Module != nil { @@ -324,3 +268,61 @@ func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV } return inputConfig, nil } + +func (g *inputConfig) Type() InputConfigType { + return g.inputType +} + +func (g *inputConfig) Location() string { + return g.location +} + +func (g *inputConfig) Compression() string { + return g.compression +} + +func (g *inputConfig) StripComponents() *uint32 { + return g.stripComponents +} + +func (g *inputConfig) Subdir() string { + return g.subdir +} + +func (g *inputConfig) Branch() string { + return g.branch +} + +func (g *inputConfig) Tag() string { + return g.tag +} + +func (g *inputConfig) Ref() string { + return g.ref +} + +func (g *inputConfig) Depth() *uint32 { + return g.depth +} + +func (g *inputConfig) RecurseSubmodules() bool { + return g.recurseSubmodules +} + +func (g *inputConfig) IncludePackageFiles() bool { + return g.includePackageFiles +} + +func (g *inputConfig) ExcludePaths() []string { + return g.excludePaths +} + +func (g *inputConfig) IncludePaths() []string { + return g.includePaths +} + +func (g *inputConfig) IncludeTypes() []string { + return g.includeTypes +} + +func (g *inputConfig) isInputConfig() {} From 265ae8e802ce716b1f9012d3f2a792ea7694709f Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 29 Nov 2023 17:21:11 -0500 Subject: [PATCH 37/66] generate with inputs; with some tests skipped due to proto file ref not implemented and breaking change in --path and --exclude-path --- make/buf/all.mk | 2 + private/buf/bufctl/controller.go | 1 + private/buf/buffetch/buffetch.go | 9 +- private/buf/buffetch/internal/internal.go | 8 +- private/buf/buffetch/internal/ref_parser.go | 283 ++++++++++++++---- .../buf/buffetch/internal/ref_parser_test.go | 2 +- private/buf/buffetch/ref_parser.go | 50 ++++ private/buf/buffetch/ref_parser_test.go | 1 + private/buf/bufgen/bufgen.go | 41 +-- private/buf/bufgen/generator.go | 152 +--------- .../buf/cmd/buf/command/generate/generate.go | 108 ++++++- .../cmd/buf/command/generate/generate_test.go | 22 +- .../command/generate/generate_unix_test.go | 8 + private/bufpkg/bufconfig/buf_gen_yaml_file.go | 48 ++- private/bufpkg/bufconfig/generate_config.go | 13 - private/bufpkg/bufconfig/input_config.go | 58 ++-- 16 files changed, 491 insertions(+), 315 deletions(-) diff --git a/make/buf/all.mk b/make/buf/all.mk index 83cb17f65e..796e21c425 100644 --- a/make/buf/all.mk +++ b/make/buf/all.mk @@ -61,6 +61,7 @@ testbufnew: installbuf ./private/buf/bufcurl/... \ ./private/buf/buffetch/... \ ./private/buf/bufformat/... \ + ./private/buf/bufgen/... \ ./private/buf/bufprint/... \ ./private/buf/bufworkspace/... \ ./private/buf/cmd/buf/command/alpha/package/... \ @@ -74,6 +75,7 @@ testbufnew: installbuf ./private/buf/cmd/buf/command/build/... \ ./private/buf/cmd/buf/command/breaking/... \ ./private/buf/cmd/buf/command/convert/... \ + ./private/buf/cmd/buf/command/generate/... \ ./private/buf/cmd/buf/command/lint/... \ ./private/buf/cmd/buf/command/lsfiles/... \ ./private/buf/cmd/buf/command/mod/... \ diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index be0a643120..1deecb0db3 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -63,6 +63,7 @@ type Controller interface { dirPath string, options ...FunctionOption, ) (bufworkspace.UpdateableWorkspace, error) + // TODO: rename to GetImageForInputString, but in a separate PR to minimize merge conflicts GetImage( ctx context.Context, input string, diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index ebdfcfcd41..9201191828 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -20,6 +20,7 @@ import ( "net/http" "github.com/bufbuild/buf/private/buf/buffetch/internal" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/git" @@ -41,15 +42,8 @@ const ( useProtoNamesKey = "use_proto_names" useEnumNumbersKey = "use_enum_numbers" - - // TODO: - // SubDirKey Key = "subdir" ) -// TODO: -// type Key struct -// GetModuleRef(context.Context, string, map[Key]string) - var ( // MessageFormatsString is the string representation of all message formats. // @@ -178,6 +172,7 @@ type RefParser interface { // GetRef gets the reference for the message file, source bucket, or module. GetRef(ctx context.Context, value string) (Ref, error) + GetRefForInputConfig(ctx context.Context, inputConfig bufconfig.InputConfig) (Ref, error) } // NewRefParser returns a new RefParser. diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index 75d9134a10..71c5946f3a 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -19,6 +19,7 @@ import ( "io" "net/http" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/git" @@ -361,7 +362,12 @@ type RefParser interface { // // The options should be used to validate that you are getting one of the correct formats. GetParsedRef(ctx context.Context, value string, options ...GetParsedRefOption) (ParsedRef, error) - GetParsedRefForInputConfig(ctx context.Context, bufconfig.InputConfig, options ...GetParsedRefOption) (ParsedRef, error) + // GetParsedRefForInputConfig gets the ParsedRef for the input config. + // + // The returned ParsedRef will be either a ParsedSingleRef, ParsedArchiveRef, ParsedDirRef, ParsedGitRef, or ParsedModuleRef. + // + // The options should be used to validate that you are getting one of the correct formats. + GetParsedRefForInputConfig(ctx context.Context, inputConfig bufconfig.InputConfig, options ...GetParsedRefOption) (ParsedRef, error) } // NewRefParser returns a new RefParser. diff --git a/private/buf/buffetch/internal/ref_parser.go b/private/buf/buffetch/internal/ref_parser.go index e88c61a15a..704c9421d7 100644 --- a/private/buf/buffetch/internal/ref_parser.go +++ b/private/buf/buffetch/internal/ref_parser.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/git" "github.com/bufbuild/buf/private/pkg/normalpath" @@ -62,18 +63,54 @@ func (a *refParser) GetParsedRef( for _, option := range options { option(getParsedRefOptions) } - return a.getParsedRef(ctx, value, getParsedRefOptions.allowedFormats) + return a.getParsedRefForInputString(ctx, value, getParsedRefOptions.allowedFormats) } -func (a *refParser) getParsedRef( +func (a *refParser) GetParsedRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + options ...GetParsedRefOption, +) (ParsedRef, error) { + getParsedRefOptions := newGetParsedRefOptions() + for _, option := range options { + option(getParsedRefOptions) + } + return a.getParsedRefForInputConfig(ctx, inputConfig, getParsedRefOptions.allowedFormats) +} + +func (a *refParser) getParsedRefForInputString( ctx context.Context, value string, allowedFormats map[string]struct{}, ) (ParsedRef, error) { - rawRef, err := a.getRawRef(value) + // path is never empty after returning from this function + path, options, err := getRawPathAndOptionsForInputString(value) if err != nil { return nil, err } + rawRef, err := a.getRawRefFromInputString(path, value, options) + if err != nil { + return nil, err + } + return a.parseRawRef(rawRef, allowedFormats) +} + +func (a *refParser) getParsedRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + allowedFormats map[string]struct{}, +) (ParsedRef, error) { + rawRef, err := a.getRawRefFromInputConfig(inputConfig) + if err != nil { + return nil, err + } + return a.parseRawRef(rawRef, allowedFormats) +} + +func (a *refParser) parseRawRef( + rawRef *RawRef, + allowedFormats map[string]struct{}, +) (ParsedRef, error) { singleFormatInfo, singleOK := a.singleFormatToInfo[rawRef.Format] archiveFormatInfo, archiveOK := a.archiveFormatToInfo[rawRef.Format] _, dirOK := a.dirFormatToInfo[rawRef.Format] @@ -120,13 +157,12 @@ func (a *refParser) getParsedRef( return nil, NewFormatUnknownError(rawRef.Format) } -// validated per rules on rawRef -func (a *refParser) getRawRef(value string) (*RawRef, error) { - // path is never empty after returning from this function - path, options, err := getRawPathAndOptions(value) - if err != nil { - return nil, err - } +func (a *refParser) getRawRefFromInputString( + path string, + // TODO: need a better name + displayName string, + options map[string]string, +) (*RawRef, error) { rawRef := &RawRef{ Path: path, UnrecognizedOptions: make(map[string]string), @@ -144,37 +180,23 @@ func (a *refParser) getRawRef(value string) (*RawRef, error) { } rawRef.Format = value case "compression": - switch value { - case "none": - rawRef.CompressionType = CompressionTypeNone - case "gzip": - rawRef.CompressionType = CompressionTypeGzip - case "zstd": - rawRef.CompressionType = CompressionTypeZstd - default: - return nil, NewCompressionUnknownError(value) + compressionType, err := parseCompressionType(value) + if err != nil { + return nil, err } + rawRef.CompressionType = compressionType case "branch": - if rawRef.GitBranch != "" || rawRef.GitTag != "" { - return nil, NewCannotSpecifyGitBranchAndTagError() - } rawRef.GitBranch = value case "tag": - if rawRef.GitBranch != "" || rawRef.GitTag != "" { - return nil, NewCannotSpecifyGitBranchAndTagError() - } rawRef.GitTag = value case "ref": rawRef.GitRef = value case "depth": - depth, err := strconv.ParseUint(value, 10, 32) + depth, err := parseGitDepth(value) if err != nil { - return nil, NewDepthParseError(value) - } - if depth == 0 { - return nil, NewDepthZeroError() + return nil, err } - rawRef.GitDepth = uint32(depth) + rawRef.GitDepth = depth case "recurse_submodules": // TODO: need to refactor to make sure this is not set for any non-git input // ie right now recurse_submodules=false will not error @@ -194,17 +216,16 @@ func (a *refParser) getRawRef(value string) (*RawRef, error) { } rawRef.ArchiveStripComponents = uint32(stripComponents) case "subdir": - subDirPath, err := normalpath.NormalizeAndValidate(value) + subDirPath, err := parseSubDirPath(value) if err != nil { return nil, err } - if subDirPath != "." { - rawRef.SubDirPath = subDirPath - } + rawRef.SubDirPath = subDirPath case "include_package_files": switch value { case "true": rawRef.IncludePackageFiles = true + // TODO: perhaps empty values should not be accepted case "false", "": rawRef.IncludePackageFiles = false default: @@ -214,57 +235,176 @@ func (a *refParser) getRawRef(value string) (*RawRef, error) { rawRef.UnrecognizedOptions[key] = value } } + // This cannot be set ahead of time, it can only happen after all options are read. + if rawRef.Format == "git" && rawRef.GitDepth == 0 { + // Default to 1 + rawRef.GitDepth = 1 + if rawRef.GitRef != "" { + // Default to 50 when using ref + rawRef.GitDepth = 50 + } + } + if err := a.validateRawRef(displayName, rawRef); err != nil { + return nil, err + } + return rawRef, nil +} - if rawRef.Format == "" { - return nil, NewFormatCannotBeDeterminedError(value) +func (a *refParser) getRawRefFromInputConfig( + inputConfig bufconfig.InputConfig, +) (*RawRef, error) { + rawRef := &RawRef{ + Path: inputConfig.Location(), + UnrecognizedOptions: make(map[string]string), + } + if a.rawRefProcessor != nil { + if err := a.rawRefProcessor(rawRef); err != nil { + return nil, err + } + } + switch inputConfig.Type() { + case bufconfig.InputConfigTypeModule: + rawRef.Format = "mod" + case bufconfig.InputConfigTypeDirectory: + rawRef.Format = "dir" + case bufconfig.InputConfigTypeGitRepo: + rawRef.Format = "git" + case bufconfig.InputConfigTypeProtoFile: + rawRef.Format = "protofile" + case bufconfig.InputConfigTypeTarball: + rawRef.Format = "tar" + case bufconfig.InputConfigTypeZipArchive: + rawRef.Format = "zip" + case bufconfig.InputConfigTypeBinaryImage: + rawRef.Format = "binpb" + case bufconfig.InputConfigTypeJSONImage: + rawRef.Format = "jsonpb" + case bufconfig.InputConfigTypeTextImage: + rawRef.Format = "txtpb" + } + // This cannot be set ahead of time, it can only happen after all options are read. + if rawRef.GitDepth == 0 { + // Default to 1 + rawRef.GitDepth = 1 + if rawRef.GitRef != "" { + // Default to 50 when using ref + rawRef.GitDepth = 50 + } + } + var err error + rawRef.CompressionType, err = parseCompressionType(inputConfig.Compression()) + if err != nil { + return nil, err } + rawRef.GitBranch = inputConfig.Branch() + rawRef.GitTag = inputConfig.Tag() + if err := a.validateRawRef(inputConfig.Location(), rawRef); err != nil { + return nil, err + } + rawRef.GitRef = inputConfig.Ref() + // TODO: might change rawRef.Depth into a pointer or use some other way to handle the case where 0 is specified + if inputConfig.Depth() != nil { + if *inputConfig.Depth() == 0 { + return nil, NewDepthZeroError() + } + } + rawRef.GitRecurseSubmodules = inputConfig.RecurseSubmodules() + rawRef.IncludePackageFiles = inputConfig.IncludePackageFiles() + rawRef.SubDirPath, err = parseSubDirPath(inputConfig.Subdir()) + if err != nil { + return nil, err + } + if err := a.validateRawRef(inputConfig.Location(), rawRef); err != nil { + return nil, err + } + return rawRef, nil +} +func (a *refParser) validateRawRef( + displayName string, + rawRef *RawRef, +) error { + // probably move everything below this point to a new function, perhaps called validateRawRef + if rawRef.Format == "" { + return NewFormatCannotBeDeterminedError(displayName) + } _, gitOK := a.gitFormatToInfo[rawRef.Format] archiveFormatInfo, archiveOK := a.archiveFormatToInfo[rawRef.Format] _, singleOK := a.singleFormatToInfo[rawRef.Format] if gitOK { - if rawRef.GitRef != "" && rawRef.GitTag != "" { - return nil, NewCannotSpecifyTagWithRefError() + if rawRef.GitBranch != "" && rawRef.GitTag != "" { + return NewCannotSpecifyGitBranchAndTagError() } - if rawRef.GitDepth == 0 { - // Default to 1 - rawRef.GitDepth = 1 - if rawRef.GitRef != "" { - // Default to 50 when using ref - rawRef.GitDepth = 50 - } + if rawRef.GitRef != "" && rawRef.GitTag != "" { + return NewCannotSpecifyTagWithRefError() } } else { if rawRef.GitBranch != "" || rawRef.GitTag != "" || rawRef.GitRef != "" || rawRef.GitRecurseSubmodules || rawRef.GitDepth > 0 { - return nil, NewOptionsInvalidForFormatError(rawRef.Format, value) + return NewOptionsInvalidForFormatError(rawRef.Format, displayName) } } // not an archive format if !archiveOK { if rawRef.ArchiveStripComponents > 0 { - return nil, NewOptionsInvalidForFormatError(rawRef.Format, value) + return NewOptionsInvalidForFormatError(rawRef.Format, displayName) } } else { if archiveFormatInfo.archiveType == ArchiveTypeZip && rawRef.CompressionType != 0 { - return nil, NewCannotSpecifyCompressionForZipError() + return NewCannotSpecifyCompressionForZipError() } } if !singleOK && !archiveOK { if rawRef.CompressionType != 0 { - return nil, NewOptionsInvalidForFormatError(rawRef.Format, value) + return NewOptionsInvalidForFormatError(rawRef.Format, displayName) } } if !archiveOK && !gitOK { if rawRef.SubDirPath != "" { - return nil, NewOptionsInvalidForFormatError(rawRef.Format, value) + return NewOptionsInvalidForFormatError(rawRef.Format, displayName) } } - return rawRef, nil + return nil +} + +// empty value is an error +func parseCompressionType(value string) (CompressionType, error) { + switch value { + case "none": + return CompressionTypeNone, nil + case "gzip": + return CompressionTypeGzip, nil + case "zstd": + return CompressionTypeZstd, nil + default: + return 0, NewCompressionUnknownError(value) + } } -// getRawPathAndOptions returns the raw path and options from the value provided, +func parseGitDepth(value string) (uint32, error) { + depth, err := strconv.ParseUint(value, 10, 32) + if err != nil { + return 0, NewDepthParseError(value) + } + if depth == 0 { + return 0, NewDepthZeroError() + } + return uint32(depth), nil +} + +func parseSubDirPath(value string) (string, error) { + subDirPath, err := normalpath.NormalizeAndValidate(value) + if err != nil { + return "", err + } + if subDirPath == "." { + return "", nil + } + return subDirPath, nil +} + +// getRawPathAndOptionsForInputString returns the raw path and options from the value provided, // the rawPath will be non-empty when returning without error here. -func getRawPathAndOptions(value string) (string, map[string]string, error) { +func getRawPathAndOptionsForInputString(value string) (string, map[string]string, error) { value = strings.TrimSpace(value) if value == "" { return "", nil, newValueEmptyError() @@ -304,6 +444,41 @@ func getRawPathAndOptions(value string) (string, map[string]string, error) { } } +// // TODO: this is a very temporary (and not clean) solution. +// // To be deleted shortly +// func getRawPathAndOptionsForInputConfig(inputConfig bufconfig.InputConfig) (string, map[string]string, error) { +// var refString = inputConfig.Location() +// refOptionKeyToValue := map[string]string{} +// if inputConfig.Compression() != "" { +// refOptionKeyToValue["compression"] = inputConfig.Compression() +// } +// if inputConfig.StripComponents() != nil { +// refOptionKeyToValue["strip_components"] = strconv.FormatUint(uint64(*inputConfig.StripComponents()), 10) +// } +// if inputConfig.Subdir() != "" { +// refOptionKeyToValue["subdir"] = inputConfig.Subdir() +// } +// if inputConfig.Branch() != "" { +// refOptionKeyToValue["branch"] = inputConfig.Branch() +// } +// if inputConfig.Tag() != "" { +// refOptionKeyToValue["tag"] = inputConfig.Tag() +// } +// if inputConfig.Ref() != "" { +// refOptionKeyToValue["ref"] = inputConfig.Ref() +// } +// if inputConfig.Depth() != nil { +// refOptionKeyToValue["depth"] = strconv.FormatUint(uint64(*inputConfig.Depth()), 10) +// } +// if inputConfig.RecurseSubmodules() { +// refOptionKeyToValue["recurse_submodules"] = "true" +// } +// if inputConfig.IncludePackageFiles() { +// refOptionKeyToValue["include_package_files"] = "true" +// } +// return refString, refOptionKeyToValue, nil +// } + func getSingleRef( rawRef *RawRef, defaultCompressionType CompressionType, diff --git a/private/buf/buffetch/internal/ref_parser_test.go b/private/buf/buffetch/internal/ref_parser_test.go index 83fbd6a721..669ceda078 100644 --- a/private/buf/buffetch/internal/ref_parser_test.go +++ b/private/buf/buffetch/internal/ref_parser_test.go @@ -81,7 +81,7 @@ func testGetRawPathAndOptionsError( ) { t.Run(value, func(t *testing.T) { t.Parallel() - _, _, err := getRawPathAndOptions(value) + _, _, err := getRawPathAndOptionsForInputString(value) assert.EqualError(t, err, expectedErr.Error()) }) } diff --git a/private/buf/buffetch/ref_parser.go b/private/buf/buffetch/ref_parser.go index 8babcfbf84..a0945de902 100644 --- a/private/buf/buffetch/ref_parser.go +++ b/private/buf/buffetch/ref_parser.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/bufbuild/buf/private/buf/buffetch/internal" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/syserror" @@ -204,6 +205,7 @@ func newSourceOrModuleRefParser(logger *zap.Logger) *refParser { } } +// TODO: rename to GetRefForString func (a *refParser) GetRef( ctx context.Context, value string, @@ -234,6 +236,36 @@ func (a *refParser) GetRef( } } +func (a *refParser) GetRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, +) (_ Ref, retErr error) { + parsedRef, err := a.getParsedRefForInputConfig(ctx, inputConfig, allFormats) + if err != nil { + return nil, err + } + switch t := parsedRef.(type) { + case internal.ParsedSingleRef: + messageEncoding, err := parseMessageEncoding(t.Format()) + if err != nil { + return nil, err + } + return newMessageRef(t, messageEncoding) + case internal.ParsedArchiveRef: + return newSourceRef(t), nil + case internal.ParsedDirRef: + return newSourceRef(t), nil + case internal.ParsedGitRef: + return newSourceRef(t), nil + case internal.ParsedModuleRef: + return newModuleRef(t), nil + case internal.ProtoFileRef: + return newProtoFileRef(t), nil + default: + return nil, fmt.Errorf("unknown ParsedRef type: %T", parsedRef) + } +} + func (a *refParser) GetSourceOrModuleRef( ctx context.Context, value string, @@ -327,6 +359,7 @@ func (a *refParser) GetModuleRef( return newModuleRef(parsedModuleRef), nil } +// TODO: rename to getParsedRefForString func (a *refParser) getParsedRef( ctx context.Context, value string, @@ -344,6 +377,23 @@ func (a *refParser) getParsedRef( return parsedRef, nil } +func (a *refParser) getParsedRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + allowedFormats []string, +) (internal.ParsedRef, error) { + parsedRef, err := a.fetchRefParser.GetParsedRefForInputConfig( + ctx, + inputConfig, + internal.WithAllowedFormats(allowedFormats...), + ) + if err != nil { + return nil, err + } + a.checkDeprecated(parsedRef) + return parsedRef, nil +} + func (a *refParser) checkDeprecated(parsedRef internal.ParsedRef) { format := parsedRef.Format() if replacementFormat, ok := deprecatedCompressionFormatToReplacementFormat[format]; ok { diff --git a/private/buf/buffetch/ref_parser_test.go b/private/buf/buffetch/ref_parser_test.go index 40b4ad562e..f829b8c45a 100644 --- a/private/buf/buffetch/ref_parser_test.go +++ b/private/buf/buffetch/ref_parser_test.go @@ -30,6 +30,7 @@ import ( "go.uber.org/zap" ) +// TODO: test ref from input config as well. func TestGetParsedRefSuccess(t *testing.T) { t.Parallel() // This allows us to test an os-agnostic root directory diff --git a/private/buf/bufgen/bufgen.go b/private/buf/bufgen/bufgen.go index d1be9663c5..0f64b70d09 100644 --- a/private/buf/bufgen/bufgen.go +++ b/private/buf/bufgen/bufgen.go @@ -116,15 +116,16 @@ type Generator interface { // NewGenerator returns a new Generator. func NewGenerator( logger *zap.Logger, - //controller bufctl.Controller, storageosProvider storageos.Provider, runner command.Runner, wasmPluginExecutor bufwasm.PluginExecutor, + // Pass a clientConfig instead of a CodeGenerationServiceClient because the + // plugins' remotes/registries is not known at this time, and remotes/registries + // may be different for different plugins. clientConfig *connectclient.Config, ) Generator { return newGenerator( logger, - //controller, storageosProvider, runner, wasmPluginExecutor, @@ -171,42 +172,6 @@ func GenerateWithWASMEnabled() GenerateOption { } } -// GenerateWithInputOverride says to generate for this input only, ignoring inputs -// from the config. -func GenerateWithInputOverride(input string) GenerateOption { - return func(generateOptions *generateOptions) { - generateOptions.input = input - } -} - -// GenerateWithModuleConfigPath says to build the image with a module config at this path. -func GenerateWithModuleConfigPath(moduleConfigPath string) GenerateOption { - return func(generateOptions *generateOptions) { - generateOptions.moduleConfigPath = moduleConfigPath - } -} - -// GenerateWithIncludePaths says to only generate code for these paths. -func GenerateWithIncludePaths(includePaths []string) GenerateOption { - return func(generateOptions *generateOptions) { - generateOptions.includePaths = includePaths - } -} - -// GenerateWithExcludePaths says to not generate code for these paths. -func GenerateWithExcludePaths(excludePaths []string) GenerateOption { - return func(generateOptions *generateOptions) { - generateOptions.excludePaths = excludePaths - } -} - -// GenerateWithIncludeTypes says to only generate code for these types. -func GenerateWithIncludeTypes(includeTypes []string) GenerateOption { - return func(generateOptions *generateOptions) { - generateOptions.includeTypes = includeTypes - } -} - // Config is a configuration. type Config struct { // Required diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index f3fe436a05..0fb1f126e1 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -19,10 +19,8 @@ import ( "errors" "fmt" "path/filepath" - "strconv" connect "connectrpc.com/connect" - "github.com/bufbuild/buf/private/buf/bufctl" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify" @@ -44,11 +42,8 @@ import ( "google.golang.org/protobuf/types/pluginpb" ) -const defaultInput = "." - type generator struct { logger *zap.Logger - controller bufctl.Controller storageosProvider storageos.Provider pluginexecGenerator bufpluginexec.Generator clientConfig *connectclient.Config @@ -56,7 +51,6 @@ type generator struct { func newGenerator( logger *zap.Logger, - controller bufctl.Controller, storageosProvider storageos.Provider, runner command.Runner, wasmPluginExecutor bufwasm.PluginExecutor, @@ -64,7 +58,6 @@ func newGenerator( ) *generator { return &generator{ logger: logger, - controller: controller, storageosProvider: storageosProvider, pluginexecGenerator: bufpluginexec.NewGenerator(logger, storageosProvider, runner, wasmPluginExecutor), clientConfig: clientConfig, @@ -92,34 +85,21 @@ func (g *generator) Generate( ctx context.Context, container app.EnvStdioContainer, config bufconfig.GenerateConfig, + images []bufimage.Image, options ...GenerateOption, ) error { generateOptions := newGenerateOptions() for _, option := range options { option(generateOptions) } - inputImages, err := getInputImages( - ctx, - g.logger, - g.controller, - generateOptions.input, - config, - generateOptions.moduleConfigPath, - generateOptions.includePaths, - generateOptions.excludePaths, - generateOptions.includeTypes, - ) - if err != nil { - return err - } - for _, inputImage := range inputImages { - if err := bufimagemodify.Modify(ctx, inputImage, config.GenerateManagedConfig()); err != nil { + for _, image := range images { + if err := bufimagemodify.Modify(ctx, image, config.GenerateManagedConfig()); err != nil { return err } if err := g.generateCode( ctx, container, - inputImage, + image, generateOptions.baseOutDirPath, config.GeneratePluginConfigs(), generateOptions.includeImports, @@ -181,123 +161,6 @@ func (g *generator) generateCode( return nil } -// TODO: this is a very temporary solution, although it would be nice if buffetch exposes function that parses ref from a map -func refStringForInputConfig( - ctx context.Context, - logger *zap.Logger, - inputConfig bufconfig.GenerateInputConfig, -) string { - var refString = inputConfig.Location() - refOptionKeyToValue := map[string]string{} - if inputConfig.Compression() != "" { - refOptionKeyToValue["compression"] = inputConfig.Compression() - } - if inputConfig.StripComponents() != nil { - refOptionKeyToValue["strip_components"] = strconv.FormatUint(uint64(*inputConfig.StripComponents()), 10) - } - if inputConfig.Subdir() != "" { - refOptionKeyToValue["subdir"] = inputConfig.Subdir() - } - if inputConfig.Branch() != "" { - refOptionKeyToValue["branch"] = inputConfig.Branch() - } - if inputConfig.Tag() != "" { - refOptionKeyToValue["tag"] = inputConfig.Tag() - } - if inputConfig.Ref() != "" { - refOptionKeyToValue["ref"] = inputConfig.Ref() - } - // TODO: != 0 - if inputConfig.Depth() != nil { - refOptionKeyToValue["depth"] = strconv.FormatUint(uint64(*inputConfig.Depth()), 10) - } - if inputConfig.RecurseSubmodules() { - refOptionKeyToValue["recurse_submodules"] = "true" - } - if inputConfig.IncludePackageFiles() { - refOptionKeyToValue["include_package_files"] = "true" - } - if len(refOptionKeyToValue) == 0 { - return refString - } - refString += "#" - for key, value := range refOptionKeyToValue { - refString += key + "=" + value - } - return refString -} - -func getInputImages( - ctx context.Context, - logger *zap.Logger, - controller bufctl.Controller, - inputSpecified string, - config bufconfig.GenerateConfig, - moduleConfigOverride string, - includePathsOverride []string, - excludePathsOverride []string, - includeTypesOverride []string, -) ([]bufimage.Image, error) { - var inputImages []bufimage.Image - // If input is specified on the command line, we use that. If input is not - // specified on the command line, but the config has no inputs, use the default input. - if inputSpecified != "" || len(config.GenerateInputConfigs()) == 0 { - input := defaultInput - if inputSpecified != "" { - input = inputSpecified - } - var includeTypes []string - if typesConfig := config.GenerateTypeConfig(); typesConfig != nil { - includeTypes = typesConfig.IncludeTypes() - } - if len(includeTypesOverride) > 0 { - includeTypes = includeTypesOverride - } - inputImage, err := controller.GetImage( - ctx, - input, - bufctl.WithConfigOverride(moduleConfigOverride), - bufctl.WithTargetPaths(includePathsOverride, excludePathsOverride), - bufctl.WithImageTypes(includeTypes), - ) - if err != nil { - return nil, err - } - inputImages = []bufimage.Image{inputImage} - } else { - for _, inputConfig := range config.GenerateInputConfigs() { - includePaths := inputConfig.IncludePaths() - if len(includePathsOverride) > 0 { - includePaths = includePathsOverride - } - excludePaths := inputConfig.ExcludePaths() - if len(excludePathsOverride) > 0 { - excludePaths = excludePathsOverride - } - // In V2 we do not need to look at inputConfig.GenerateTypeConfig().IncludeTypes() - // because inputConfig.GenerateTypeConfig() is always nil. - // TODO: document the above in godoc - includeTypes := inputConfig.IncludeTypes() - if len(includeTypesOverride) > 0 { - includeTypes = includeTypesOverride - } - input := refStringForInputConfig(ctx, logger, inputConfig) - inputImage, err := controller.GetImage( - ctx, - input, - bufctl.WithConfigOverride(moduleConfigOverride), - bufctl.WithTargetPaths(includePaths, excludePaths), - bufctl.WithImageTypes(includeTypes), - ) - if err != nil { - return nil, err - } - inputImages = append(inputImages, inputImage) - } - } - return inputImages, nil -} - func (g *generator) execPlugins( ctx context.Context, container app.EnvStdioContainer, @@ -573,13 +436,6 @@ type generateOptions struct { includeImports bool includeWellKnownTypes bool wasmEnabled bool - // image/input specific options: - input string - moduleConfigPath string - // TODO: unify naming: includePaths / pathsIncluded / pathSpecified - includePaths []string - excludePaths []string - includeTypes []string } func newGenerateOptions() *generateOptions { diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 2d9cb6287d..6e765ddc80 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -26,6 +26,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufgen" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufwasm" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appflag" @@ -34,6 +35,7 @@ import ( "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/spf13/cobra" "github.com/spf13/pflag" + "go.uber.org/zap" ) const ( @@ -321,7 +323,7 @@ func run( if err != nil { return err } - var config bufconfig.GenerateConfig + var bufGenYAMLFile bufconfig.BufGenYAMLFile templatePathExtension := filepath.Ext(flags.Template) switch { case flags.Template == "": @@ -329,7 +331,7 @@ func run( if err != nil { return err } - config, err = bufconfig.GetBufGenYAMLFileForPrefix(ctx, bucket, ".") + bufGenYAMLFile, err = bufconfig.GetBufGenYAMLFileForPrefix(ctx, bucket, ".") if err != nil { return err } @@ -339,23 +341,37 @@ func run( if err != nil { return err } - config, err = bufconfig.ReadBufGenYAMLFile(configReader) + bufGenYAMLFile, err = bufconfig.ReadBufGenYAMLFile(configReader) if err != nil { return err } default: - config, err = bufconfig.ReadBufGenYAMLFile(strings.NewReader(flags.Template)) + bufGenYAMLFile, err = bufconfig.ReadBufGenYAMLFile(strings.NewReader(flags.Template)) if err != nil { return err } } + images, err := getInputImages( + ctx, + logger, + controller, + input, + bufGenYAMLFile, + flags.Config, + flags.Paths, + flags.ExcludePaths, + flags.Types, + ) + if err != nil { + return err + } generateOptions := []bufgen.GenerateOption{ bufgen.GenerateWithBaseOutDirPath(flags.BaseOutDirPath), - bufgen.GenerateWithIncludePaths(flags.Paths), - bufgen.GenerateWithExcludePaths(flags.ExcludePaths), - bufgen.GenerateWithIncludeTypes(flags.Types), - bufgen.GenerateWithInputOverride(input), - bufgen.GenerateWithModuleConfigPath(flags.Config), + // bufgen.GenerateWithIncludePaths(flags.Paths), + // bufgen.GenerateWithExcludePaths(flags.ExcludePaths), + // bufgen.GenerateWithIncludeTypes(flags.Types), + // bufgen.GenerateWithInputOverride(input), + // bufgen.GenerateWithModuleConfigPath(flags.Config), } if flags.IncludeImports { generateOptions = append( @@ -381,7 +397,6 @@ func run( } return bufgen.NewGenerator( logger, - controller, storageosProvider, command.NewRunner(), wasmPluginExecutor, @@ -389,7 +404,78 @@ func run( ).Generate( ctx, container, - config, + bufGenYAMLFile.GenerateConfig(), + images, generateOptions..., ) } + +func getInputImages( + ctx context.Context, + logger *zap.Logger, + controller bufctl.Controller, + inputSpecified string, + bufGenYAMLFile bufconfig.BufGenYAMLFile, + moduleConfigOverride string, + includePathsOverride []string, + excludePathsOverride []string, + includeTypesOverride []string, +) ([]bufimage.Image, error) { + var inputImages []bufimage.Image + // If input is specified on the command line, we use that. If input is not + // specified on the command line, but the config has no inputs, use the default input. + if inputSpecified != "" || len(bufGenYAMLFile.InputConfigs()) == 0 { + input := "." + if inputSpecified != "" { + input = inputSpecified + } + var includeTypes []string + if typesConfig := bufGenYAMLFile.GenerateConfig().GenerateTypeConfig(); typesConfig != nil { + includeTypes = typesConfig.IncludeTypes() + } + if len(includeTypesOverride) > 0 { + includeTypes = includeTypesOverride + } + inputImage, err := controller.GetImage( + ctx, + input, + bufctl.WithConfigOverride(moduleConfigOverride), + bufctl.WithTargetPaths(includePathsOverride, excludePathsOverride), + bufctl.WithImageTypes(includeTypes), + ) + if err != nil { + return nil, err + } + inputImages = []bufimage.Image{inputImage} + } else { + for _, inputConfig := range bufGenYAMLFile.InputConfigs() { + includePaths := inputConfig.IncludePaths() + if len(includePathsOverride) > 0 { + includePaths = includePathsOverride + } + excludePaths := inputConfig.ExcludePaths() + if len(excludePathsOverride) > 0 { + excludePaths = excludePathsOverride + } + // In V2 we do not need to look at generateTypeConfig.IncludeTypes() + // because it is always nil. + // TODO: document the above in godoc + includeTypes := inputConfig.IncludeTypes() + if len(includeTypesOverride) > 0 { + includeTypes = includeTypesOverride + } + inputImage, err := controller.GetImageForInputConfig( + ctx, + inputConfig, + bufctl.WithConfigOverride(moduleConfigOverride), + bufctl.WithTargetPaths(includePaths, excludePaths), + bufctl.WithImageTypes(includeTypes), + ) + if err != nil { + return nil, err + } + inputImages = append(inputImages, inputImage) + } + } + return inputImages, nil +} diff --git a/private/buf/cmd/buf/command/generate/generate_test.go b/private/buf/cmd/buf/command/generate/generate_test.go index 285af712c3..b7e1e463ac 100644 --- a/private/buf/cmd/buf/command/generate/generate_test.go +++ b/private/buf/cmd/buf/command/generate/generate_test.go @@ -152,6 +152,8 @@ func TestOutputFlag(t *testing.T) { } func TestProtoFileRefIncludePackageFiles(t *testing.T) { + // TODO: un-skip this once proto file ref is implemented + t.Skip() t.Parallel() tempDirPath := t.TempDir() testRunSuccess( @@ -186,6 +188,8 @@ func TestGenerateDuplicatePlugins(t *testing.T) { } func TestOutputWithPathEqualToExclude(t *testing.T) { + // TODO: un-skip this once --path and --exclude-path are updated + t.Skip() t.Parallel() tempDirPath := t.TempDir() testRunStdoutStderr( @@ -517,7 +521,23 @@ func testRunStdoutStderr(t *testing.T, stdin io.Reader, expectedExitCode int, ex func(name string) *appcmd.Command { return NewCommand( name, - appflag.NewBuilder(name), + appflag.NewBuilder( + name, + appflag.BuilderWithInterceptor( + // TODO: use the real interceptor. Currently in buf.go, NewBuilder receives appflag.BuilderWithInterceptor(newErrorInterceptor()). + // However we cannot depend on newErrorInterceptor because it would create an import cycle, not to mention it needs to be exported first. + // This can depend on newErroInterceptor when it's moved to a separate package and made public. + func(next func(context.Context, appflag.Container) error) func(context.Context, appflag.Container) error { + return func(ctx context.Context, container appflag.Container) error { + err := next(ctx, container) + if err == nil { + return nil + } + return fmt.Errorf("Failure: %w", err) + } + }, + ), + ), ) }, expectedExitCode, diff --git a/private/buf/cmd/buf/command/generate/generate_unix_test.go b/private/buf/cmd/buf/command/generate/generate_unix_test.go index aef2ea35da..58108c7606 100644 --- a/private/buf/cmd/buf/command/generate/generate_unix_test.go +++ b/private/buf/cmd/buf/command/generate/generate_unix_test.go @@ -26,6 +26,8 @@ import ( ) func TestProtoFileRef(t *testing.T) { + // TODO: un-skip this once --path and --exclude-path are updated + t.Skip() t.Parallel() tempDirPath := t.TempDir() testRunSuccess( @@ -77,6 +79,8 @@ func TestOutputWithExclude(t *testing.T) { } func TestOutputWithPathWithinExclude(t *testing.T) { + // TODO: un-skip this once --path and --exclude-path are updated + t.Skip() t.Parallel() tempDirPath := t.TempDir() testRunSuccess( @@ -124,6 +128,8 @@ func TestOutputWithExcludeWithinPath(t *testing.T) { } func TestOutputWithNestedExcludeAndTargetPaths(t *testing.T) { + // TODO: un-skip this once --path and --exclude-path are updated + t.Skip() t.Parallel() tempDirPath := t.TempDir() testRunSuccess( @@ -160,6 +166,8 @@ func TestOutputWithNestedExcludeAndTargetPaths(t *testing.T) { } func TestWorkspaceGenerateWithExcludeAndTargetPaths(t *testing.T) { + // TODO: un-skip this once --path and --exclude-path are updated + t.Skip() t.Parallel() tempDirPath := t.TempDir() testRunSuccess( diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 09bc2f90c2..8427df19b4 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -20,6 +20,7 @@ import ( "fmt" "io" + "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/syserror" ) @@ -89,22 +90,36 @@ func WriteBufGenYAMLFile(writer io.Writer, bufGenYAMLFile BufGenYAMLFile) error // *** PRIVATE *** type bufGenYAMLFile struct { - GenerateConfig + generateConfig GenerateConfig + inputConfigs []InputConfig fileVersion FileVersion } -func newBufGenYAMLFile(fileVersion FileVersion, generateConfig GenerateConfig) (*bufGenYAMLFile, error) { +func newBufGenYAMLFile( + fileVersion FileVersion, + generateConfig GenerateConfig, + inputConfigs []InputConfig, +) *bufGenYAMLFile { return &bufGenYAMLFile{ - GenerateConfig: generateConfig, fileVersion: fileVersion, - }, errors.New("TODO") + generateConfig: generateConfig, + inputConfigs: inputConfigs, + } } func (g *bufGenYAMLFile) FileVersion() FileVersion { return g.fileVersion } +func (g *bufGenYAMLFile) GenerateConfig() GenerateConfig { + return g.generateConfig +} + +func (g *bufGenYAMLFile) InputConfigs() []InputConfig { + return g.inputConfigs +} + func (*bufGenYAMLFile) isBufGenYAMLFile() {} func (*bufGenYAMLFile) isFile() {} @@ -129,10 +144,11 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error if err != nil { return nil, err } - return &bufGenYAMLFile{ - fileVersion: fileVersion, - GenerateConfig: generateConfig, - }, nil + return newBufGenYAMLFile( + fileVersion, + generateConfig, + nil, + ), nil case FileVersionV2: var externalGenYAMLFile externalBufGenYAMLFileV2 if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { @@ -142,10 +158,18 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error if err != nil { return nil, err } - return &bufGenYAMLFile{ - fileVersion: fileVersion, - GenerateConfig: generateConfig, - }, nil + inputConfigs, err := slicesext.MapError( + externalGenYAMLFile.Inputs, + newInputConfigFromExternalInputConfigV2, + ) + if err != nil { + return nil, err + } + return newBufGenYAMLFile( + fileVersion, + generateConfig, + inputConfigs, + ), nil default: // This is a system error since we've already parsed. return nil, syserror.Newf("unknown FileVersion: %v", fileVersion) diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index 0086032cb2..ebba7b0e50 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -75,17 +75,9 @@ func newGenerateConfigFromExternalFileV2( if err != nil { return nil, err } - inputConfigs, err := slicesext.MapError( - externalFile.Inputs, - newInputConfigFromExternalInputConfigV2, - ) - if err != nil { - return nil, err - } return &generateConfig{ managedConfig: managedConfig, pluginConfigs: pluginConfigs, - inputConfigs: inputConfigs, }, nil } @@ -95,7 +87,6 @@ type generateConfig struct { pluginConfigs []GeneratePluginConfig managedConfig GenerateManagedConfig typeConfig GenerateTypeConfig - inputConfigs []GenerateInputConfig } func (g *generateConfig) GeneratePluginConfigs() []GeneratePluginConfig { @@ -110,8 +101,4 @@ func (g *generateConfig) GenerateTypeConfig() GenerateTypeConfig { return g.typeConfig } -func (g *generateConfig) GenerateInputConfigs() []GenerateInputConfig { - return g.inputConfigs -} - func (*generateConfig) isGenerateConfig() {} diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 96552da255..7505ba370a 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -269,60 +269,60 @@ func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV return inputConfig, nil } -func (g *inputConfig) Type() InputConfigType { - return g.inputType +func (i *inputConfig) Type() InputConfigType { + return i.inputType } -func (g *inputConfig) Location() string { - return g.location +func (i *inputConfig) Location() string { + return i.location } -func (g *inputConfig) Compression() string { - return g.compression +func (i *inputConfig) Compression() string { + return i.compression } -func (g *inputConfig) StripComponents() *uint32 { - return g.stripComponents +func (i *inputConfig) StripComponents() *uint32 { + return i.stripComponents } -func (g *inputConfig) Subdir() string { - return g.subdir +func (i *inputConfig) Subdir() string { + return i.subdir } -func (g *inputConfig) Branch() string { - return g.branch +func (i *inputConfig) Branch() string { + return i.branch } -func (g *inputConfig) Tag() string { - return g.tag +func (i *inputConfig) Tag() string { + return i.tag } -func (g *inputConfig) Ref() string { - return g.ref +func (i *inputConfig) Ref() string { + return i.ref } -func (g *inputConfig) Depth() *uint32 { - return g.depth +func (i *inputConfig) Depth() *uint32 { + return i.depth } -func (g *inputConfig) RecurseSubmodules() bool { - return g.recurseSubmodules +func (i *inputConfig) RecurseSubmodules() bool { + return i.recurseSubmodules } -func (g *inputConfig) IncludePackageFiles() bool { - return g.includePackageFiles +func (i *inputConfig) IncludePackageFiles() bool { + return i.includePackageFiles } -func (g *inputConfig) ExcludePaths() []string { - return g.excludePaths +func (i *inputConfig) ExcludePaths() []string { + return i.excludePaths } -func (g *inputConfig) IncludePaths() []string { - return g.includePaths +func (i *inputConfig) IncludePaths() []string { + return i.includePaths } -func (g *inputConfig) IncludeTypes() []string { - return g.includeTypes +func (i *inputConfig) IncludeTypes() []string { + return i.includeTypes } -func (g *inputConfig) isInputConfig() {} +func (i *inputConfig) isInputConfig() {} From 0eb678c1599499a6bed30a2857070370d8e6b5e4 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Wed, 29 Nov 2023 19:37:24 -0500 Subject: [PATCH 38/66] migrate --- TODO.txt | 2 + private/buf/buffetch/buffetch.go | 29 +++ private/buf/buffetch/internal/internal.go | 62 ++++++ private/buf/buffetch/internal/ref_parser.go | 38 +--- .../buf/cmd/buf/command/generate/generate.go | 28 ++- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 7 +- private/bufpkg/bufconfig/input_config.go | 176 ++++++++++++++++-- 7 files changed, 288 insertions(+), 54 deletions(-) diff --git a/TODO.txt b/TODO.txt index 32ad43e83b..c73cb70c9c 100644 --- a/TODO.txt +++ b/TODO.txt @@ -26,3 +26,5 @@ - document behavior of file searching, config override - fix tamper-proofing - go through all todos + +- update behavior of --path and --exclude-path to make it match what's on main, esp. --path a/b.proto --exclude-path a diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index 9201191828..0a333d2a63 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -16,6 +16,7 @@ package buffetch import ( "context" + "fmt" "io" "net/http" @@ -378,3 +379,31 @@ func NewWriter( logger, ) } + +// GetInputConfigForRef returns the input config for the ref. +func GetInputConfigForRef(ref Ref) (bufconfig.InputConfig, error) { + switch t := ref.(type) { + case MessageRef: + switch t.MessageEncoding() { + case MessageEncodingBinpb: + return bufconfig.NewBinaryImageInputConfig( + t.Path(), + t.internalSingleRef().CompressionType().String(), + ), nil + case MessageEncodingJSON: + return bufconfig.NewJSONImageInputConfig( + t.Path(), + t.internalSingleRef().CompressionType().String(), + ), nil + case MessageEncodingTxtpb: + return bufconfig.NewBinaryImageInputConfig( + t.Path(), + t.internalSingleRef().CompressionType().String(), + ), nil + default: + // TODO: handle the YAML cas + return nil, fmt.Errorf("unknown encoding: %v", t.MessageEncoding()) + } + } + return internal.GetInputConfigForRef(ref.internalRef()) +} diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index 71c5946f3a..3ed0bf5660 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -16,8 +16,11 @@ package internal import ( "context" + "errors" + "fmt" "io" "net/http" + "strconv" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufmodule" @@ -85,6 +88,20 @@ type ArchiveType int // CompressionType is a compression type. type CompressionType int +// String implements fmt.Stringer +func (c CompressionType) String() string { + switch c { + case CompressionTypeNone: + return "none" + case CompressionTypeGzip: + return "gzip" + case CompressionTypeZstd: + return "zstd" + default: + return strconv.Itoa(int(c)) + } +} + // Ref is a reference. type Ref interface { ref() @@ -848,3 +865,48 @@ func WithPutFileNoFileCompression() PutFileOption { // GetModuleOption is a GetModule option. type GetModuleOption func(*getModuleOptions) + +// TODO: delete this -- this cannot handle git +// GetInputConfigForRef returns the input config for the ref. +func GetInputConfigForRef(ref Ref) (bufconfig.InputConfig, error) { + switch t := ref.(type) { + case ArchiveRef: + switch t.ArchiveType() { + case ArchiveTypeZip: + return bufconfig.NewZipArchiveInputConfig( + t.Path(), + t.SubDirPath(), + uint32ToPointer(t.StripComponents()), + ), nil + case ArchiveTypeTar: + return bufconfig.NewTarballInputConfig( + t.Path(), + t.SubDirPath(), + t.CompressionType().String(), + uint32ToPointer(t.StripComponents()), + ), nil + default: + return nil, fmt.Errorf("invalid archive type: %v", t.ArchiveType()) + } + case DirRef: + return bufconfig.NewDirectoryInputConfig( + t.Path(), + ), nil + case ModuleRef: + return bufconfig.NewModuleInputConfig( + t.ModuleRef().String(), + ), nil + case ProtoFileRef: + return bufconfig.NewProtoFileInputConfig( + t.Path(), + ), nil + case GitRef: + return nil, errors.New("TODO: git") + default: + return nil, fmt.Errorf("unexpected Ref of type %T", ref) + } +} + +func uint32ToPointer(value uint32) *uint32 { + return &value +} diff --git a/private/buf/buffetch/internal/ref_parser.go b/private/buf/buffetch/internal/ref_parser.go index 704c9421d7..f01742a122 100644 --- a/private/buf/buffetch/internal/ref_parser.go +++ b/private/buf/buffetch/internal/ref_parser.go @@ -310,7 +310,7 @@ func (a *refParser) getRawRefFromInputConfig( } rawRef.GitRecurseSubmodules = inputConfig.RecurseSubmodules() rawRef.IncludePackageFiles = inputConfig.IncludePackageFiles() - rawRef.SubDirPath, err = parseSubDirPath(inputConfig.Subdir()) + rawRef.SubDirPath, err = parseSubDirPath(inputConfig.SubDir()) if err != nil { return nil, err } @@ -366,6 +366,7 @@ func (a *refParser) validateRawRef( return nil } +// TODO: these functions may not be necessary // empty value is an error func parseCompressionType(value string) (CompressionType, error) { switch value { @@ -444,41 +445,6 @@ func getRawPathAndOptionsForInputString(value string) (string, map[string]string } } -// // TODO: this is a very temporary (and not clean) solution. -// // To be deleted shortly -// func getRawPathAndOptionsForInputConfig(inputConfig bufconfig.InputConfig) (string, map[string]string, error) { -// var refString = inputConfig.Location() -// refOptionKeyToValue := map[string]string{} -// if inputConfig.Compression() != "" { -// refOptionKeyToValue["compression"] = inputConfig.Compression() -// } -// if inputConfig.StripComponents() != nil { -// refOptionKeyToValue["strip_components"] = strconv.FormatUint(uint64(*inputConfig.StripComponents()), 10) -// } -// if inputConfig.Subdir() != "" { -// refOptionKeyToValue["subdir"] = inputConfig.Subdir() -// } -// if inputConfig.Branch() != "" { -// refOptionKeyToValue["branch"] = inputConfig.Branch() -// } -// if inputConfig.Tag() != "" { -// refOptionKeyToValue["tag"] = inputConfig.Tag() -// } -// if inputConfig.Ref() != "" { -// refOptionKeyToValue["ref"] = inputConfig.Ref() -// } -// if inputConfig.Depth() != nil { -// refOptionKeyToValue["depth"] = strconv.FormatUint(uint64(*inputConfig.Depth()), 10) -// } -// if inputConfig.RecurseSubmodules() { -// refOptionKeyToValue["recurse_submodules"] = "true" -// } -// if inputConfig.IncludePackageFiles() { -// refOptionKeyToValue["include_package_files"] = "true" -// } -// return refString, refOptionKeyToValue, nil -// } - func getSingleRef( rawRef *RawRef, defaultCompressionType CompressionType, diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 6e765ddc80..87180182ae 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -51,6 +51,7 @@ const ( disableSymlinksFlagName = "disable-symlinks" typeFlagName = "type" typeDeprecatedFlagName = "include-types" + migrateFlagName = "migrate" ) // NewCommand returns a new Command. @@ -211,6 +212,7 @@ type flags struct { // want to find out what will break if we do. Types []string TypesDeprecated []string + Migrate bool // special InputHashtag string } @@ -279,6 +281,12 @@ func (f *flags) Bind(flagSet *pflag.FlagSet) { nil, "The types (package, message, enum, extension, service, method) that should be included in this image. When specified, the resulting image will only include descriptors to describe the requested types. Flag usage overrides buf.gen.yaml", ) + flagSet.BoolVar( + &f.Migrate, + migrateFlagName, + false, + "Migrate the generation template to the latest version", + ) _ = flagSet.MarkDeprecated(typeDeprecatedFlagName, fmt.Sprintf("Use --%s instead", typeFlagName)) _ = flagSet.MarkHidden(typeDeprecatedFlagName) } @@ -335,21 +343,37 @@ func run( if err != nil { return err } + if flags.Migrate { + if err := bufconfig.PutBufGenYAMLFileForPrefix(ctx, bucket, ".", bufGenYAMLFile); err != nil { + return err + } + } case templatePathExtension == ".yaml" || templatePathExtension == ".yml" || templatePathExtension == ".json": // We should not read from a bucket at "." because this path can jump context. - configReader, err := os.Open(flags.Template) + configFile, err := os.Open(flags.Template) if err != nil { return err } - bufGenYAMLFile, err = bufconfig.ReadBufGenYAMLFile(configReader) + bufGenYAMLFile, err = bufconfig.ReadBufGenYAMLFile(configFile) if err != nil { return err } + if flags.Migrate { + if err := bufconfig.WriteBufGenYAMLFile(configFile, bufGenYAMLFile); err != nil { + return err + } + } default: bufGenYAMLFile, err = bufconfig.ReadBufGenYAMLFile(strings.NewReader(flags.Template)) if err != nil { return err } + if flags.Migrate { + return fmt.Errorf( + "invalid template: %q, migration can only apply to a file on disk with extension .yaml, .yml or .json", + flags.Template, + ) + } } images, err := getInputImages( ctx, diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 8427df19b4..a52b8461f4 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -177,12 +177,9 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error } func writeBufGenYAMLFile(writer io.Writer, bufGenYAMLFile BufGenYAMLFile) error { + // TODO: is this check necessary? switch fileVersion := bufGenYAMLFile.FileVersion(); fileVersion { - case FileVersionV1Beta1: - return errors.New("TODO") - case FileVersionV1: - return errors.New("TODO") - case FileVersionV2: + case FileVersionV1Beta1, FileVersionV1, FileVersionV2: return errors.New("TODO") default: // This is a system error since we've already parsed. diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 7505ba370a..2256e15fbd 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -125,9 +125,9 @@ type InputConfig interface { // StripComponents returns the number of directories to strip for tar or zip // inputs, not empty only if format is tarball or zip archive. StripComponents() *uint32 - // Subdir returns the subdirectory to use, not empty only if format is one + // SubDir returns the subdirectory to use, not empty only if format is one // git repo, tarball and zip archive. - Subdir() string + SubDir() string // Branch returns the git branch to checkout out, not empty only if format is git. Branch() string // Tag returns the git tag to checkout, not empty only if format is git. @@ -142,10 +142,10 @@ type InputConfig interface { // IncludePackageFiles returns other files in the same package as the proto file, // not empty only if format is proto file. IncludePackageFiles() bool - // ExcludePaths returns paths not to generate for. - ExcludePaths() []string // IncludePaths returns paths to generate for. IncludePaths() []string + // ExcludePaths returns paths not to generate for. + ExcludePaths() []string // IncludeTypes returns the types to generate. If GenerateConfig.GenerateTypeConfig() // returns a non-empty list of types. IncludeTypes() []string @@ -153,14 +153,168 @@ type InputConfig interface { isInputConfig() } +// NewInputConfig returns a new input config. +func NewInputConfig( + inputType InputConfigType, + location string, + compression string, + stripComponents *uint32, + subDir string, + branch string, + tag string, + ref string, + depth *uint32, + recurseSubmodules bool, + includePackageFiles bool, + includePaths []string, + excludePaths []string, + includeTypes []string, +) InputConfig { + return &inputConfig{ + inputType: inputType, + location: location, + compression: compression, + stripComponents: stripComponents, + subDir: subDir, + branch: branch, + tag: tag, + ref: ref, + depth: depth, + recurseSubmodules: recurseSubmodules, + includePackageFiles: includePackageFiles, + includePaths: includePaths, + excludePaths: excludePaths, + includeTypes: includeTypes, + } +} + +// NewGitRepoInputConfig returns an input config for a git repo. +func NewGitRepoInputConfig( + location string, + subDir string, + branch string, + tag string, + ref string, + depth *uint32, + recurseSubModules bool, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeGitRepo, + location: location, + subDir: subDir, + branch: branch, + tag: tag, + ref: ref, + depth: depth, + recurseSubmodules: recurseSubModules, + } +} + +// NewModuleInputConfig returns an input config for a module. +func NewModuleInputConfig( + location string, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeModule, + location: location, + } +} + +// NewDirectoryInputConfig returns an input config for a directory. +func NewDirectoryInputConfig( + location string, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeDirectory, + location: location, + } +} + +// NewProtoFileInputConfig returns an input config for a proto file. +func NewProtoFileInputConfig( + location string, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeProtoFile, + location: location, + } +} + +// NewTarballInputConfig returns an input config for a tarball. +func NewTarballInputConfig( + location string, + subDir string, + compression string, + stripComponents *uint32, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeTarball, + location: location, + subDir: subDir, + compression: compression, + stripComponents: stripComponents, + } +} + +// NewZipArchiveInputConfig returns an input config for a zip archive. +func NewZipArchiveInputConfig( + location string, + subDir string, + stripComponents *uint32, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeZipArchive, + location: location, + subDir: subDir, + stripComponents: stripComponents, + } +} + +// NewBinaryImageInputConfig returns an input config for a binary image. +func NewBinaryImageInputConfig( + location string, + compression string, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeBinaryImage, + location: location, + compression: compression, + } +} + +// NewJSONImageInputConfig returns an input config for a JSON image. +func NewJSONImageInputConfig( + location string, + compression string, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeJSONImage, + location: location, + compression: compression, + } +} + +// NewTextImageInputConfig returns an input config for a text image. +func NewTextImageInputConfig( + location string, + compression string, +) InputConfig { + return &inputConfig{ + inputType: InputConfigTypeTextImage, + location: location, + compression: compression, + } +} + // *** PRIVATE *** type inputConfig struct { - inputType InputConfigType - location string - compression string + inputType InputConfigType + location string + compression string + // TODO: does it make sense to be a pointer? stripComponents *uint32 - subdir string + subDir string branch string tag string ref string @@ -222,7 +376,7 @@ func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV } if externalConfig.Subdir != nil { options = append(options, optionSubdir) - inputConfig.subdir = *externalConfig.Subdir + inputConfig.subDir = *externalConfig.Subdir } if externalConfig.Branch != nil { options = append(options, optionBranch) @@ -285,8 +439,8 @@ func (i *inputConfig) StripComponents() *uint32 { return i.stripComponents } -func (i *inputConfig) Subdir() string { - return i.subdir +func (i *inputConfig) SubDir() string { + return i.subDir } func (i *inputConfig) Branch() string { From 6b66e0d005b0e649bffc04fe20c461a0bc45c96d Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 11:07:14 -0500 Subject: [PATCH 39/66] update TODO --- TODO.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TODO.txt b/TODO.txt index c73cb70c9c..5283fcf1b1 100644 --- a/TODO.txt +++ b/TODO.txt @@ -27,4 +27,6 @@ - fix tamper-proofing - go through all todos -- update behavior of --path and --exclude-path to make it match what's on main, esp. --path a/b.proto --exclude-path a +- Un-skip relevant tests in generate_test.go and generate_unix_test.go, when + - the behavior of --path and --exclude-path is updated to match what's on main + - proto file ref is implemented From 77729a411e650105319d3533ab731d20b9dde1ed Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 11:24:24 -0500 Subject: [PATCH 40/66] commit --- private/buf/buffetch/buffetch.go | 21 +++++ private/buf/buffetch/ref_parser.go | 108 +++++++++++++++++++++-- private/bufpkg/bufconfig/input_config.go | 2 +- 3 files changed, 122 insertions(+), 9 deletions(-) diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index 0a333d2a63..00bcfa36f3 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -133,18 +133,33 @@ type ProtoFileRef interface { type MessageRefParser interface { // GetMessageRef gets the reference for the message file. GetMessageRef(ctx context.Context, value string) (MessageRef, error) + // GetMessageRefForInputConfig gets the reference for the message file. + GetMessageRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + ) (MessageRef, error) } // SourceRefParser is a source ref parser for Buf. type SourceRefParser interface { // GetSourceRef gets the reference for the source file. GetSourceRef(ctx context.Context, value string) (SourceRef, error) + // GetSourceRef gets the reference for the source file. + GetSourceRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + ) (SourceRef, error) } // DirRefParser is a dif ref parser for Buf. type DirRefParser interface { // GetDirRef gets the reference for the source file. GetDirRef(ctx context.Context, value string) (DirRef, error) + // GetDirRefForInputConfig gets the reference for the source file. + GetDirRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + ) (DirRef, error) } // ModuleRefParser is a source ref parser for Buf. @@ -162,6 +177,11 @@ type SourceOrModuleRefParser interface { // GetSourceOrModuleRef gets the reference for the message file or source bucket. GetSourceOrModuleRef(ctx context.Context, value string) (SourceOrModuleRef, error) + // GetSourceOrModuleRefForInputConfig gets the reference for the message file or source bucket. + GetSourceOrModuleRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + ) (SourceOrModuleRef, error) } // RefParser is a ref parser for Buf. @@ -173,6 +193,7 @@ type RefParser interface { // GetRef gets the reference for the message file, source bucket, or module. GetRef(ctx context.Context, value string) (Ref, error) + // GetRefForInputConfig gets the reference for the message file, source bucket, or module. GetRefForInputConfig(ctx context.Context, inputConfig bufconfig.InputConfig) (Ref, error) } diff --git a/private/buf/buffetch/ref_parser.go b/private/buf/buffetch/ref_parser.go index a0945de902..0c715f62e9 100644 --- a/private/buf/buffetch/ref_parser.go +++ b/private/buf/buffetch/ref_parser.go @@ -205,11 +205,10 @@ func newSourceOrModuleRefParser(logger *zap.Logger) *refParser { } } -// TODO: rename to GetRefForString func (a *refParser) GetRef( ctx context.Context, value string, -) (_ Ref, retErr error) { +) (Ref, error) { parsedRef, err := a.getParsedRef(ctx, value, allFormats) if err != nil { return nil, err @@ -239,7 +238,7 @@ func (a *refParser) GetRef( func (a *refParser) GetRefForInputConfig( ctx context.Context, inputConfig bufconfig.InputConfig, -) (_ Ref, retErr error) { +) (Ref, error) { parsedRef, err := a.getParsedRefForInputConfig(ctx, inputConfig, allFormats) if err != nil { return nil, err @@ -269,7 +268,7 @@ func (a *refParser) GetRefForInputConfig( func (a *refParser) GetSourceOrModuleRef( ctx context.Context, value string, -) (_ SourceOrModuleRef, retErr error) { +) (SourceOrModuleRef, error) { parsedRef, err := a.getParsedRef(ctx, value, sourceOrModuleFormats) if err != nil { return nil, err @@ -292,10 +291,36 @@ func (a *refParser) GetSourceOrModuleRef( } } +func (a *refParser) GetSourceOrModuleRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, +) (SourceOrModuleRef, error) { + parsedRef, err := a.getParsedRefForInputConfig(ctx, inputConfig, sourceOrModuleFormats) + if err != nil { + return nil, err + } + switch t := parsedRef.(type) { + case internal.ParsedSingleRef: + return nil, fmt.Errorf("invalid ParsedRef type for source or module: %T", parsedRef) + case internal.ParsedArchiveRef: + return newSourceRef(t), nil + case internal.ParsedDirRef: + return newSourceRef(t), nil + case internal.ParsedGitRef: + return newSourceRef(t), nil + case internal.ParsedModuleRef: + return newModuleRef(t), nil + case internal.ProtoFileRef: + return newProtoFileRef(t), nil + default: + return nil, fmt.Errorf("unknown ParsedRef type: %T", parsedRef) + } +} + func (a *refParser) GetMessageRef( ctx context.Context, value string, -) (_ MessageRef, retErr error) { +) (MessageRef, error) { parsedRef, err := a.getParsedRef(ctx, value, messageFormats) if err != nil { return nil, err @@ -311,10 +336,29 @@ func (a *refParser) GetMessageRef( return newMessageRef(parsedSingleRef, messageEncoding) } +func (a *refParser) GetMessageRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, +) (MessageRef, error) { + parsedRef, err := a.getParsedRefForInputConfig(ctx, inputConfig, messageFormats) + if err != nil { + return nil, err + } + parsedSingleRef, ok := parsedRef.(internal.ParsedSingleRef) + if !ok { + return nil, fmt.Errorf("invalid ParsedRef type for message: %T", parsedRef) + } + messageEncoding, err := parseMessageEncoding(parsedSingleRef.Format()) + if err != nil { + return nil, err + } + return newMessageRef(parsedSingleRef, messageEncoding) +} + func (a *refParser) GetSourceRef( ctx context.Context, value string, -) (_ SourceRef, retErr error) { +) (SourceRef, error) { parsedRef, err := a.getParsedRef(ctx, value, sourceFormats) if err != nil { return nil, err @@ -327,10 +371,26 @@ func (a *refParser) GetSourceRef( return newSourceRef(parsedBucketRef), nil } +func (a *refParser) GetSourceRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, +) (SourceRef, error) { + parsedRef, err := a.getParsedRefForInputConfig(ctx, inputConfig, sourceFormats) + if err != nil { + return nil, err + } + parsedBucketRef, ok := parsedRef.(internal.ParsedBucketRef) + if !ok { + // this should never happen + return nil, fmt.Errorf("invalid ParsedRef type for source: %T", parsedRef) + } + return newSourceRef(parsedBucketRef), nil +} + func (a *refParser) GetDirRef( ctx context.Context, value string, -) (_ DirRef, retErr error) { +) (DirRef, error) { parsedRef, err := a.getParsedRef(ctx, value, dirFormats) if err != nil { return nil, err @@ -343,10 +403,26 @@ func (a *refParser) GetDirRef( return newDirRef(parsedDirRef), nil } +func (a *refParser) GetDirRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, +) (DirRef, error) { + parsedRef, err := a.getParsedRefForInputConfig(ctx, inputConfig, dirFormats) + if err != nil { + return nil, err + } + parsedDirRef, ok := parsedRef.(internal.ParsedDirRef) + if !ok { + // this should never happen + return nil, fmt.Errorf("invalid ParsedRef type for source: %T", parsedRef) + } + return newDirRef(parsedDirRef), nil +} + func (a *refParser) GetModuleRef( ctx context.Context, value string, -) (_ ModuleRef, retErr error) { +) (ModuleRef, error) { parsedRef, err := a.getParsedRef(ctx, value, moduleFormats) if err != nil { return nil, err @@ -359,6 +435,22 @@ func (a *refParser) GetModuleRef( return newModuleRef(parsedModuleRef), nil } +func (a *refParser) GetModuleRefForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, +) (ModuleRef, error) { + parsedRef, err := a.getParsedRefForInputConfig(ctx, inputConfig, moduleFormats) + if err != nil { + return nil, err + } + parsedModuleRef, ok := parsedRef.(internal.ParsedModuleRef) + if !ok { + // this should never happen + return nil, fmt.Errorf("invalid ParsedRef type for source: %T", parsedRef) + } + return newModuleRef(parsedModuleRef), nil +} + // TODO: rename to getParsedRefForString func (a *refParser) getParsedRef( ctx context.Context, diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 2256e15fbd..82fec799cf 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -21,7 +21,7 @@ import ( "strconv" ) -// TODO: InputFormat? +// TODO: input type? type InputConfigType int const ( From c3400bc0265580e108be9213d13139b8fde36d76 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 14:51:02 -0500 Subject: [PATCH 41/66] migrate, without flags; need testing --- private/buf/buffetch/buffetch.go | 1 + .../buf/cmd/buf/command/generate/generate.go | 3 + private/bufpkg/bufconfig/buf_gen_yaml_file.go | 33 ++++- .../bufpkg/bufconfig/generate_config_test.go | 32 ++--- .../bufconfig/generate_managed_config.go | 56 ++++++++ .../bufconfig/generate_plugin_config.go | 133 ++++++++++++++---- private/bufpkg/bufconfig/input_config.go | 59 ++++++++ 7 files changed, 264 insertions(+), 53 deletions(-) diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index 00bcfa36f3..17bf969a2c 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -191,6 +191,7 @@ type RefParser interface { DirRefParser SourceOrModuleRefParser + // TODO: should this be renamed to GetRefForString? // GetRef gets the reference for the message file, source bucket, or module. GetRef(ctx context.Context, value string) (Ref, error) // GetRefForInputConfig gets the reference for the message file, source bucket, or module. diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 87180182ae..46b35f2b3a 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -343,6 +343,7 @@ func run( if err != nil { return err } + // TODO: Call a bufconfig.MigrateBufGenYAML instead if flags.Migrate { if err := bufconfig.PutBufGenYAMLFileForPrefix(ctx, bucket, ".", bufGenYAMLFile); err != nil { return err @@ -358,6 +359,7 @@ func run( if err != nil { return err } + // TODO: Call a bufconfig.MigrateBufGenYAML instead if flags.Migrate { if err := bufconfig.WriteBufGenYAMLFile(configFile, bufGenYAMLFile); err != nil { return err @@ -368,6 +370,7 @@ func run( if err != nil { return err } + // TODO: Call a bufconfig.MigrateBufGenYAML instead if flags.Migrate { return fmt.Errorf( "invalid template: %q, migration can only apply to a file on disk with extension .yaml, .yml or .json", diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index a52b8461f4..662cab5e67 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -20,6 +20,7 @@ import ( "fmt" "io" + "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/syserror" @@ -180,7 +181,37 @@ func writeBufGenYAMLFile(writer io.Writer, bufGenYAMLFile BufGenYAMLFile) error // TODO: is this check necessary? switch fileVersion := bufGenYAMLFile.FileVersion(); fileVersion { case FileVersionV1Beta1, FileVersionV1, FileVersionV2: - return errors.New("TODO") + // Regardless of version, we write the file as v2: + // TODO name external..config + pluginConfigs, err := slicesext.MapError( + bufGenYAMLFile.GenerateConfig().GeneratePluginConfigs(), + newExternalGeneratePluginConfigV2FromPluginConfig, + ) + if err != nil { + return err + } + managedConfig := newExternalManagedConfigV2FromGenerateManagedConfig( + bufGenYAMLFile.GenerateConfig().GenerateManagedConfig(), + ) + inputConfigs, err := slicesext.MapError( + bufGenYAMLFile.InputConfigs(), + newExternalInputConfigV2FromInputConfig, + ) + if err != nil { + return err + } + externalBufGenYAMLFileV2 := externalBufGenYAMLFileV2{ + Version: "v2", + Plugins: pluginConfigs, + Managed: managedConfig, + Inputs: inputConfigs, + } + data, err := encoding.MarshalYAML(&externalBufGenYAMLFileV2) + if err != nil { + return err + } + _, err = writer.Write(append(bufLockFileHeader, data...)) + return err default: // This is a system error since we've already parsed. return syserror.Newf("unknown FileVersion: %v", fileVersion) diff --git a/private/bufpkg/bufconfig/generate_config_test.go b/private/bufpkg/bufconfig/generate_config_test.go index 2a13a3f6f4..3f8d9dc7f3 100644 --- a/private/bufpkg/bufconfig/generate_config_test.go +++ b/private/bufpkg/bufconfig/generate_config_test.go @@ -49,21 +49,22 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "java", out: "java/out", - opt: "a=b,c", - strategy: GenerateStrategyAll, + // one string because it's one string in the config + opts: []string{"a=b,c"}, + strategy: toPointer(GenerateStrategyAll), }, }, }, }, { - description: "name_local_plugin_strategy", + description: "plugin_local_plugin_strategy", externalConfig: externalBufGenYAMLFileV1{ Version: "v1", Plugins: []externalGeneratePluginConfigV1{ { Plugin: "java", Out: "java/out", - Opt: "a=b,c", + Opt: "a", Strategy: "all", }, }, @@ -74,8 +75,8 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "java", out: "java/out", - opt: "a=b,c", - strategy: GenerateStrategyAll, + opts: []string{"a"}, + strategy: toPointer(GenerateStrategyAll), }, }, }, @@ -101,8 +102,8 @@ func TestParseConfigFromExternalV1(t *testing.T) { name: "go", out: "go/out", path: []string{"go", "run", "goplugin"}, - opt: "a=b,c", - strategy: GenerateStrategyDirectory, + opts: []string{"a=b", "c"}, + strategy: toPointer(GenerateStrategyDirectory), }, }, }, @@ -128,8 +129,8 @@ func TestParseConfigFromExternalV1(t *testing.T) { name: "go", out: "go/out", path: []string{"go", "run", "goplugin"}, - opt: "a=b,c", - strategy: GenerateStrategyDirectory, + opts: []string{"a=b", "c"}, + strategy: toPointer(GenerateStrategyDirectory), }, }, }, @@ -153,7 +154,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { name: "go2", out: "go2/out", path: []string{"protoc-gen-go"}, - strategy: GenerateStrategyDirectory, }, }, }, @@ -177,7 +177,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { name: "go2", out: "go2/out", path: []string{"protoc-gen-go"}, - strategy: GenerateStrategyDirectory, }, }, }, @@ -201,7 +200,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { name: "cpp", out: "cpp/out", protocPath: "path/to/protoc", - strategy: GenerateStrategyDirectory, }, }, }, @@ -225,7 +223,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { name: "cpp", out: "cpp/out", protocPath: "path/to/protoc", - strategy: GenerateStrategyDirectory, }, }, }, @@ -298,7 +295,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "go", out: "go/out", - strategy: GenerateStrategyDirectory, }, }, managedConfig: &generateManagedConfig{}, @@ -336,7 +332,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "go", out: "go/out", - strategy: GenerateStrategyDirectory, }, }, managedConfig: &generateManagedConfig{ @@ -415,7 +410,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "go", out: "go/out", - strategy: GenerateStrategyDirectory, }, }, managedConfig: &generateManagedConfig{ @@ -471,7 +465,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "go", out: "go/out", - strategy: GenerateStrategyDirectory, }, }, managedConfig: &generateManagedConfig{ @@ -522,7 +515,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "go", out: "go/out", - strategy: GenerateStrategyDirectory, }, }, managedConfig: &generateManagedConfig{ @@ -572,7 +564,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "go", out: "go/out", - strategy: GenerateStrategyDirectory, }, }, managedConfig: &generateManagedConfig{ @@ -628,7 +619,6 @@ func TestParseConfigFromExternalV1(t *testing.T) { pluginConfigType: PluginConfigTypeLocal, name: "go", out: "go/out", - strategy: GenerateStrategyDirectory, }, }, managedConfig: &generateManagedConfig{ diff --git a/private/bufpkg/bufconfig/generate_managed_config.go b/private/bufpkg/bufconfig/generate_managed_config.go index 12793cdaa4..1bdcc05d44 100644 --- a/private/bufpkg/bufconfig/generate_managed_config.go +++ b/private/bufpkg/bufconfig/generate_managed_config.go @@ -682,3 +682,59 @@ func overrideRulesForPerFileOverridesV1( } return overrideRules, nil } + +func newExternalManagedConfigV2FromGenerateManagedConfig( + managedConfig GenerateManagedConfig, +) externalGenerateManagedConfigV2 { + if managedConfig == nil { + return externalGenerateManagedConfigV2{} + } + var externalDisables []externalManagedDisableConfigV2 + for _, disable := range managedConfig.Disables() { + var fileOptionName string + if disable.FileOption() != FileOptionUnspecified { + fileOptionName = disable.FileOption().String() + } + var fieldOptionName string + if disable.FieldOption() != FieldOptionUnspecified { + fieldOptionName = disable.FieldOption().String() + } + externalDisables = append( + externalDisables, + externalManagedDisableConfigV2{ + FileOption: fileOptionName, + FieldOption: fieldOptionName, + Module: disable.ModuleFullName(), + Path: disable.Path(), + Field: disable.FieldName(), + }, + ) + } + var externalOverrides []externalManagedOverrideConfigV2 + for _, override := range managedConfig.Overrides() { + var fileOptionName string + if override.FileOption() != FileOptionUnspecified { + fileOptionName = override.FileOption().String() + } + var fieldOptionName string + if override.FieldOption() != FieldOptionUnspecified { + fieldOptionName = override.FieldOption().String() + } + externalOverrides = append( + externalOverrides, + externalManagedOverrideConfigV2{ + FileOption: fileOptionName, + FieldOption: fieldOptionName, + Module: override.ModuleFullName(), + Path: override.Path(), + Field: override.FieldName(), + Value: override.Value(), + }, + ) + } + return externalGenerateManagedConfigV2{ + Enabled: true, + Disable: externalDisables, + Override: externalOverrides, + } +} diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index bcf8ae7957..812f8bdb32 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -18,11 +18,14 @@ import ( "errors" "fmt" "math" + "os/exec" "strings" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" + "github.com/bufbuild/buf/private/bufpkg/bufpluginexec" "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" "github.com/bufbuild/buf/private/pkg/encoding" + "github.com/bufbuild/buf/private/pkg/syserror" ) const remoteAlphaPluginDeprecationMessage = "the remote field no longer works as " + @@ -104,25 +107,34 @@ type GeneratePluginConfig interface { isGeneratePluginConfig() } -func parseStrategy(s string) (GenerateStrategy, error) { +func parseStrategy(s string) (*GenerateStrategy, error) { + var strategy GenerateStrategy switch s { - case "", "directory": - return GenerateStrategyDirectory, nil + case "": + return nil, nil + case "directory": + strategy = GenerateStrategyDirectory case "all": - return GenerateStrategyAll, nil + strategy = GenerateStrategyAll default: - return 0, fmt.Errorf("unknown strategy: %s", s) + return nil, fmt.Errorf("unknown strategy: %s", s) } + return &strategy, nil } type pluginConfig struct { + // TODO: perhaps make some of these pointers so that whether a field is + // specified in external config is preserved. This way, we can migrate more + // accurately. + // But what do we do about opt and path? Opt() and Path() could then return an error. + // Or define Migrate(templateOverride) error, which probably works better. pluginConfigType PluginConfigType name string out string - opt string + opts []string includeImports bool includeWKT bool - strategy GenerateStrategy + strategy *GenerateStrategy path []string protocPath string remoteHost string @@ -142,7 +154,7 @@ func (p *pluginConfig) Out() string { } func (p *pluginConfig) Opt() string { - return p.opt + return strings.Join(p.opts, ",") } func (p *pluginConfig) IncludeImports() bool { @@ -154,7 +166,10 @@ func (p *pluginConfig) IncludeWKT() bool { } func (p *pluginConfig) Strategy() GenerateStrategy { - return p.strategy + if p.strategy == nil { + return GenerateStrategyDirectory + } + return *p.strategy } func (p *pluginConfig) Path() []string { @@ -215,7 +230,7 @@ func newPluginConfigFromExternalV1( if err != nil { return nil, err } - opt, err := encoding.InterfaceSliceOrStringToCommaSepString(externalConfig.Opt) + opt, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Opt) if err != nil { return nil, err } @@ -357,7 +372,7 @@ func newPluginConfigFromExternalV2( if err != nil { return nil, err } - opt, err := encoding.InterfaceSliceOrStringToCommaSepString(externalConfig.Opt) + opt, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Opt) if err != nil { return nil, err } @@ -415,24 +430,21 @@ func newPluginConfigFromExternalV2( // TODO: unify parameter order func newLocalPluginConfig( name string, - strategy GenerateStrategy, + strategy *GenerateStrategy, out string, - opt string, + opt []string, includeImports bool, includeWKT bool, ) (*pluginConfig, error) { if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } - if strategy == 0 { - strategy = GenerateStrategyDirectory - } return &pluginConfig{ pluginConfigType: PluginConfigTypeLocal, name: name, strategy: strategy, out: out, - opt: opt, + opts: opt, includeImports: includeImports, includeWKT: includeWKT, }, nil @@ -441,9 +453,9 @@ func newLocalPluginConfig( func newBinaryPluginConfig( name string, path []string, - strategy GenerateStrategy, + strategy *GenerateStrategy, out string, - opt string, + opt []string, includeImports bool, includeWKT bool, ) (*pluginConfig, error) { @@ -453,16 +465,13 @@ func newBinaryPluginConfig( if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } - if strategy == 0 { - strategy = GenerateStrategyDirectory - } return &pluginConfig{ pluginConfigType: PluginConfigTypeBinary, name: name, path: path, strategy: strategy, out: out, - opt: opt, + opts: opt, includeImports: includeImports, includeWKT: includeWKT, }, nil @@ -472,23 +481,20 @@ func newProtocBuiltinPluginConfig( name string, protocPath string, out string, - opt string, + opt []string, includeImports bool, includeWKT bool, - strategy GenerateStrategy, + strategy *GenerateStrategy, ) (*pluginConfig, error) { if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } - if strategy == 0 { - strategy = GenerateStrategyDirectory - } return &pluginConfig{ pluginConfigType: PluginConfigTypeProtocBuiltin, name: name, protocPath: protocPath, out: out, - opt: opt, + opts: opt, strategy: strategy, includeImports: includeImports, includeWKT: includeWKT, @@ -499,7 +505,7 @@ func newRemotePluginConfig( name string, revision int, out string, - opt string, + opt []string, includeImports bool, includeWKT bool, ) (*pluginConfig, error) { @@ -517,12 +523,73 @@ func newRemotePluginConfig( remoteHost: remoteHost, revision: revision, out: out, - opt: opt, + opts: opt, includeImports: includeImports, includeWKT: includeWKT, }, nil } +func newExternalGeneratePluginConfigV2FromPluginConfig( + generatePluginConfig GeneratePluginConfig, +) (externalGeneratePluginConfigV2, error) { + pluginConfig, ok := generatePluginConfig.(*pluginConfig) + if !ok { + return externalGeneratePluginConfigV2{}, syserror.Newf("unknown implementation of GeneratePluginConfig: %T", generatePluginConfig) + } + externalPluginConfigV2 := externalGeneratePluginConfigV2{ + Out: generatePluginConfig.Out(), + IncludeImports: generatePluginConfig.IncludeImports(), + IncludeWKT: generatePluginConfig.IncludeWKT(), + } + opts := pluginConfig.opts + switch { + case len(opts) == 1: + externalPluginConfigV2.Opt = opts[0] + case len(opts) > 1: + externalPluginConfigV2.Opt = opts + } + strategy := pluginConfig.strategy + switch { + case strategy != nil && *strategy == GenerateStrategyDirectory: + externalPluginConfigV2.Strategy = toPointer("directory") + case strategy != nil && *strategy == GenerateStrategyAll: + externalPluginConfigV2.Strategy = toPointer("all") + } + switch generatePluginConfig.Type() { + case PluginConfigTypeRemote: + externalPluginConfigV2.Remote = toPointer(generatePluginConfig.Name()) + if revision := generatePluginConfig.Revision(); revision != 0 { + externalPluginConfigV2.Revision = &revision + } + case PluginConfigTypeBinary: + path := generatePluginConfig.Path() + switch { + case len(path) == 1: + externalPluginConfigV2.Binary = path[0] + case len(path) > 1: + externalPluginConfigV2.Binary = path + } + case PluginConfigTypeProtocBuiltin: + externalPluginConfigV2.ProtocBuiltin = toPointer(generatePluginConfig.Name()) + if protocPath := generatePluginConfig.ProtocPath(); protocPath != "" { + externalPluginConfigV2.ProtocPath = &protocPath + } + case PluginConfigTypeLocal: + binaryName := "protoc-gen-" + generatePluginConfig.Name() + _, err := exec.LookPath(binaryName) + if err == nil || errors.Is(err, exec.ErrDot) { + externalPluginConfigV2.Binary = binaryName + break + } + if _, isProtocBuiltin := bufpluginexec.ProtocProxyPluginNames[generatePluginConfig.Name()]; isProtocBuiltin { + externalPluginConfigV2.ProtocBuiltin = toPointer(generatePluginConfig.Name()) + break + } + return externalGeneratePluginConfigV2{}, fmt.Errorf("plugin %s is not found locally and %s is not built-in to protoc", binaryName, generatePluginConfig.Name()) + } + return externalPluginConfigV2, nil +} + func parseRemoteHostName(fullName string) (string, error) { if identity, err := bufpluginref.PluginIdentityForString(fullName); err == nil { return identity.Remote(), nil @@ -546,3 +613,7 @@ func checkPathAndStrategyUnset(plugin externalGeneratePluginConfigV1, pluginIden } return nil } + +func toPointer[T any](value T) *T { + return &value +} diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 82fec799cf..6586559e49 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -19,6 +19,8 @@ import ( "errors" "fmt" "strconv" + + "github.com/bufbuild/buf/private/pkg/syserror" ) // TODO: input type? @@ -423,6 +425,63 @@ func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV return inputConfig, nil } +func newExternalInputConfigV2FromInputConfig( + inputConfig InputConfig, +) (externalInputConfigV2, error) { + externalInputConfigV2 := externalInputConfigV2{} + switch inputConfig.Type() { + case InputConfigTypeGitRepo: + externalInputConfigV2.GitRepo = toPointer(inputConfig.Location()) + case InputConfigTypeDirectory: + externalInputConfigV2.Directory = toPointer(inputConfig.Location()) + case InputConfigTypeModule: + externalInputConfigV2.Module = toPointer(inputConfig.Location()) + case InputConfigTypeProtoFile: + externalInputConfigV2.ProtoFile = toPointer(inputConfig.Location()) + case InputConfigTypeZipArchive: + externalInputConfigV2.ZipArchive = toPointer(inputConfig.Location()) + case InputConfigTypeTarball: + externalInputConfigV2.Tarball = toPointer(inputConfig.Location()) + case InputConfigTypeBinaryImage: + externalInputConfigV2.BinaryImage = toPointer(inputConfig.Location()) + case InputConfigTypeJSONImage: + externalInputConfigV2.JSONImage = toPointer(inputConfig.Location()) + case InputConfigTypeTextImage: + externalInputConfigV2.TextImage = toPointer(inputConfig.Location()) + default: + return externalInputConfigV2, syserror.Newf("unknown input config type: %v", inputConfig.Type()) + } + if inputConfig.Branch() != "" { + externalInputConfigV2.Branch = toPointer(inputConfig.Branch()) + } + if inputConfig.Ref() != "" { + externalInputConfigV2.Ref = toPointer(inputConfig.Ref()) + } + if inputConfig.Tag() != "" { + externalInputConfigV2.Tag = toPointer(inputConfig.Tag()) + } + externalInputConfigV2.Depth = inputConfig.Depth() + // TODO: make RecurseSubmodules return a pointer for more accurate representation + if inputConfig.RecurseSubmodules() { + externalInputConfigV2.RecurseSubmodules = toPointer(inputConfig.RecurseSubmodules()) + } + if inputConfig.Compression() != "" { + externalInputConfigV2.Compression = toPointer(inputConfig.Compression()) + } + externalInputConfigV2.StripComponents = inputConfig.StripComponents() + if inputConfig.SubDir() != "" { + externalInputConfigV2.Subdir = toPointer(inputConfig.SubDir()) + } + // TODO: make IncludePackageFiles return a pointer for more accurate representation + if inputConfig.IncludePackageFiles() { + externalInputConfigV2.IncludePackageFiles = toPointer(inputConfig.IncludePackageFiles()) + } + externalInputConfigV2.IncludePaths = inputConfig.IncludePaths() + externalInputConfigV2.ExcludePaths = inputConfig.ExcludePaths() + externalInputConfigV2.Types = inputConfig.IncludeTypes() + return externalInputConfigV2, nil +} + func (i *inputConfig) Type() InputConfigType { return i.inputType } From cda47af058a3e53232b31a0bbfde13dc8dc07c6d Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 14:52:42 -0500 Subject: [PATCH 42/66] dont migrate if v2 --- private/buf/cmd/buf/command/generate/generate.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 46b35f2b3a..0e2549b525 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -343,8 +343,7 @@ func run( if err != nil { return err } - // TODO: Call a bufconfig.MigrateBufGenYAML instead - if flags.Migrate { + if flags.Migrate && bufGenYAMLFile.FileVersion() != bufconfig.FileVersionV2 { if err := bufconfig.PutBufGenYAMLFileForPrefix(ctx, bucket, ".", bufGenYAMLFile); err != nil { return err } @@ -359,8 +358,7 @@ func run( if err != nil { return err } - // TODO: Call a bufconfig.MigrateBufGenYAML instead - if flags.Migrate { + if flags.Migrate && bufGenYAMLFile.FileVersion() != bufconfig.FileVersionV2 { if err := bufconfig.WriteBufGenYAMLFile(configFile, bufGenYAMLFile); err != nil { return err } @@ -370,8 +368,7 @@ func run( if err != nil { return err } - // TODO: Call a bufconfig.MigrateBufGenYAML instead - if flags.Migrate { + if flags.Migrate && bufGenYAMLFile.FileVersion() != bufconfig.FileVersionV2 { return fmt.Errorf( "invalid template: %q, migration can only apply to a file on disk with extension .yaml, .yml or .json", flags.Template, From 8ecef918584891649a96dbc8e97739fafb70d729 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 15:25:10 -0500 Subject: [PATCH 43/66] parse v1beta1 --- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 15 +++++- private/bufpkg/bufconfig/generate_config.go | 23 +++++++++ .../bufconfig/generate_managed_config.go | 50 ++++++++++++++++++- .../bufconfig/generate_plugin_config.go | 39 +++++++++++++++ 4 files changed, 124 insertions(+), 3 deletions(-) diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 662cab5e67..7e496d3705 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -16,7 +16,6 @@ package bufconfig import ( "context" - "errors" "fmt" "io" @@ -135,7 +134,19 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error } switch fileVersion { case FileVersionV1Beta1: - return nil, errors.New("TODO") + var externalGenYAMLFile externalBufGenYAMLV1Beta1 + if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { + return nil, fmt.Errorf("invalid as version %v: %w", fileVersion, err) + } + generateConfig, err := newGenerateConfigFromExternalFileV1Beta1(externalGenYAMLFile) + if err != nil { + return nil, err + } + return newBufGenYAMLFile( + fileVersion, + generateConfig, + nil, + ), nil case FileVersionV1: var externalGenYAMLFile externalBufGenYAMLFileV1 if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index ebba7b0e50..bfd66d8a70 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -36,6 +36,29 @@ type GenerateConfig interface { isGenerateConfig() } +func newGenerateConfigFromExternalFileV1Beta1( + externalFile externalBufGenYAMLV1Beta1, +) (GenerateConfig, error) { + managedConfig, err := newManagedConfigFromExternalV1Beta1(externalFile.Options) + if err != nil { + return nil, err + } + if len(externalFile.Plugins) == 0 { + return nil, errors.New("must specifiy at least one plugin") + } + pluginConfigs, err := slicesext.MapError( + externalFile.Plugins, + newPluginConfigFromExternalV1Beta1, + ) + if err != nil { + return nil, err + } + return &generateConfig{ + pluginConfigs: pluginConfigs, + managedConfig: managedConfig, + }, nil +} + // TODO: check if this is consistent with the rest of bufconfig (esp. its name and whether it should exist) func newGenerateConfigFromExternalFileV1( externalFile externalBufGenYAMLFileV1, diff --git a/private/bufpkg/bufconfig/generate_managed_config.go b/private/bufpkg/bufconfig/generate_managed_config.go index 1bdcc05d44..bdc370c96a 100644 --- a/private/bufpkg/bufconfig/generate_managed_config.go +++ b/private/bufpkg/bufconfig/generate_managed_config.go @@ -412,10 +412,58 @@ func newManagedConfigFromExternalV1( }, nil } +func newManagedConfigFromExternalV1Beta1( + externalConfig externalOptionsConfigV1Beta1, +) (GenerateManagedConfig, error) { + var ( + overrides []ManagedOverrideRule + ) + if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { + override, err := NewFileOptionOverrideRule( + "", + "", + FileOptionCcEnableArenas, + *externalCCEnableArenas, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { + override, err := NewFileOptionOverrideRule( + "", + "", + FileOptionJavaMultipleFiles, + *externalJavaMultipleFiles, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) + } + if externalOptimizeFor := externalConfig.OptimizeFor; externalOptimizeFor != "" { + defaultOverride, err := NewFileOptionOverrideRule( + "", + "", + FileOptionOptimizeFor, + externalOptimizeFor, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, defaultOverride) + } + return &generateManagedConfig{ + overrides: overrides, + }, nil +} + func newManagedConfigFromExternalV2( externalConfig externalGenerateManagedConfigV2, ) (GenerateManagedConfig, error) { - if externalConfig.isEmpty() { + // TODO: add test case for non-empty config but disabled + if !externalConfig.Enabled { return nil, nil } // TODO: log warning if disabled but non-empty diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index 812f8bdb32..6354b38935 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -190,6 +190,45 @@ func (p *pluginConfig) Revision() int { func (p *pluginConfig) isGeneratePluginConfig() {} +// TODO: compare with the old implementation +func newPluginConfigFromExternalV1Beta1( + externalConfig externalGeneratePluginConfigV1Beta1, +) (GeneratePluginConfig, error) { + if externalConfig.Name == "" { + return nil, errors.New("plugin name is required") + } + if externalConfig.Out == "" { + return nil, fmt.Errorf("out is required for plugin %s", externalConfig.Name) + } + strategy, err := parseStrategy(externalConfig.Strategy) + if err != nil { + return nil, err + } + opt, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Opt) + if err != nil { + return nil, err + } + if externalConfig.Path != "" { + return newBinaryPluginConfig( + externalConfig.Name, + []string{externalConfig.Path}, + strategy, + externalConfig.Out, + opt, + false, + false, + ) + } + return newLocalPluginConfig( + externalConfig.Name, + strategy, + externalConfig.Out, + opt, + false, + false, + ) +} + // TODO: figure out where is the best place to do parameter validation, here or in new*plugin. func newPluginConfigFromExternalV1( externalConfig externalGeneratePluginConfigV1, From fd43bd788c02bbfa41bd707ae05dbb9f6f2e30fb Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 17:26:48 -0500 Subject: [PATCH 44/66] deleting unused code in bufgen --- private/buf/bufgen/bufgen.go | 431 ---------- private/buf/bufgen/bufgen_test.go | 33 - private/buf/bufgen/config.go | 666 ---------------- private/buf/bufgen/config_test.go | 746 ------------------ private/buf/bufgen/provider.go | 72 -- private/buf/bufgen/provider_test.go | 76 -- .../cmd/buf/command/generate/generate_test.go | 20 +- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 14 + private/bufpkg/bufconfig/generate_config.go | 33 +- 9 files changed, 49 insertions(+), 2042 deletions(-) delete mode 100644 private/buf/bufgen/bufgen_test.go delete mode 100644 private/buf/bufgen/config.go delete mode 100644 private/buf/bufgen/config_test.go delete mode 100644 private/buf/bufgen/provider.go delete mode 100644 private/buf/bufgen/provider_test.go diff --git a/private/buf/bufgen/bufgen.go b/private/buf/bufgen/bufgen.go index 0f64b70d09..9741d40ab3 100644 --- a/private/buf/bufgen/bufgen.go +++ b/private/buf/bufgen/bufgen.go @@ -19,23 +19,17 @@ package bufgen import ( "context" - "encoding/json" "fmt" "strconv" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" - "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" "github.com/bufbuild/buf/private/bufpkg/bufwasm" "github.com/bufbuild/buf/private/pkg/app" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/connectclient" - "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/storage/storageos" "go.uber.org/zap" - "google.golang.org/protobuf/types/descriptorpb" ) const ( @@ -85,19 +79,6 @@ func (s Strategy) String() string { } } -// Provider is a provider. -type Provider interface { - // GetConfig gets the Config for the YAML data at ExternalConfigFilePath. - // - // If the data is of length 0, returns the default config. - GetConfig(ctx context.Context, readBucket storage.ReadBucket) (*Config, error) -} - -// NewProvider returns a new Provider. -func NewProvider(logger *zap.Logger) Provider { - return newProvider(logger) -} - // Generator generates Protobuf stubs based on configurations. type Generator interface { // Generate calls the generation logic. @@ -171,415 +152,3 @@ func GenerateWithWASMEnabled() GenerateOption { generateOptions.wasmEnabled = true } } - -// Config is a configuration. -type Config struct { - // Required - PluginConfigs []*PluginConfig - // Optional - ManagedConfig *ManagedConfig - // Optional - TypesConfig *TypesConfig -} - -// PluginConfig is a plugin configuration. -type PluginConfig struct { - // One of Plugin, Name or Remote is required - Plugin string - Name string - Remote string - // Optional, used with Plugin to pin a specific revision - Revision int - // Required - Out string - // Optional - Opt string - // Optional, exclusive with Remote - Path []string - // Required - Strategy Strategy - // Optional - ProtocPath string -} - -// PluginName returns this PluginConfig's plugin name. -// Only one of Plugin, Name or Remote will be set. -func (p *PluginConfig) PluginName() string { - if p == nil { - return "" - } - if p.Plugin != "" { - return p.Plugin - } - if p.Name != "" { - return p.Name - } - if p.Remote != "" { - return p.Remote - } - return "" -} - -// IsRemote returns true if the PluginConfig uses a remotely executed plugin. -func (p *PluginConfig) IsRemote() bool { - return p.GetRemoteHostname() != "" -} - -// GetRemoteHostname returns the hostname of the remote plugin. -func (p *PluginConfig) GetRemoteHostname() string { - if p == nil { - return "" - } - if identity, err := bufpluginref.PluginIdentityForString(p.Plugin); err == nil { - return identity.Remote() - } - if reference, err := bufpluginref.PluginReferenceForString(p.Plugin, 0); err == nil { - return reference.Remote() - } - if p.Remote == "" { - return "" - } - if remote, _, _, _, err := bufremoteplugin.ParsePluginVersionPath(p.Remote); err == nil { - return remote - } - return "" -} - -// ManagedConfig is the managed mode configuration. -type ManagedConfig struct { - CcEnableArenas *bool - JavaMultipleFiles *bool - JavaStringCheckUtf8 *bool - JavaPackagePrefixConfig *JavaPackagePrefixConfig - CsharpNameSpaceConfig *CsharpNameSpaceConfig - OptimizeForConfig *OptimizeForConfig - GoPackagePrefixConfig *GoPackagePrefixConfig - ObjcClassPrefixConfig *ObjcClassPrefixConfig - RubyPackageConfig *RubyPackageConfig - Override map[string]map[string]string -} - -// JavaPackagePrefixConfig is the java_package prefix configuration. -type JavaPackagePrefixConfig struct { - Default string - Except []bufmodule.ModuleFullName - // bufmodule.ModuleFullName -> java_package prefix. - Override map[bufmodule.ModuleFullName]string -} - -type OptimizeForConfig struct { - Default descriptorpb.FileOptions_OptimizeMode - Except []bufmodule.ModuleFullName - // bufmodule.ModuleFullName -> optimize_for. - Override map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode -} - -// GoPackagePrefixConfig is the go_package prefix configuration. -type GoPackagePrefixConfig struct { - Default string - Except []bufmodule.ModuleFullName - // bufmodule.ModuleFullName -> go_package prefix. - Override map[bufmodule.ModuleFullName]string -} - -// ObjcClassPrefixConfig is the objc_class_prefix configuration. -type ObjcClassPrefixConfig struct { - Default string - Except []bufmodule.ModuleFullName - // bufmodule.ModuleFullName -> objc_class_prefix. - Override map[bufmodule.ModuleFullName]string -} - -// RubyPackgeConfig is the ruby_package configuration. -type RubyPackageConfig struct { - Except []bufmodule.ModuleFullName - // bufmodule.ModuleFullName -> ruby_package. - Override map[bufmodule.ModuleFullName]string -} - -// CsharpNameSpaceConfig is the csharp_namespace configuration. -type CsharpNameSpaceConfig struct { - Except []bufmodule.ModuleFullName - // bufmodule.ModuleFullName -> csharp_namespace prefix. - Override map[bufmodule.ModuleFullName]string -} - -// TypesConfig is a types configuration -type TypesConfig struct { - Include []string -} - -// ReadConfig reads the configuration from the OS or an override, if any. -// -// Only use in CLI tools. -func ReadConfig( - ctx context.Context, - logger *zap.Logger, - provider Provider, - readBucket storage.ReadBucket, - options ...ReadConfigOption, -) (*Config, error) { - return readConfig( - ctx, - logger, - provider, - readBucket, - options..., - ) -} - -// ReadConfigOption is an option for ReadConfig. -type ReadConfigOption func(*readConfigOptions) - -// ReadConfigWithOverride sets the override. -// -// If override is set, this will first check if the override ends in .json or .yaml, if so, -// this reads the file at this path and uses it. Otherwise, this assumes this is configuration -// data in either JSON or YAML format, and unmarshals it. -// -// If no override is set, this reads ExternalConfigFilePath in the bucket. -func ReadConfigWithOverride(override string) ReadConfigOption { - return func(readConfigOptions *readConfigOptions) { - readConfigOptions.override = override - } -} - -// ConfigExists checks if a generation configuration file exists. -func ConfigExists(ctx context.Context, readBucket storage.ReadBucket) (bool, error) { - return storage.Exists(ctx, readBucket, ExternalConfigFilePath) -} - -// ExternalConfigV1 is an external configuration. -type ExternalConfigV1 struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` - Plugins []ExternalPluginConfigV1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` - Managed ExternalManagedConfigV1 `json:"managed,omitempty" yaml:"managed,omitempty"` - Types ExternalTypesConfigV1 `json:"types,omitempty" yaml:"types,omitempty"` -} - -// ExternalPluginConfigV1 is an external plugin configuration. -type ExternalPluginConfigV1 struct { - Plugin string `json:"plugin,omitempty" yaml:"plugin,omitempty"` - Revision int `json:"revision,omitempty" yaml:"revision,omitempty"` - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Remote string `json:"remote,omitempty" yaml:"remote,omitempty"` - Out string `json:"out,omitempty" yaml:"out,omitempty"` - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` - Path interface{} `json:"path,omitempty" yaml:"path,omitempty"` - ProtocPath string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` - Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` -} - -// ExternalManagedConfigV1 is an external managed mode configuration. -// -// Only use outside of this package for testing. -type ExternalManagedConfigV1 struct { - Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` - CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` - JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` - JavaStringCheckUtf8 *bool `json:"java_string_check_utf8,omitempty" yaml:"java_string_check_utf8,omitempty"` - JavaPackagePrefix ExternalJavaPackagePrefixConfigV1 `json:"java_package_prefix,omitempty" yaml:"java_package_prefix,omitempty"` - CsharpNamespace ExternalCsharpNamespaceConfigV1 `json:"csharp_namespace,omitempty" yaml:"csharp_namespace,omitempty"` - OptimizeFor ExternalOptimizeForConfigV1 `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` - GoPackagePrefix ExternalGoPackagePrefixConfigV1 `json:"go_package_prefix,omitempty" yaml:"go_package_prefix,omitempty"` - ObjcClassPrefix ExternalObjcClassPrefixConfigV1 `json:"objc_class_prefix,omitempty" yaml:"objc_class_prefix,omitempty"` - RubyPackage ExternalRubyPackageConfigV1 `json:"ruby_package,omitempty" yaml:"ruby_package,omitempty"` - Override map[string]map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// IsEmpty returns true if the config is empty, excluding the 'Enabled' setting. -func (e ExternalManagedConfigV1) IsEmpty() bool { - return e.CcEnableArenas == nil && - e.JavaMultipleFiles == nil && - e.JavaStringCheckUtf8 == nil && - e.JavaPackagePrefix.IsEmpty() && - e.CsharpNamespace.IsEmpty() && - e.CsharpNamespace.IsEmpty() && - e.OptimizeFor.IsEmpty() && - e.GoPackagePrefix.IsEmpty() && - e.ObjcClassPrefix.IsEmpty() && - e.RubyPackage.IsEmpty() && - len(e.Override) == 0 -} - -// ExternalJavaPackagePrefixConfigV1 is the external java_package prefix configuration. -type ExternalJavaPackagePrefixConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// IsEmpty returns true if the config is empty. -func (e ExternalJavaPackagePrefixConfigV1) IsEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// UnmarshalYAML satisfies the yaml.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for java_package_prefix. -func (e *ExternalJavaPackagePrefixConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { - return e.unmarshalWith(unmarshal) -} - -// UnmarshalJSON satisfies the json.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for java_package_prefix. -func (e *ExternalJavaPackagePrefixConfigV1) UnmarshalJSON(data []byte) error { - unmarshal := func(v interface{}) error { - return json.Unmarshal(data, v) - } - - return e.unmarshalWith(unmarshal) -} - -// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. -func (e *ExternalJavaPackagePrefixConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { - var prefix string - if err := unmarshal(&prefix); err == nil { - e.Default = prefix - return nil - } - - type rawExternalJavaPackagePrefixConfigV1 ExternalJavaPackagePrefixConfigV1 - if err := unmarshal((*rawExternalJavaPackagePrefixConfigV1)(e)); err != nil { - return err - } - - return nil -} - -// ExternalOptimizeForConfigV1 is the external optimize_for configuration. -type ExternalOptimizeForConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// IsEmpty returns true if the config is empty -func (e ExternalOptimizeForConfigV1) IsEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// UnmarshalYAML satisfies the yaml.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for optimize_for. -func (e *ExternalOptimizeForConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { - return e.unmarshalWith(unmarshal) -} - -// UnmarshalJSON satisfies the json.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for optimize_for. -func (e *ExternalOptimizeForConfigV1) UnmarshalJSON(data []byte) error { - unmarshal := func(v interface{}) error { - return json.Unmarshal(data, v) - } - - return e.unmarshalWith(unmarshal) -} - -// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. -func (e *ExternalOptimizeForConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { - var optimizeFor string - if err := unmarshal(&optimizeFor); err == nil { - e.Default = optimizeFor - return nil - } - - type rawExternalOptimizeForConfigV1 ExternalOptimizeForConfigV1 - if err := unmarshal((*rawExternalOptimizeForConfigV1)(e)); err != nil { - return err - } - - return nil -} - -// ExternalGoPackagePrefixConfigV1 is the external go_package prefix configuration. -type ExternalGoPackagePrefixConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// IsEmpty returns true if the config is empty. -func (e ExternalGoPackagePrefixConfigV1) IsEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// ExternalCsharpNamespaceConfigV1 is the external csharp_namespace configuration. -type ExternalCsharpNamespaceConfigV1 struct { - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// IsEmpty returns true if the config is empty. -func (e ExternalCsharpNamespaceConfigV1) IsEmpty() bool { - return len(e.Except) == 0 && - len(e.Override) == 0 -} - -// ExternalRubyPackageConfigV1 is the external ruby_package configuration -type ExternalRubyPackageConfigV1 struct { - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// IsEmpty returns true is the config is empty -func (e ExternalRubyPackageConfigV1) IsEmpty() bool { - return len(e.Except) == 0 && len(e.Override) == 0 -} - -// ExternalObjcClassPrefixConfigV1 is the external objc_class_prefix configuration. -type ExternalObjcClassPrefixConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -func (e ExternalObjcClassPrefixConfigV1) IsEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// ExternalConfigV1Beta1 is an external configuration. -type ExternalConfigV1Beta1 struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` - Managed bool `json:"managed,omitempty" yaml:"managed,omitempty"` - Plugins []ExternalPluginConfigV1Beta1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` - Options ExternalOptionsConfigV1Beta1 `json:"options,omitempty" yaml:"options,omitempty"` -} - -// ExternalPluginConfigV1Beta1 is an external plugin configuration. -type ExternalPluginConfigV1Beta1 struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Out string `json:"out,omitempty" yaml:"out,omitempty"` - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` - Path string `json:"path,omitempty" yaml:"path,omitempty"` - Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` -} - -// ExternalOptionsConfigV1Beta1 is an external options configuration. -type ExternalOptionsConfigV1Beta1 struct { - CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` - JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` - OptimizeFor string `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` -} - -// ExternalConfigVersion defines the subset of all config -// file versions that is used to determine the configuration version. -type ExternalConfigVersion struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` -} - -// ExternalTypesConfigV1 is an external types configuration. -type ExternalTypesConfigV1 struct { - Include []string `json:"include,omitempty" yaml:"include"` -} - -// IsEmpty returns true if e is empty. -func (e ExternalTypesConfigV1) IsEmpty() bool { - return len(e.Include) == 0 -} diff --git a/private/buf/bufgen/bufgen_test.go b/private/buf/bufgen/bufgen_test.go deleted file mode 100644 index a135aeb25e..0000000000 --- a/private/buf/bufgen/bufgen_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufgen - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPluginConfig_GetRemoteHostname(t *testing.T) { - t.Parallel() - assertPluginConfigRemoteHostname := func(config *PluginConfig, expected string) { - t.Helper() - assert.Equal(t, config.GetRemoteHostname(), expected) - } - assertPluginConfigRemoteHostname(&PluginConfig{Plugin: "buf.build/protocolbuffers/go:v1.28.1"}, "buf.build") - assertPluginConfigRemoteHostname(&PluginConfig{Plugin: "buf.build/protocolbuffers/go"}, "buf.build") - assertPluginConfigRemoteHostname(&PluginConfig{Remote: "buf.build/protocolbuffers/plugins/go:v1.28.1-1"}, "buf.build") - assertPluginConfigRemoteHostname(&PluginConfig{Remote: "buf.build/protocolbuffers/plugins/go"}, "buf.build") -} diff --git a/private/buf/bufgen/config.go b/private/buf/bufgen/config.go deleted file mode 100644 index 541ed442de..0000000000 --- a/private/buf/bufgen/config.go +++ /dev/null @@ -1,666 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufgen - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" - "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" - "github.com/bufbuild/buf/private/pkg/encoding" - "github.com/bufbuild/buf/private/pkg/normalpath" - "github.com/bufbuild/buf/private/pkg/storage" - "go.uber.org/zap" - "google.golang.org/protobuf/types/descriptorpb" -) - -func readConfig( - ctx context.Context, - logger *zap.Logger, - provider Provider, - readBucket storage.ReadBucket, - options ...ReadConfigOption, -) (*Config, error) { - readConfigOptions := newReadConfigOptions() - for _, option := range options { - option(readConfigOptions) - } - if override := readConfigOptions.override; override != "" { - switch filepath.Ext(override) { - case ".json": - return getConfigJSONFile(logger, override) - case ".yaml", ".yml": - return getConfigYAMLFile(logger, override) - default: - return getConfigJSONOrYAMLData(logger, override) - } - } - return provider.GetConfig(ctx, readBucket) -} - -func getConfigJSONFile(logger *zap.Logger, file string) (*Config, error) { - data, err := os.ReadFile(file) - if err != nil { - return nil, fmt.Errorf("could not read file %s: %v", file, err) - } - return getConfig( - logger, - encoding.UnmarshalJSONNonStrict, - encoding.UnmarshalJSONStrict, - data, - file, - ) -} - -func getConfigYAMLFile(logger *zap.Logger, file string) (*Config, error) { - data, err := os.ReadFile(file) - if err != nil { - return nil, fmt.Errorf("could not read file %s: %v", file, err) - } - return getConfig( - logger, - encoding.UnmarshalYAMLNonStrict, - encoding.UnmarshalYAMLStrict, - data, - file, - ) -} - -func getConfigJSONOrYAMLData(logger *zap.Logger, data string) (*Config, error) { - return getConfig( - logger, - encoding.UnmarshalJSONOrYAMLNonStrict, - encoding.UnmarshalJSONOrYAMLStrict, - []byte(data), - "Generate configuration data", - ) -} - -func getConfig( - logger *zap.Logger, - unmarshalNonStrict func([]byte, interface{}) error, - unmarshalStrict func([]byte, interface{}) error, - data []byte, - id string, -) (*Config, error) { - var externalConfigVersion ExternalConfigVersion - if err := unmarshalNonStrict(data, &externalConfigVersion); err != nil { - return nil, err - } - switch externalConfigVersion.Version { - case V1Beta1Version: - var externalConfigV1Beta1 ExternalConfigV1Beta1 - if err := unmarshalStrict(data, &externalConfigV1Beta1); err != nil { - return nil, err - } - if err := validateExternalConfigV1Beta1(externalConfigV1Beta1, id); err != nil { - return nil, err - } - return newConfigV1Beta1(externalConfigV1Beta1, id) - case V1Version: - var externalConfigV1 ExternalConfigV1 - if err := unmarshalStrict(data, &externalConfigV1); err != nil { - return nil, err - } - if err := validateExternalConfigV1(externalConfigV1, id); err != nil { - return nil, err - } - return newConfigV1(logger, externalConfigV1, id) - default: - return nil, fmt.Errorf(`%s has no version set. Please add "version: %s"`, id, V1Version) - } -} - -func newConfigV1(logger *zap.Logger, externalConfig ExternalConfigV1, id string) (*Config, error) { - managedConfig, err := newManagedConfigV1(logger, externalConfig.Managed) - if err != nil { - return nil, err - } - pluginConfigs := make([]*PluginConfig, 0, len(externalConfig.Plugins)) - for _, plugin := range externalConfig.Plugins { - strategy, err := ParseStrategy(plugin.Strategy) - if err != nil { - return nil, err - } - opt, err := encoding.InterfaceSliceOrStringToCommaSepString(plugin.Opt) - if err != nil { - return nil, err - } - path, err := encoding.InterfaceSliceOrStringToStringSlice(plugin.Path) - if err != nil { - return nil, err - } - pluginConfig := &PluginConfig{ - Plugin: plugin.Plugin, - Revision: plugin.Revision, - Name: plugin.Name, - Remote: plugin.Remote, - Out: plugin.Out, - Opt: opt, - Path: path, - ProtocPath: plugin.ProtocPath, - Strategy: strategy, - } - if pluginConfig.IsRemote() { - // Always use StrategyAll for remote plugins - pluginConfig.Strategy = StrategyAll - } - pluginConfigs = append(pluginConfigs, pluginConfig) - } - typesConfig := newTypesConfigV1(externalConfig.Types) - return &Config{ - PluginConfigs: pluginConfigs, - ManagedConfig: managedConfig, - TypesConfig: typesConfig, - }, nil -} - -func validateExternalConfigV1(externalConfig ExternalConfigV1, id string) error { - if len(externalConfig.Plugins) == 0 { - return fmt.Errorf("%s: no plugins set", id) - } - for _, plugin := range externalConfig.Plugins { - var numPluginIdentifiers int - var pluginIdentifier string - for _, possibleIdentifier := range []string{plugin.Plugin, plugin.Name, plugin.Remote} { - if possibleIdentifier != "" { - numPluginIdentifiers++ - // Doesn't matter if we reassign here - we only allow one to be set below - pluginIdentifier = possibleIdentifier - } - } - if numPluginIdentifiers == 0 { - return fmt.Errorf("%s: one of plugin, name or remote is required", id) - } - if numPluginIdentifiers > 1 { - return fmt.Errorf("%s: only one of plugin, name, or remote can be set", id) - } - if plugin.Out == "" { - return fmt.Errorf("%s: plugin %s out is required", id, pluginIdentifier) - } - switch { - case plugin.Plugin != "": - if bufpluginref.IsPluginReferenceOrIdentity(pluginIdentifier) { - // plugin.Plugin is a remote plugin - if err := checkPathAndStrategyUnset(id, plugin, pluginIdentifier); err != nil { - return err - } - } else { - // plugin.Plugin is a local plugin - verify it isn't using an alpha remote plugin path - if _, _, _, _, err := bufremoteplugin.ParsePluginVersionPath(pluginIdentifier); err == nil { - return fmt.Errorf("%s: invalid plugin reference: %s", id, pluginIdentifier) - } - } - case plugin.Remote != "": - // Remote generation alpha features have been deprecated, but we continue to detect - // the remote field to surface a better error message. - return fmt.Errorf("%s: the remote field no longer works as the remote generation alpha has been deprecated, see the migration guide to now-stable remote plugins: %s", - id, - "https://buf.build/docs/migration-guides/migrate-remote-generation-alpha/#migrate-to-remote-plugins", - ) - case plugin.Name != "": - // Check that the plugin name doesn't look like a plugin reference - if bufpluginref.IsPluginReferenceOrIdentity(pluginIdentifier) { - return fmt.Errorf("%s: invalid local plugin name: %s", id, pluginIdentifier) - } - // Check that the plugin name doesn't look like an alpha remote plugin - if _, _, _, _, err := bufremoteplugin.ParsePluginVersionPath(pluginIdentifier); err == nil { - return fmt.Errorf("%s: invalid plugin name %s, did you mean to use a remote plugin?", id, pluginIdentifier) - } - default: - // unreachable - validated above - return errors.New("one of plugin, name, or remote is required") - } - } - return nil -} - -func checkPathAndStrategyUnset(id string, plugin ExternalPluginConfigV1, pluginIdentifier string) error { - if plugin.Path != nil { - return fmt.Errorf("%s: remote plugin %s cannot specify a path", id, pluginIdentifier) - } - if plugin.Strategy != "" { - return fmt.Errorf("%s: remote plugin %s cannot specify a strategy", id, pluginIdentifier) - } - if plugin.ProtocPath != "" { - return fmt.Errorf("%s: remote plugin %s cannot specify a protoc path", id, pluginIdentifier) - } - return nil -} - -func newManagedConfigV1(logger *zap.Logger, externalManagedConfig ExternalManagedConfigV1) (*ManagedConfig, error) { - if !externalManagedConfig.Enabled { - if !externalManagedConfig.IsEmpty() && logger != nil { - logger.Sugar().Warn("managed mode options are set but are not enabled") - } - return nil, nil - } - javaPackagePrefixConfig, err := newJavaPackagePrefixConfigV1(externalManagedConfig.JavaPackagePrefix) - if err != nil { - return nil, err - } - csharpNamespaceConfig, err := newCsharpNamespaceConfigV1(externalManagedConfig.CsharpNamespace) - if err != nil { - return nil, err - } - optimizeForConfig, err := newOptimizeForConfigV1(externalManagedConfig.OptimizeFor) - if err != nil { - return nil, err - } - goPackagePrefixConfig, err := newGoPackagePrefixConfigV1(externalManagedConfig.GoPackagePrefix) - if err != nil { - return nil, err - } - objcClassPrefixConfig, err := newObjcClassPrefixConfigV1(externalManagedConfig.ObjcClassPrefix) - if err != nil { - return nil, err - } - rubyPackageConfig, err := newRubyPackageConfigV1(externalManagedConfig.RubyPackage) - if err != nil { - return nil, err - } - override := externalManagedConfig.Override - for overrideID, overrideValue := range override { - for importPath := range overrideValue { - normalizedImportPath, err := normalpath.NormalizeAndValidate(importPath) - if err != nil { - return nil, fmt.Errorf( - "failed to normalize import path: %s provided for override: %s", - importPath, - overrideID, - ) - } - if importPath != normalizedImportPath { - return nil, fmt.Errorf( - "override can only take normalized import paths, invalid import path: %s provided for override: %s", - importPath, - overrideID, - ) - } - } - } - return &ManagedConfig{ - CcEnableArenas: externalManagedConfig.CcEnableArenas, - JavaMultipleFiles: externalManagedConfig.JavaMultipleFiles, - JavaStringCheckUtf8: externalManagedConfig.JavaStringCheckUtf8, - JavaPackagePrefixConfig: javaPackagePrefixConfig, - CsharpNameSpaceConfig: csharpNamespaceConfig, - OptimizeForConfig: optimizeForConfig, - GoPackagePrefixConfig: goPackagePrefixConfig, - ObjcClassPrefixConfig: objcClassPrefixConfig, - RubyPackageConfig: rubyPackageConfig, - Override: override, - }, nil -} - -func newJavaPackagePrefixConfigV1(externalJavaPackagePrefixConfig ExternalJavaPackagePrefixConfigV1) (*JavaPackagePrefixConfig, error) { - if externalJavaPackagePrefixConfig.IsEmpty() { - return nil, nil - } - if externalJavaPackagePrefixConfig.Default == "" { - return nil, errors.New("java_package_prefix setting requires a default value") - } - seenModuleFullNames := make(map[string]struct{}, len(externalJavaPackagePrefixConfig.Except)) - except := make([]bufmodule.ModuleFullName, 0, len(externalJavaPackagePrefixConfig.Except)) - for _, moduleName := range externalJavaPackagePrefixConfig.Except { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid java_package_prefix except: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid java_package_prefix except: %q is defined multiple times", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - except = append(except, moduleFullName) - } - override := make(map[bufmodule.ModuleFullName]string, len(externalJavaPackagePrefixConfig.Override)) - for moduleName, javaPackagePrefix := range externalJavaPackagePrefixConfig.Override { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid java_package_prefix override key: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid java_package_prefix override: %q is already defined as an except", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - override[moduleFullName] = javaPackagePrefix - } - return &JavaPackagePrefixConfig{ - Default: externalJavaPackagePrefixConfig.Default, - Except: except, - Override: override, - }, nil -} - -func newOptimizeForConfigV1(externalOptimizeForConfigV1 ExternalOptimizeForConfigV1) (*OptimizeForConfig, error) { - if externalOptimizeForConfigV1.IsEmpty() { - return nil, nil - } - if externalOptimizeForConfigV1.Default == "" { - return nil, errors.New("optimize_for setting requires a default value") - } - value, ok := descriptorpb.FileOptions_OptimizeMode_value[externalOptimizeForConfigV1.Default] - if !ok { - return nil, fmt.Errorf( - "invalid optimize_for default value; expected one of %v", - enumMapToStringSlice(descriptorpb.FileOptions_OptimizeMode_value), - ) - } - defaultOptimizeFor := descriptorpb.FileOptions_OptimizeMode(value) - seenModuleFullNames := make(map[string]struct{}, len(externalOptimizeForConfigV1.Except)) - except := make([]bufmodule.ModuleFullName, 0, len(externalOptimizeForConfigV1.Except)) - for _, moduleName := range externalOptimizeForConfigV1.Except { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid optimize_for except: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid optimize_for except: %q is defined multiple times", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - except = append(except, moduleFullName) - } - override := make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode, len(externalOptimizeForConfigV1.Override)) - for moduleName, optimizeFor := range externalOptimizeForConfigV1.Override { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid optimize_for override key: %w", err) - } - value, ok := descriptorpb.FileOptions_OptimizeMode_value[optimizeFor] - if !ok { - return nil, fmt.Errorf( - "invalid optimize_for override value; expected one of %v", - enumMapToStringSlice(descriptorpb.FileOptions_OptimizeMode_value), - ) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf( - "invalid optimize_for override: %q is already defined as an except", - moduleFullName.String(), - ) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - override[moduleFullName] = descriptorpb.FileOptions_OptimizeMode(value) - } - return &OptimizeForConfig{ - Default: defaultOptimizeFor, - Except: except, - Override: override, - }, nil -} - -func newGoPackagePrefixConfigV1(externalGoPackagePrefixConfig ExternalGoPackagePrefixConfigV1) (*GoPackagePrefixConfig, error) { - if externalGoPackagePrefixConfig.IsEmpty() { - return nil, nil - } - if externalGoPackagePrefixConfig.Default == "" { - return nil, errors.New("go_package_prefix setting requires a default value") - } - defaultGoPackagePrefix, err := normalpath.NormalizeAndValidate(externalGoPackagePrefixConfig.Default) - if err != nil { - return nil, fmt.Errorf("invalid go_package_prefix default: %w", err) - } - seenModuleFullNames := make(map[string]struct{}, len(externalGoPackagePrefixConfig.Except)) - except := make([]bufmodule.ModuleFullName, 0, len(externalGoPackagePrefixConfig.Except)) - for _, moduleName := range externalGoPackagePrefixConfig.Except { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid go_package_prefix except: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid go_package_prefix except: %q is defined multiple times", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - except = append(except, moduleFullName) - } - override := make(map[bufmodule.ModuleFullName]string, len(externalGoPackagePrefixConfig.Override)) - for moduleName, goPackagePrefix := range externalGoPackagePrefixConfig.Override { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid go_package_prefix override key: %w", err) - } - normalizedGoPackagePrefix, err := normalpath.NormalizeAndValidate(goPackagePrefix) - if err != nil { - return nil, fmt.Errorf("invalid go_package_prefix override value: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid go_package_prefix override: %q is already defined as an except", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - override[moduleFullName] = normalizedGoPackagePrefix - } - return &GoPackagePrefixConfig{ - Default: defaultGoPackagePrefix, - Except: except, - Override: override, - }, nil -} - -func newRubyPackageConfigV1( - externalRubyPackageConfig ExternalRubyPackageConfigV1, -) (*RubyPackageConfig, error) { - if externalRubyPackageConfig.IsEmpty() { - return nil, nil - } - seenModuleFullNames := make(map[string]struct{}, len(externalRubyPackageConfig.Except)) - except := make([]bufmodule.ModuleFullName, 0, len(externalRubyPackageConfig.Except)) - for _, moduleName := range externalRubyPackageConfig.Except { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid ruby_package except: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid ruby_package except: %q is defined multiple times", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - except = append(except, moduleFullName) - } - override := make(map[bufmodule.ModuleFullName]string, len(externalRubyPackageConfig.Override)) - for moduleName, rubyPackage := range externalRubyPackageConfig.Override { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid ruby_package override key: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid ruby_package override: %q is already defined as an except", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - override[moduleFullName] = rubyPackage - } - return &RubyPackageConfig{ - Except: except, - Override: override, - }, nil -} - -func newCsharpNamespaceConfigV1( - externalCsharpNamespaceConfig ExternalCsharpNamespaceConfigV1, -) (*CsharpNameSpaceConfig, error) { - if externalCsharpNamespaceConfig.IsEmpty() { - return nil, nil - } - seenModuleFullNames := make(map[string]struct{}, len(externalCsharpNamespaceConfig.Except)) - except := make([]bufmodule.ModuleFullName, 0, len(externalCsharpNamespaceConfig.Except)) - for _, moduleName := range externalCsharpNamespaceConfig.Except { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid csharp_namespace except: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid csharp_namespace except: %q is defined multiple times", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - except = append(except, moduleFullName) - } - override := make(map[bufmodule.ModuleFullName]string, len(externalCsharpNamespaceConfig.Override)) - for moduleName, csharpNamespace := range externalCsharpNamespaceConfig.Override { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid csharp_namespace override key: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid csharp_namespace override: %q is already defined as an except", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - override[moduleFullName] = csharpNamespace - } - return &CsharpNameSpaceConfig{ - Except: except, - Override: override, - }, nil -} - -func newObjcClassPrefixConfigV1(externalObjcClassPrefixConfig ExternalObjcClassPrefixConfigV1) (*ObjcClassPrefixConfig, error) { - if externalObjcClassPrefixConfig.IsEmpty() { - return nil, nil - } - // It's ok to have an empty default, which will have the same effect as previously enabling managed mode. - defaultObjcClassPrefix := externalObjcClassPrefixConfig.Default - seenModuleFullNames := make(map[string]struct{}, len(externalObjcClassPrefixConfig.Except)) - except := make([]bufmodule.ModuleFullName, 0, len(externalObjcClassPrefixConfig.Except)) - for _, moduleName := range externalObjcClassPrefixConfig.Except { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid objc_class_prefix except: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid objc_class_prefix except: %q is defined multiple times", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - except = append(except, moduleFullName) - } - override := make(map[bufmodule.ModuleFullName]string, len(externalObjcClassPrefixConfig.Override)) - for moduleName, objcClassPrefix := range externalObjcClassPrefixConfig.Override { - moduleFullName, err := bufmodule.ParseModuleFullName(moduleName) - if err != nil { - return nil, fmt.Errorf("invalid objc_class_prefix override key: %w", err) - } - if _, ok := seenModuleFullNames[moduleFullName.String()]; ok { - return nil, fmt.Errorf("invalid objc_class_prefix override: %q is already defined as an except", moduleFullName.String()) - } - seenModuleFullNames[moduleFullName.String()] = struct{}{} - override[moduleFullName] = objcClassPrefix - } - return &ObjcClassPrefixConfig{ - Default: defaultObjcClassPrefix, - Except: except, - Override: override, - }, nil -} - -func newConfigV1Beta1(externalConfig ExternalConfigV1Beta1, id string) (*Config, error) { - managedConfig, err := newManagedConfigV1Beta1(externalConfig.Options, externalConfig.Managed) - if err != nil { - return nil, err - } - pluginConfigs := make([]*PluginConfig, 0, len(externalConfig.Plugins)) - for _, plugin := range externalConfig.Plugins { - strategy, err := ParseStrategy(plugin.Strategy) - if err != nil { - return nil, err - } - opt, err := encoding.InterfaceSliceOrStringToCommaSepString(plugin.Opt) - if err != nil { - return nil, err - } - pluginConfig := &PluginConfig{ - Name: plugin.Name, - Out: plugin.Out, - Opt: opt, - Strategy: strategy, - } - if plugin.Path != "" { - pluginConfig.Path = []string{plugin.Path} - } - pluginConfigs = append(pluginConfigs, pluginConfig) - } - return &Config{ - PluginConfigs: pluginConfigs, - ManagedConfig: managedConfig, - }, nil -} - -func validateExternalConfigV1Beta1(externalConfig ExternalConfigV1Beta1, id string) error { - if len(externalConfig.Plugins) == 0 { - return fmt.Errorf("%s: no plugins set", id) - } - for _, plugin := range externalConfig.Plugins { - if plugin.Name == "" { - return fmt.Errorf("%s: plugin name is required", id) - } - if plugin.Out == "" { - return fmt.Errorf("%s: plugin %s out is required", id, plugin.Name) - } - } - return nil -} - -func newManagedConfigV1Beta1(externalOptionsConfig ExternalOptionsConfigV1Beta1, enabled bool) (*ManagedConfig, error) { - if !enabled || externalOptionsConfig == (ExternalOptionsConfigV1Beta1{}) { - return nil, nil - } - var optimizeForConfig *OptimizeForConfig - if externalOptionsConfig.OptimizeFor != "" { - value, ok := descriptorpb.FileOptions_OptimizeMode_value[externalOptionsConfig.OptimizeFor] - if !ok { - return nil, fmt.Errorf( - "invalid optimize_for value; expected one of %v", - enumMapToStringSlice(descriptorpb.FileOptions_OptimizeMode_value), - ) - } - optimizeForConfig = &OptimizeForConfig{ - Default: descriptorpb.FileOptions_OptimizeMode(value), - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - } - } - return &ManagedConfig{ - CcEnableArenas: externalOptionsConfig.CcEnableArenas, - JavaMultipleFiles: externalOptionsConfig.JavaMultipleFiles, - OptimizeForConfig: optimizeForConfig, - }, nil -} - -// enumMapToStringSlice is a convenience function for mapping Protobuf enums -// into a slice of strings. -func enumMapToStringSlice(enums map[string]int32) []string { - slice := make([]string, 0, len(enums)) - for enum := range enums { - slice = append(slice, enum) - } - return slice -} - -type readConfigOptions struct { - override string -} - -func newReadConfigOptions() *readConfigOptions { - return &readConfigOptions{} -} - -func newTypesConfigV1(externalConfig ExternalTypesConfigV1) *TypesConfig { - if externalConfig.IsEmpty() { - return nil - } - return &TypesConfig{ - Include: externalConfig.Include, - } -} diff --git a/private/buf/bufgen/config_test.go b/private/buf/bufgen/config_test.go deleted file mode 100644 index 1e1dc541c0..0000000000 --- a/private/buf/bufgen/config_test.go +++ /dev/null @@ -1,746 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufgen - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/pkg/storage" - "github.com/bufbuild/buf/private/pkg/storage/storagemem" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "google.golang.org/protobuf/types/descriptorpb" -) - -func TestReadConfigV1Beta1(t *testing.T) { - t.Parallel() - truth := true - successConfig := &Config{ - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Opt: "plugins=connect", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - ManagedConfig: &ManagedConfig{ - CcEnableArenas: &truth, - JavaMultipleFiles: &truth, - JavaStringCheckUtf8: nil, - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_CODE_SIZE, - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - }, - }, - } - successConfig2 := &Config{ - ManagedConfig: &ManagedConfig{ - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_SPEED, - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - }, - }, - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Opt: "plugins=connect,foo=bar", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - } - successConfig3 := &Config{ - ManagedConfig: &ManagedConfig{ - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_LITE_RUNTIME, - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - }, - }, - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - } - successConfig4 := &Config{ - ManagedConfig: &ManagedConfig{ - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_LITE_RUNTIME, - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - }, - }, - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Strategy: StrategyAll, - }, - }, - } - ctx := context.Background() - nopLogger := zap.NewNop() - provider := NewProvider(zap.NewNop()) - readBucket, err := storagemem.NewReadBucket(nil) - require.NoError(t, err) - - testReadConfigSuccess := func(t *testing.T, configPath string, expected *Config) { - t.Helper() - config, err := ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(configPath)) - require.NoError(t, err) - assert.Equal(t, expected, config) - data, err := os.ReadFile(configPath) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - assert.Equal(t, expected, config) - } - - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success1.yaml"), successConfig) - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success1.json"), successConfig) - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success2.yaml"), successConfig2) - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success2.json"), successConfig2) - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success3.yaml"), successConfig3) - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success3.json"), successConfig3) - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success3.yml"), successConfig3) - testReadConfigSuccess(t, filepath.Join("testdata", "v1beta1", "gen_success4_nopath.yaml"), successConfig4) - - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1beta1", "gen_error1.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1beta1", "gen_error2.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1beta1", "gen_error3.yaml")) -} - -func TestReadConfigV1(t *testing.T) { - t.Parallel() - truth := true - successConfig := &Config{ - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Opt: "plugins=connect", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - ManagedConfig: &ManagedConfig{ - CcEnableArenas: &truth, - JavaMultipleFiles: &truth, - JavaStringCheckUtf8: &truth, - JavaPackagePrefixConfig: &JavaPackagePrefixConfig{ - Default: "org", - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]string), - }, - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_CODE_SIZE, - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - }, - Override: map[string]map[string]string{ - bufimagemodify.JavaPackageID: {"a.proto": "override"}, - }, - }, - TypesConfig: &TypesConfig{ - Include: []string{ - "buf.alpha.lint.v1.IDPaths", - }, - }, - } - successConfig2 := &Config{ - ManagedConfig: &ManagedConfig{ - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_SPEED, - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - }, - }, - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Opt: "plugins=connect,foo=bar", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - } - successConfig3 := &Config{ - ManagedConfig: &ManagedConfig{ - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_LITE_RUNTIME, - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode), - }, - }, - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - } - successConfig4 := &Config{ - PluginConfigs: []*PluginConfig{ - { - Plugin: "someremote.com/owner/myplugin:v1.1.0-1", - Out: "gen/go", - Strategy: StrategyAll, - }, - }, - } - successConfig5 := &Config{ - PluginConfigs: []*PluginConfig{ - { - Plugin: "someremote.com/owner/myplugin", - Out: "gen/go", - Strategy: StrategyAll, - }, - }, - } - moduleFullName, err := bufmodule.NewModuleFullName( - "someremote.com", - "owner", - "repo", - ) - require.NoError(t, err) - successConfig6 := &Config{ - ManagedConfig: &ManagedConfig{ - JavaPackagePrefixConfig: &JavaPackagePrefixConfig{ - Default: "org", - Except: []bufmodule.ModuleFullName{moduleFullName}, - Override: make(map[bufmodule.ModuleFullName]string), - }, - }, - PluginConfigs: []*PluginConfig{ - { - Plugin: "someremote.com/owner/myplugin", - Out: "gen/go", - Strategy: StrategyAll, - }, - }, - } - successConfig7 := &Config{ - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Opt: "plugins=connect", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - ManagedConfig: nil, - } - moduleFullName1 := mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "repo", - ) - moduleFullName2 := mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "foo", - ) - moduleFullName3 := mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "bar", - ) - moduleFullName4 := mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "baz", - ) - successConfig8 := &Config{ - ManagedConfig: &ManagedConfig{ - CsharpNameSpaceConfig: &CsharpNameSpaceConfig{ - Except: []bufmodule.ModuleFullName{ - moduleFullName1, - }, - Override: map[bufmodule.ModuleFullName]string{ - moduleFullName2: "a", - moduleFullName3: "b", - moduleFullName4: "c", - }, - }, - ObjcClassPrefixConfig: &ObjcClassPrefixConfig{ - Default: "default", - Except: []bufmodule.ModuleFullName{ - moduleFullName1, - }, - Override: map[bufmodule.ModuleFullName]string{ - moduleFullName2: "a", - moduleFullName3: "b", - moduleFullName4: "c", - }, - }, - RubyPackageConfig: &RubyPackageConfig{ - Except: []bufmodule.ModuleFullName{ - moduleFullName1, - }, - Override: map[bufmodule.ModuleFullName]string{ - moduleFullName2: "x", - moduleFullName3: "y", - moduleFullName4: "z", - }, - }, - }, - PluginConfigs: []*PluginConfig{ - { - Plugin: "someremote.com/owner/myplugin", - Out: "gen/go", - Strategy: StrategyAll, - }, - }, - } - successConfig9 := &Config{ - ManagedConfig: &ManagedConfig{ - OptimizeForConfig: &OptimizeForConfig{ - Default: descriptorpb.FileOptions_CODE_SIZE, - Except: []bufmodule.ModuleFullName{ - mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "repo", - ), - mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "foo", - ), - }, - Override: map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode{ - mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "bar", - ): descriptorpb.FileOptions_SPEED, - mustCreateModuleFullName( - t, - "someremote.com", - "owner", - "baz", - ): descriptorpb.FileOptions_LITE_RUNTIME, - }, - }, - }, - PluginConfigs: []*PluginConfig{ - { - Plugin: "someremote.com/owner/myplugin", - Out: "gen/go", - Strategy: StrategyAll, - }, - }, - } - - ctx := context.Background() - nopLogger := zap.NewNop() - provider := NewProvider(zap.NewNop()) - readBucket, err := storagemem.NewReadBucket(nil) - require.NoError(t, err) - config, err := ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success1.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - data, err := os.ReadFile(filepath.Join("testdata", "v1", "gen_success1.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success1.json"))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success1.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success2.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig2, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success2.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig2, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success2.json"))) - require.NoError(t, err) - require.Equal(t, successConfig2, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success2.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig2, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success3.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig3, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success3.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig3, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success3.json"))) - require.NoError(t, err) - require.Equal(t, successConfig3, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success3.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig3, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success3.yml"))) - require.NoError(t, err) - require.Equal(t, successConfig3, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success3.yml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig3, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success4.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig4, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success4.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig4, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success4.json"))) - require.NoError(t, err) - require.Equal(t, successConfig4, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success4.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig4, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success4.yml"))) - require.NoError(t, err) - require.Equal(t, successConfig4, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success4.yml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig4, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success5.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig5, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success5.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig5, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success5.json"))) - require.NoError(t, err) - require.Equal(t, successConfig5, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success5.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig5, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success5.yml"))) - require.NoError(t, err) - require.Equal(t, successConfig5, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success5.yml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig5, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success6.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig6, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success6.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig6, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success6.json"))) - require.NoError(t, err) - require.Equal(t, successConfig6, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success6.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig6, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success6.yml"))) - require.NoError(t, err) - require.Equal(t, successConfig6, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success6.yml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig6, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success7.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig7, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success7.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig7, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success7.json"))) - require.NoError(t, err) - require.Equal(t, successConfig7, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success7.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig7, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success7.yml"))) - require.NoError(t, err) - require.Equal(t, successConfig7, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success7.yml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig7, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success8.yaml"))) - require.NoError(t, err) - assertConfigsWithEqualCsharpnamespace(t, successConfig8, config) - assertConfigsWithEqualObjcPrefix(t, successConfig8, config) - assertConfigsWithEqualRubyPackage(t, successConfig8, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success8.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - assertConfigsWithEqualCsharpnamespace(t, successConfig8, config) - assertConfigsWithEqualObjcPrefix(t, successConfig8, config) - assertConfigsWithEqualRubyPackage(t, successConfig8, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success8.json"))) - require.NoError(t, err) - assertConfigsWithEqualCsharpnamespace(t, successConfig8, config) - assertConfigsWithEqualObjcPrefix(t, successConfig8, config) - assertConfigsWithEqualRubyPackage(t, successConfig8, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success8.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - assertConfigsWithEqualCsharpnamespace(t, successConfig8, config) - assertConfigsWithEqualObjcPrefix(t, successConfig8, config) - assertConfigsWithEqualRubyPackage(t, successConfig8, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success8.yml"))) - require.NoError(t, err) - assertConfigsWithEqualCsharpnamespace(t, successConfig8, config) - assertConfigsWithEqualObjcPrefix(t, successConfig8, config) - assertConfigsWithEqualRubyPackage(t, successConfig8, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success8.yml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - assertConfigsWithEqualCsharpnamespace(t, successConfig8, config) - assertConfigsWithEqualObjcPrefix(t, successConfig8, config) - assertConfigsWithEqualRubyPackage(t, successConfig8, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success9.yaml"))) - require.NoError(t, err) - assertConfigsWithEqualOptimizeFor(t, successConfig9, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success9.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - assertConfigsWithEqualOptimizeFor(t, successConfig9, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success9.json"))) - require.NoError(t, err) - assertConfigsWithEqualOptimizeFor(t, successConfig9, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success9.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - assertConfigsWithEqualOptimizeFor(t, successConfig9, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "gen_success9.yml"))) - require.NoError(t, err) - assertConfigsWithEqualOptimizeFor(t, successConfig9, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "gen_success9.yml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - assertConfigsWithEqualOptimizeFor(t, successConfig9, config) - - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error1.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error2.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error3.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error4.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error5.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error6.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error7.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error8.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error9.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error10.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error11.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error12.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error13.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error14.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error15.yaml")) - assertContainsReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "gen_error15.yaml"), "the remote field no longer works") - - successConfig = &Config{ - PluginConfigs: []*PluginConfig{ - { - Name: "go", - Out: "gen/go", - Opt: "plugins=connect", - Path: []string{"/path/to/foo"}, - Strategy: StrategyAll, - }, - }, - ManagedConfig: &ManagedConfig{ - GoPackagePrefixConfig: &GoPackagePrefixConfig{ - Default: "github.com/foo/bar/gen/go", - Except: make([]bufmodule.ModuleFullName, 0), - Override: make(map[bufmodule.ModuleFullName]string), - }, - }, - } - readBucket, err = storagemem.NewReadBucket(nil) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "go_gen_success1.yaml"))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "go_gen_success1.yaml")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(filepath.Join("testdata", "v1", "go_gen_success1.json"))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - data, err = os.ReadFile(filepath.Join("testdata", "v1", "go_gen_success1.json")) - require.NoError(t, err) - config, err = ReadConfig(ctx, nopLogger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.NoError(t, err) - require.Equal(t, successConfig, config) - - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "go_gen_error2.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "go_gen_error3.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "go_gen_error4.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "go_gen_error5.yaml")) - testReadConfigError(t, nopLogger, provider, readBucket, filepath.Join("testdata", "v1", "go_gen_error6.yaml")) -} - -func testReadConfigError(t *testing.T, logger *zap.Logger, provider Provider, readBucket storage.ReadBucket, testFilePath string) { - ctx := context.Background() - _, err := ReadConfig(ctx, logger, provider, readBucket, ReadConfigWithOverride(testFilePath)) - require.Error(t, err) - data, err := os.ReadFile(testFilePath) - require.NoError(t, err) - _, err = ReadConfig(ctx, logger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.Error(t, err) -} - -func assertContainsReadConfigError(t *testing.T, logger *zap.Logger, provider Provider, readBucket storage.ReadBucket, testFilePath string, message string) { - ctx := context.Background() - _, err := ReadConfig(ctx, logger, provider, readBucket, ReadConfigWithOverride(testFilePath)) - require.Error(t, err) - data, err := os.ReadFile(testFilePath) - require.NoError(t, err) - _, err = ReadConfig(ctx, logger, provider, readBucket, ReadConfigWithOverride(string(data))) - require.Error(t, err) - assert.Contains(t, err.Error(), message) -} - -func mustCreateModuleFullName( - t *testing.T, - remote string, - owner string, - repository string, -) bufmodule.ModuleFullName { - moduleFullName, err := bufmodule.NewModuleFullName(remote, owner, repository) - require.NoError(t, err) - return moduleFullName -} - -func assertConfigsWithEqualObjcPrefix(t *testing.T, successConfig *Config, config *Config) { - require.Equal(t, successConfig.PluginConfigs, config.PluginConfigs) - require.NotNil(t, successConfig.ManagedConfig) - require.NotNil(t, config.ManagedConfig) - require.NotNil(t, successConfig.ManagedConfig.ObjcClassPrefixConfig) - require.NotNil(t, config.ManagedConfig.ObjcClassPrefixConfig) - successObjcPrefixConfig := successConfig.ManagedConfig.ObjcClassPrefixConfig - objcPrefixConfig := config.ManagedConfig.ObjcClassPrefixConfig - require.Equal(t, successObjcPrefixConfig.Default, objcPrefixConfig.Default) - require.Equal(t, successObjcPrefixConfig.Except, objcPrefixConfig.Except) - assertEqualModuleFullNameKeyedMaps(t, successObjcPrefixConfig.Override, objcPrefixConfig.Override) -} - -func assertConfigsWithEqualCsharpnamespace(t *testing.T, successConfig *Config, config *Config) { - require.Equal(t, successConfig.PluginConfigs, config.PluginConfigs) - require.NotNil(t, successConfig.ManagedConfig) - require.NotNil(t, config.ManagedConfig) - require.NotNil(t, successConfig.ManagedConfig.CsharpNameSpaceConfig) - require.NotNil(t, config.ManagedConfig.CsharpNameSpaceConfig) - successCsharpConfig := successConfig.ManagedConfig.CsharpNameSpaceConfig - csharpConfig := config.ManagedConfig.CsharpNameSpaceConfig - require.Equal(t, successCsharpConfig.Except, csharpConfig.Except) - assertEqualModuleFullNameKeyedMaps(t, successCsharpConfig.Override, csharpConfig.Override) -} - -func assertConfigsWithEqualRubyPackage(t *testing.T, successConfig *Config, config *Config) { - require.Equal(t, successConfig.PluginConfigs, config.PluginConfigs) - require.NotNil(t, successConfig.ManagedConfig) - require.NotNil(t, config.ManagedConfig) - require.NotNil(t, successConfig.ManagedConfig.RubyPackageConfig) - require.NotNil(t, config.ManagedConfig.RubyPackageConfig) - successRubyConfig := successConfig.ManagedConfig.RubyPackageConfig - rubyConfig := config.ManagedConfig.RubyPackageConfig - require.Equal(t, successRubyConfig.Except, rubyConfig.Except) - assertEqualModuleFullNameKeyedMaps(t, successRubyConfig.Override, rubyConfig.Override) -} - -func assertConfigsWithEqualOptimizeFor(t *testing.T, successConfig *Config, config *Config) { - require.Equal(t, successConfig.PluginConfigs, config.PluginConfigs) - require.NotNil(t, successConfig.ManagedConfig) - require.NotNil(t, config.ManagedConfig) - successOptimizeForConfig := successConfig.ManagedConfig.OptimizeForConfig - require.NotNil(t, successOptimizeForConfig) - optimizeForConfig := config.ManagedConfig.OptimizeForConfig - require.NotNil(t, optimizeForConfig) - require.Equal(t, successOptimizeForConfig.Default, optimizeForConfig.Default) - require.Equal(t, successOptimizeForConfig.Except, optimizeForConfig.Except) - assertEqualModuleFullNameKeyedMaps(t, optimizeForConfig.Override, optimizeForConfig.Override) -} - -func assertEqualModuleFullNameKeyedMaps[V any](t *testing.T, m1 map[bufmodule.ModuleFullName]V, m2 map[bufmodule.ModuleFullName]V) { - require.Equal(t, len(m1), len(m2)) - keyedM1 := make(map[string]V, len(m1)) - keyedM2 := make(map[string]V, len(m2)) - for k, v := range m1 { - keyedM1[k.String()] = v - } - for k, v := range m2 { - keyedM2[k.String()] = v - } - require.Equal(t, keyedM1, keyedM2) -} diff --git a/private/buf/bufgen/provider.go b/private/buf/bufgen/provider.go deleted file mode 100644 index 79698151d5..0000000000 --- a/private/buf/bufgen/provider.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufgen - -import ( - "context" - "io" - - "github.com/bufbuild/buf/private/pkg/encoding" - "github.com/bufbuild/buf/private/pkg/storage" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" - "go.uber.org/multierr" - "go.uber.org/zap" -) - -type provider struct { - logger *zap.Logger - tracer trace.Tracer -} - -func newProvider(logger *zap.Logger) *provider { - return &provider{ - logger: logger, - tracer: otel.GetTracerProvider().Tracer("bufbuild/buf"), - } -} - -func (p *provider) GetConfig(ctx context.Context, readBucket storage.ReadBucket) (_ *Config, retErr error) { - ctx, span := p.tracer.Start(ctx, "get_config") - defer span.End() - defer func() { - if retErr != nil { - span.RecordError(retErr) - span.SetStatus(codes.Error, retErr.Error()) - } - }() - - readObjectCloser, err := readBucket.Get(ctx, ExternalConfigFilePath) - if err != nil { - // There is no default generate template, so we propagate all errors, including - // storage.ErrNotExist. - return nil, err - } - defer func() { - retErr = multierr.Append(retErr, readObjectCloser.Close()) - }() - data, err := io.ReadAll(readObjectCloser) - if err != nil { - return nil, err - } - return getConfig( - p.logger, - encoding.UnmarshalYAMLNonStrict, - encoding.UnmarshalYAMLStrict, - data, - `File "`+readObjectCloser.ExternalPath()+`"`, - ) -} diff --git a/private/buf/bufgen/provider_test.go b/private/buf/bufgen/provider_test.go deleted file mode 100644 index 9860a0765d..0000000000 --- a/private/buf/bufgen/provider_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufgen - -import ( - "context" - "errors" - "fmt" - "io/fs" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/pkg/storage/storagemem" - "github.com/bufbuild/buf/private/pkg/storage/storageos" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -const testConfigFileData = ` -version: %s -plugins: - - name: go - out: private/gen/proto/go - opt: paths=source_relative -` - -func TestProviderV1Beta1(t *testing.T) { - t.Parallel() - testProvider(t, "v1beta1") -} - -func TestProviderV1(t *testing.T) { - t.Parallel() - testProvider(t, "v1") -} - -func TestProviderError(t *testing.T) { - t.Parallel() - storageosProvider := storageos.NewProvider() - readWriteBucket, err := storageosProvider.NewReadWriteBucket(".") - require.NoError(t, err) - - provider := NewProvider(zap.NewNop()) - _, err = provider.GetConfig(context.Background(), readWriteBucket) - require.True(t, errors.Is(err, fs.ErrNotExist)) -} - -func testProvider(t *testing.T, version string) { - storageosProvider := storageos.NewProvider() - readWriteBucket, err := storageosProvider.NewReadWriteBucket(filepath.Join("testdata", version)) - require.NoError(t, err) - - nopLogger := zap.NewNop() - provider := NewProvider(zap.NewNop()) - actual, err := provider.GetConfig(context.Background(), readWriteBucket) - require.NoError(t, err) - - emptyBucket, err := storagemem.NewReadBucket(nil) - require.NoError(t, err) - expected, err := ReadConfig(context.Background(), nopLogger, provider, emptyBucket, ReadConfigWithOverride(fmt.Sprintf(testConfigFileData, version))) - require.NoError(t, err) - assert.Equal(t, expected, actual) -} diff --git a/private/buf/cmd/buf/command/generate/generate_test.go b/private/buf/cmd/buf/command/generate/generate_test.go index b7e1e463ac..58c4ed12b2 100644 --- a/private/buf/cmd/buf/command/generate/generate_test.go +++ b/private/buf/cmd/buf/command/generate/generate_test.go @@ -24,7 +24,6 @@ import ( "path/filepath" "testing" - "github.com/bufbuild/buf/private/buf/bufgen" "github.com/bufbuild/buf/private/buf/cmd/buf/internal/internaltesting" "github.com/bufbuild/buf/private/bufpkg/buftesting" "github.com/bufbuild/buf/private/pkg/app/appcmd" @@ -550,19 +549,20 @@ func testRunStdoutStderr(t *testing.T, stdin io.Reader, expectedExitCode int, ex } func newExternalConfigV1String(t *testing.T, plugins []*testPluginInfo, out string) string { - externalConfig := bufgen.ExternalConfigV1{ - Version: "v1", - } + externalConfig := make(map[string]interface{}) + externalConfig["version"] = "v1" + pluginConfigs := []map[string]string{} for _, plugin := range plugins { - externalConfig.Plugins = append( - externalConfig.Plugins, - bufgen.ExternalPluginConfigV1{ - Name: plugin.name, - Out: out, - Opt: plugin.opt, + pluginConfigs = append( + pluginConfigs, + map[string]string{ + "name": plugin.name, + "opt": plugin.opt, + "out": out, }, ) } + externalConfig["plugins"] = pluginConfigs data, err := json.Marshal(externalConfig) require.NoError(t, err) return string(data) diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 7e496d3705..702b903b14 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -42,6 +42,20 @@ type BufGenYAMLFile interface { isBufGenYAMLFile() } +// NewBufGenYAMLFile returns a new BufGenYAMLFile. It is validated given each +// parameter is validated. +func NewBufGenYAMLFile( + version FileVersion, + generateConfig GenerateConfig, + inputConfigs []InputConfig, +) BufGenYAMLFile { + return newBufGenYAMLFile( + version, + generateConfig, + inputConfigs, + ) +} + // GetBufGenYAMLFileForPrefix gets the buf.gen.yaml file at the given bucket prefix. // // The buf.gen.yaml file will be attempted to be read at prefix/buf.gen.yaml. diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index bfd66d8a70..eaa665424e 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -36,6 +36,31 @@ type GenerateConfig interface { isGenerateConfig() } +// NewGenerateConfig returns a validated GenerateConfig. +func NewGenerateConfig( + pluginConfigs []GeneratePluginConfig, + managedConfig GenerateManagedConfig, + typeConfig GenerateTypeConfig, +) (GenerateConfig, error) { + if len(pluginConfigs) == 0 { + // TODO: make sure this message is consistent with other cases where there is 0 plugin + return nil, errors.New("a plugin is required") + } + return &generateConfig{ + pluginConfigs: pluginConfigs, + managedConfig: managedConfig, + typeConfig: typeConfig, + }, nil +} + +// *** PRIVATE *** + +type generateConfig struct { + pluginConfigs []GeneratePluginConfig + managedConfig GenerateManagedConfig + typeConfig GenerateTypeConfig +} + func newGenerateConfigFromExternalFileV1Beta1( externalFile externalBufGenYAMLV1Beta1, ) (GenerateConfig, error) { @@ -104,14 +129,6 @@ func newGenerateConfigFromExternalFileV2( }, nil } -// *** PRIVATE *** - -type generateConfig struct { - pluginConfigs []GeneratePluginConfig - managedConfig GenerateManagedConfig - typeConfig GenerateTypeConfig -} - func (g *generateConfig) GeneratePluginConfigs() []GeneratePluginConfig { return g.pluginConfigs } From e7cb09063cd37ea75c2389b21c5e071f9574f514 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 18:30:48 -0500 Subject: [PATCH 45/66] migrate input as well --- TODO.txt | 2 + private/buf/buffetch/buffetch.go | 16 +++- private/buf/buffetch/internal/internal.go | 22 ++++- .../buf/cmd/buf/command/generate/generate.go | 95 +++++++++++++++++-- .../bufconfig/generate_plugin_config.go | 23 +++++ private/bufpkg/bufconfig/input_config.go | 26 +++++ 6 files changed, 168 insertions(+), 16 deletions(-) diff --git a/TODO.txt b/TODO.txt index 5283fcf1b1..d9dbc503bb 100644 --- a/TODO.txt +++ b/TODO.txt @@ -30,3 +30,5 @@ - Un-skip relevant tests in generate_test.go and generate_unix_test.go, when - the behavior of --path and --exclude-path is updated to match what's on main - proto file ref is implemented + +- Use syserror when possible. diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index 17bf969a2c..da6b324d8a 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -402,8 +402,16 @@ func NewWriter( ) } -// GetInputConfigForRef returns the input config for the ref. -func GetInputConfigForRef(ref Ref) (bufconfig.InputConfig, error) { +// GetInputConfigForString returns the input config for the input string. +func GetInputConfigForString( + ctx context.Context, + refParser RefParser, + value string, +) (bufconfig.InputConfig, error) { + ref, err := refParser.GetRef(ctx, value) + if err != nil { + return nil, err + } switch t := ref.(type) { case MessageRef: switch t.MessageEncoding() { @@ -423,9 +431,9 @@ func GetInputConfigForRef(ref Ref) (bufconfig.InputConfig, error) { t.internalSingleRef().CompressionType().String(), ), nil default: - // TODO: handle the YAML cas + // TODO: handle refs with YAML type return nil, fmt.Errorf("unknown encoding: %v", t.MessageEncoding()) } } - return internal.GetInputConfigForRef(ref.internalRef()) + return internal.GetInputConfigForRef(ref.internalRef(), value) } diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index 3ed0bf5660..5c927861bd 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -16,7 +16,6 @@ package internal import ( "context" - "errors" "fmt" "io" "net/http" @@ -866,9 +865,10 @@ func WithPutFileNoFileCompression() PutFileOption { // GetModuleOption is a GetModule option. type GetModuleOption func(*getModuleOptions) -// TODO: delete this -- this cannot handle git -// GetInputConfigForRef returns the input config for the ref. -func GetInputConfigForRef(ref Ref) (bufconfig.InputConfig, error) { +// GetInputConfigForRef returns the input config for the ref. A string is also +// passed because if the ref is a git ref, it would only have a git.Name, instead +// of a git branch, a git ref and a git tag. Therefore the original string is passed. +func GetInputConfigForRef(ref Ref, value string) (bufconfig.InputConfig, error) { switch t := ref.(type) { case ArchiveRef: switch t.ArchiveType() { @@ -901,7 +901,19 @@ func GetInputConfigForRef(ref Ref) (bufconfig.InputConfig, error) { t.Path(), ), nil case GitRef: - return nil, errors.New("TODO: git") + _, options, err := getRawPathAndOptionsForInputString(value) + if err != nil { + return nil, err + } + return bufconfig.NewGitRepoInputConfig( + t.Path(), + t.SubDirPath(), + options["branch"], + options["tag"], + options["ref"], + uint32ToPointer(t.Depth()), + t.RecurseSubmodules(), + ), nil default: return nil, fmt.Errorf("unexpected Ref of type %T", ref) } diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 0e2549b525..777153c206 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -23,6 +23,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/buf/bufctl" + "github.com/bufbuild/buf/private/buf/buffetch" "github.com/bufbuild/buf/private/buf/bufgen" "github.com/bufbuild/buf/private/bufpkg/bufanalysis" "github.com/bufbuild/buf/private/bufpkg/bufconfig" @@ -31,6 +32,7 @@ import ( "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appflag" "github.com/bufbuild/buf/private/pkg/command" + "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/spf13/cobra" @@ -344,9 +346,20 @@ func run( return err } if flags.Migrate && bufGenYAMLFile.FileVersion() != bufconfig.FileVersionV2 { - if err := bufconfig.PutBufGenYAMLFileForPrefix(ctx, bucket, ".", bufGenYAMLFile); err != nil { + migratedBufGenYAMLFile, err := getBufGenYAMLFileWithFlagEquivalence( + ctx, + logger, + bufGenYAMLFile, + input, + *flags, + ) + if err != nil { + return err + } + if err := bufconfig.PutBufGenYAMLFileForPrefix(ctx, bucket, ".", migratedBufGenYAMLFile); err != nil { return err } + // TODO: perhaps print a message } case templatePathExtension == ".yaml" || templatePathExtension == ".yml" || templatePathExtension == ".json": // We should not read from a bucket at "." because this path can jump context. @@ -359,9 +372,20 @@ func run( return err } if flags.Migrate && bufGenYAMLFile.FileVersion() != bufconfig.FileVersionV2 { - if err := bufconfig.WriteBufGenYAMLFile(configFile, bufGenYAMLFile); err != nil { + migratedBufGenYAMLFile, err := getBufGenYAMLFileWithFlagEquivalence( + ctx, + logger, + bufGenYAMLFile, + input, + *flags, + ) + if err != nil { + return err + } + if err := bufconfig.WriteBufGenYAMLFile(configFile, migratedBufGenYAMLFile); err != nil { return err } + // TODO: perhaps print a message } default: bufGenYAMLFile, err = bufconfig.ReadBufGenYAMLFile(strings.NewReader(flags.Template)) @@ -391,11 +415,6 @@ func run( } generateOptions := []bufgen.GenerateOption{ bufgen.GenerateWithBaseOutDirPath(flags.BaseOutDirPath), - // bufgen.GenerateWithIncludePaths(flags.Paths), - // bufgen.GenerateWithExcludePaths(flags.ExcludePaths), - // bufgen.GenerateWithIncludeTypes(flags.Types), - // bufgen.GenerateWithInputOverride(input), - // bufgen.GenerateWithModuleConfigPath(flags.Config), } if flags.IncludeImports { generateOptions = append( @@ -503,3 +522,65 @@ func getInputImages( } return inputImages, nil } + +// getBufGenYAMLFileWithFlagEquivalence returns a buf gen yaml file the same as +// the given one, except that it is overriden by flags. +// This is called only for migration. Input will be used regardless if it's empty. +func getBufGenYAMLFileWithFlagEquivalence( + ctx context.Context, + logger *zap.Logger, + bufGenYAMLFile bufconfig.BufGenYAMLFile, + input string, + flags flags, +) (bufconfig.BufGenYAMLFile, error) { + inputConfig, err := buffetch.GetInputConfigForString( + ctx, + buffetch.NewRefParser(logger), + input, + ) + if err != nil { + return nil, err + } + var includeTypes []string + if bufGenYAMLFile.GenerateConfig().GenerateTypeConfig() != nil { + includeTypes = bufGenYAMLFile.GenerateConfig().GenerateTypeConfig().IncludeTypes() + } + if len(flags.Types) > 0 { + includeTypes = flags.Types + } + inputConfig, err = bufconfig.NewInputConfigWithTargets( + inputConfig, + flags.Paths, + flags.ExcludePaths, + includeTypes, + ) + if err != nil { + return nil, err + } + pluginConfigs, err := slicesext.MapError( + bufGenYAMLFile.GenerateConfig().GeneratePluginConfigs(), + func(pluginConfig bufconfig.GeneratePluginConfig) (bufconfig.GeneratePluginConfig, error) { + return bufconfig.NewGeneratePluginWithIncludeImportsAndWKT( + pluginConfig, + flags.IncludeImports, + flags.IncludeWKT, + ) + }, + ) + if err != nil { + return nil, err + } + generateConfig, err := bufconfig.NewGenerateConfig( + pluginConfigs, + bufGenYAMLFile.GenerateConfig().GenerateManagedConfig(), + nil, + ) + if err != nil { + return nil, err + } + return bufconfig.NewBufGenYAMLFile( + bufconfig.FileVersionV2, + generateConfig, + []bufconfig.InputConfig{inputConfig}, + ), nil +} diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index 6354b38935..59ca0822d9 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -28,6 +28,8 @@ import ( "github.com/bufbuild/buf/private/pkg/syserror" ) +// TODO: move private methods below /* private */ + const remoteAlphaPluginDeprecationMessage = "the remote field no longer works as " + "the remote generation alpha has been deprecated, see the migration guide to " + "now-stable remote plugins: https://buf.build/docs/migration-guides/migrate-remote-generation-alpha/#migrate-to-remote-plugins" @@ -107,6 +109,27 @@ type GeneratePluginConfig interface { isGeneratePluginConfig() } +// NewGeneratePluginWithIncludeImportsAndWKT returns a GeneratePluginConfig the +// same as the input, with include imports and include wkt overriden. +func NewGeneratePluginWithIncludeImportsAndWKT( + config GeneratePluginConfig, + includeImports bool, + includeWKT bool, +) (GeneratePluginConfig, error) { + originalConfig, ok := config.(*pluginConfig) + if !ok { + return nil, syserror.Newf("unknown implementation of GeneratePluginConfig: %T", config) + } + pluginConfig := *originalConfig + if includeImports { + pluginConfig.includeImports = true + } + if includeWKT { + pluginConfig.includeWKT = true + } + return &pluginConfig, nil +} + func parseStrategy(s string) (*GenerateStrategy, error) { var strategy GenerateStrategy switch s { diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 6586559e49..7af7d07de5 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -190,6 +190,32 @@ func NewInputConfig( } } +// NewInputConfigWithTargets returns an input config with include paths, exclude +// paths and include types overriden. +func NewInputConfigWithTargets( + config InputConfig, + includePaths []string, + excludePaths []string, + includeTypes []string, +) (InputConfig, error) { + originalConfig, ok := config.(*inputConfig) + if !ok { + return nil, syserror.Newf("unknown implementation of InputConfig: %T", config) + } + // targetConfig is a copy of the original config. + targetConfig := *originalConfig + if len(includePaths) > 0 { + targetConfig.includePaths = includePaths + } + if len(excludePaths) > 0 { + targetConfig.excludePaths = excludePaths + } + if len(includeTypes) > 0 { + targetConfig.includeTypes = includeTypes + } + return &targetConfig, nil +} + // NewGitRepoInputConfig returns an input config for a git repo. func NewGitRepoInputConfig( location string, From dbcdaf412da708dbaacf7cb9342b6951a888a6d3 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Thu, 30 Nov 2023 18:51:09 -0500 Subject: [PATCH 46/66] --migrate runs; non-testing is finished; need more testing --- .../buf/cmd/buf/command/generate/generate.go | 3 ++ private/bufpkg/bufconfig/buf_gen_yaml_file.go | 2 +- .../bufpkg/bufconfig/buf_work_yaml_file.go | 2 +- .../bufconfig/generate_plugin_config.go | 30 ++++++++++--------- 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index 777153c206..73cf7f5a0c 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -533,6 +533,9 @@ func getBufGenYAMLFileWithFlagEquivalence( input string, flags flags, ) (bufconfig.BufGenYAMLFile, error) { + if input == "" { + input = "." + } inputConfig, err := buffetch.GetInputConfigForString( ctx, buffetch.NewRefParser(logger), diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 702b903b14..ba12b91323 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -235,7 +235,7 @@ func writeBufGenYAMLFile(writer io.Writer, bufGenYAMLFile BufGenYAMLFile) error if err != nil { return err } - _, err = writer.Write(append(bufLockFileHeader, data...)) + _, err = writer.Write(data) return err default: // This is a system error since we've already parsed. diff --git a/private/bufpkg/bufconfig/buf_work_yaml_file.go b/private/bufpkg/bufconfig/buf_work_yaml_file.go index af80cdd20b..3a9cffa4c8 100644 --- a/private/bufpkg/bufconfig/buf_work_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_work_yaml_file.go @@ -177,7 +177,7 @@ func writeBufWorkYAMLFile(writer io.Writer, bufWorkYAMLFile BufWorkYAMLFile) err if err != nil { return err } - _, err = writer.Write(append(bufLockFileHeader, data...)) + _, err = writer.Write(data) return err } diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index 59ca0822d9..daca439046 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -130,20 +130,7 @@ func NewGeneratePluginWithIncludeImportsAndWKT( return &pluginConfig, nil } -func parseStrategy(s string) (*GenerateStrategy, error) { - var strategy GenerateStrategy - switch s { - case "": - return nil, nil - case "directory": - strategy = GenerateStrategyDirectory - case "all": - strategy = GenerateStrategyAll - default: - return nil, fmt.Errorf("unknown strategy: %s", s) - } - return &strategy, nil -} +// *** PRIVATE *** type pluginConfig struct { // TODO: perhaps make some of these pointers so that whether a field is @@ -350,6 +337,21 @@ func newPluginConfigFromExternalV1( ) } +func parseStrategy(s string) (*GenerateStrategy, error) { + var strategy GenerateStrategy + switch s { + case "": + return nil, nil + case "directory": + strategy = GenerateStrategyDirectory + case "all": + strategy = GenerateStrategyAll + default: + return nil, fmt.Errorf("unknown strategy: %s", s) + } + return &strategy, nil +} + // TODO: move this up, maybe let v1 use this as well const ( typeRemote = "remote" From f61bb42353dcc1afa59c5c9f70fe37f8613a9573 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 13:53:05 -0500 Subject: [PATCH 47/66] todo --- TODO.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/TODO.txt b/TODO.txt index d9dbc503bb..c5ff2af859 100644 --- a/TODO.txt +++ b/TODO.txt @@ -26,9 +26,8 @@ - document behavior of file searching, config override - fix tamper-proofing - go through all todos - - Un-skip relevant tests in generate_test.go and generate_unix_test.go, when - the behavior of --path and --exclude-path is updated to match what's on main - proto file ref is implemented - -- Use syserror when possible. +- Use syserror when possible (especially in buf gen related code). +- Fix the issue where reading a buf.lock with empty digest errors. Run `buf mod update` to reproduce. From af9daa93b81d4a079ab1a2a1026c86be1685dd55 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 14:14:05 -0500 Subject: [PATCH 48/66] generate_config.go --- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 2 +- private/bufpkg/bufconfig/generate_config.go | 36 ++++++++++--------- private/bufpkg/bufconfig/input_config.go | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index ba12b91323..da646883f5 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -186,7 +186,7 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error } inputConfigs, err := slicesext.MapError( externalGenYAMLFile.Inputs, - newInputConfigFromExternalInputConfigV2, + newInputConfigFromExternalV2, ) if err != nil { return nil, err diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index eaa665424e..8777f5fd94 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -43,8 +43,7 @@ func NewGenerateConfig( typeConfig GenerateTypeConfig, ) (GenerateConfig, error) { if len(pluginConfigs) == 0 { - // TODO: make sure this message is consistent with other cases where there is 0 plugin - return nil, errors.New("a plugin is required") + return nil, newNoPluginsError() } return &generateConfig{ pluginConfigs: pluginConfigs, @@ -61,30 +60,26 @@ type generateConfig struct { typeConfig GenerateTypeConfig } -func newGenerateConfigFromExternalFileV1Beta1( - externalFile externalBufGenYAMLV1Beta1, +func newGenerateConfigFromExternalFileV2( + externalFile externalBufGenYAMLFileV2, ) (GenerateConfig, error) { - managedConfig, err := newManagedConfigFromExternalV1Beta1(externalFile.Options) + managedConfig, err := newManagedConfigFromExternalV2(externalFile.Managed) if err != nil { return nil, err } - if len(externalFile.Plugins) == 0 { - return nil, errors.New("must specifiy at least one plugin") - } pluginConfigs, err := slicesext.MapError( externalFile.Plugins, - newPluginConfigFromExternalV1Beta1, + newPluginConfigFromExternalV2, ) if err != nil { return nil, err } return &generateConfig{ - pluginConfigs: pluginConfigs, managedConfig: managedConfig, + pluginConfigs: pluginConfigs, }, nil } -// TODO: check if this is consistent with the rest of bufconfig (esp. its name and whether it should exist) func newGenerateConfigFromExternalFileV1( externalFile externalBufGenYAMLFileV1, ) (GenerateConfig, error) { @@ -93,7 +88,7 @@ func newGenerateConfigFromExternalFileV1( return nil, err } if len(externalFile.Plugins) == 0 { - return nil, errors.New("must specifiy at least one plugin") + return nil, newNoPluginsError() } pluginConfigs, err := slicesext.MapError( externalFile.Plugins, @@ -109,23 +104,26 @@ func newGenerateConfigFromExternalFileV1( }, nil } -func newGenerateConfigFromExternalFileV2( - externalFile externalBufGenYAMLFileV2, +func newGenerateConfigFromExternalFileV1Beta1( + externalFile externalBufGenYAMLV1Beta1, ) (GenerateConfig, error) { - managedConfig, err := newManagedConfigFromExternalV2(externalFile.Managed) + managedConfig, err := newManagedConfigFromExternalV1Beta1(externalFile.Options) if err != nil { return nil, err } + if len(externalFile.Plugins) == 0 { + return nil, newNoPluginsError() + } pluginConfigs, err := slicesext.MapError( externalFile.Plugins, - newPluginConfigFromExternalV2, + newPluginConfigFromExternalV1Beta1, ) if err != nil { return nil, err } return &generateConfig{ - managedConfig: managedConfig, pluginConfigs: pluginConfigs, + managedConfig: managedConfig, }, nil } @@ -142,3 +140,7 @@ func (g *generateConfig) GenerateTypeConfig() GenerateTypeConfig { } func (*generateConfig) isGenerateConfig() {} + +func newNoPluginsError() error { + return errors.New("must specify at least one plugin") +} diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index 7af7d07de5..bcf66842e4 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -354,7 +354,7 @@ type inputConfig struct { includePaths []string } -func newInputConfigFromExternalInputConfigV2(externalConfig externalInputConfigV2) (InputConfig, error) { +func newInputConfigFromExternalV2(externalConfig externalInputConfigV2) (InputConfig, error) { inputConfig := &inputConfig{} var inputTypes []InputConfigType var options []string From 05f477347039ae69bb4bf254e3a17f541566db56 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 14:33:41 -0500 Subject: [PATCH 49/66] buf_gen_yaml_file.go --- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 96 +++++++++---------- 1 file changed, 45 insertions(+), 51 deletions(-) diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index da646883f5..dec61579e2 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -36,7 +36,9 @@ var ( type BufGenYAMLFile interface { File + // GenerateConfig returns the generate config. GenerateConfig() GenerateConfig + // InputConfigs returns the input configs, which can be empty. InputConfigs() []InputConfig isBufGenYAMLFile() @@ -122,21 +124,6 @@ func newBufGenYAMLFile( } } -func (g *bufGenYAMLFile) FileVersion() FileVersion { - return g.fileVersion -} - -func (g *bufGenYAMLFile) GenerateConfig() GenerateConfig { - return g.generateConfig -} - -func (g *bufGenYAMLFile) InputConfigs() []InputConfig { - return g.inputConfigs -} - -func (*bufGenYAMLFile) isBufGenYAMLFile() {} -func (*bufGenYAMLFile) isFile() {} - func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error) { data, err := io.ReadAll(reader) if err != nil { @@ -203,42 +190,49 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error } func writeBufGenYAMLFile(writer io.Writer, bufGenYAMLFile BufGenYAMLFile) error { - // TODO: is this check necessary? - switch fileVersion := bufGenYAMLFile.FileVersion(); fileVersion { - case FileVersionV1Beta1, FileVersionV1, FileVersionV2: - // Regardless of version, we write the file as v2: - // TODO name external..config - pluginConfigs, err := slicesext.MapError( - bufGenYAMLFile.GenerateConfig().GeneratePluginConfigs(), - newExternalGeneratePluginConfigV2FromPluginConfig, - ) - if err != nil { - return err - } - managedConfig := newExternalManagedConfigV2FromGenerateManagedConfig( - bufGenYAMLFile.GenerateConfig().GenerateManagedConfig(), - ) - inputConfigs, err := slicesext.MapError( - bufGenYAMLFile.InputConfigs(), - newExternalInputConfigV2FromInputConfig, - ) - if err != nil { - return err - } - externalBufGenYAMLFileV2 := externalBufGenYAMLFileV2{ - Version: "v2", - Plugins: pluginConfigs, - Managed: managedConfig, - Inputs: inputConfigs, - } - data, err := encoding.MarshalYAML(&externalBufGenYAMLFileV2) - if err != nil { - return err - } - _, err = writer.Write(data) + // Regardless of version, we write the file as v2: + externalPluginConfigsV2, err := slicesext.MapError( + bufGenYAMLFile.GenerateConfig().GeneratePluginConfigs(), + newExternalGeneratePluginConfigV2FromPluginConfig, + ) + if err != nil { return err - default: - // This is a system error since we've already parsed. - return syserror.Newf("unknown FileVersion: %v", fileVersion) } + externalManagedConfigV2 := newExternalManagedConfigV2FromGenerateManagedConfig( + bufGenYAMLFile.GenerateConfig().GenerateManagedConfig(), + ) + externalInputConfigsV2, err := slicesext.MapError( + bufGenYAMLFile.InputConfigs(), + newExternalInputConfigV2FromInputConfig, + ) + if err != nil { + return err + } + externalBufGenYAMLFileV2 := externalBufGenYAMLFileV2{ + Version: FileVersionV2.String(), + Plugins: externalPluginConfigsV2, + Managed: externalManagedConfigV2, + Inputs: externalInputConfigsV2, + } + data, err := encoding.MarshalYAML(&externalBufGenYAMLFileV2) + if err != nil { + return err + } + _, err = writer.Write(data) + return err } + +func (g *bufGenYAMLFile) FileVersion() FileVersion { + return g.fileVersion +} + +func (g *bufGenYAMLFile) GenerateConfig() GenerateConfig { + return g.generateConfig +} + +func (g *bufGenYAMLFile) InputConfigs() []InputConfig { + return g.inputConfigs +} + +func (*bufGenYAMLFile) isBufGenYAMLFile() {} +func (*bufGenYAMLFile) isFile() {} From cfe73babe77c73521e7191c95b9d5650890c9a39 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 14:35:02 -0500 Subject: [PATCH 50/66] commit --- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index dec61579e2..12a79fb41c 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -124,6 +124,21 @@ func newBufGenYAMLFile( } } +func (g *bufGenYAMLFile) FileVersion() FileVersion { + return g.fileVersion +} + +func (g *bufGenYAMLFile) GenerateConfig() GenerateConfig { + return g.generateConfig +} + +func (g *bufGenYAMLFile) InputConfigs() []InputConfig { + return g.inputConfigs +} + +func (*bufGenYAMLFile) isBufGenYAMLFile() {} +func (*bufGenYAMLFile) isFile() {} + func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error) { data, err := io.ReadAll(reader) if err != nil { @@ -221,18 +236,3 @@ func writeBufGenYAMLFile(writer io.Writer, bufGenYAMLFile BufGenYAMLFile) error _, err = writer.Write(data) return err } - -func (g *bufGenYAMLFile) FileVersion() FileVersion { - return g.fileVersion -} - -func (g *bufGenYAMLFile) GenerateConfig() GenerateConfig { - return g.generateConfig -} - -func (g *bufGenYAMLFile) InputConfigs() []InputConfig { - return g.inputConfigs -} - -func (*bufGenYAMLFile) isBufGenYAMLFile() {} -func (*bufGenYAMLFile) isFile() {} From 7827f0a5e7e66847f89435c5f0dc02cf4b6d2c23 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 16:25:35 -0500 Subject: [PATCH 51/66] generate_external_config.go --- TODO.txt | 3 + private/bufpkg/bufconfig/buf_gen_yaml_file.go | 2 +- private/bufpkg/bufconfig/generate_config.go | 2 +- .../bufconfig/generate_external_config.go | 260 ++++++++---------- .../bufconfig/generate_managed_config.go | 2 +- 5 files changed, 119 insertions(+), 150 deletions(-) diff --git a/TODO.txt b/TODO.txt index c5ff2af859..a88f326350 100644 --- a/TODO.txt +++ b/TODO.txt @@ -30,4 +30,7 @@ - the behavior of --path and --exclude-path is updated to match what's on main - proto file ref is implemented - Use syserror when possible (especially in buf gen related code). +- Add Enabled() bool to GenerateManagedConfig: this is useful for printing the warning for the + case where managed mode external config is not empty but the user forgot to set enable: true. + Otherwise we would need to pass a logger to many functions, breaking the uniformity. - Fix the issue where reading a buf.lock with empty digest errors. Run `buf mod update` to reproduce. diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index 12a79fb41c..abf48cf459 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -150,7 +150,7 @@ func readBufGenYAMLFile(reader io.Reader, allowJSON bool) (BufGenYAMLFile, error } switch fileVersion { case FileVersionV1Beta1: - var externalGenYAMLFile externalBufGenYAMLV1Beta1 + var externalGenYAMLFile externalBufGenYAMLFileV1Beta1 if err := getUnmarshalStrict(allowJSON)(data, &externalGenYAMLFile); err != nil { return nil, fmt.Errorf("invalid as version %v: %w", fileVersion, err) } diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index 8777f5fd94..398b5db5bc 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -105,7 +105,7 @@ func newGenerateConfigFromExternalFileV1( } func newGenerateConfigFromExternalFileV1Beta1( - externalFile externalBufGenYAMLV1Beta1, + externalFile externalBufGenYAMLFileV1Beta1, ) (GenerateConfig, error) { managedConfig, err := newManagedConfigFromExternalV1Beta1(externalFile.Options) if err != nil { diff --git a/private/bufpkg/bufconfig/generate_external_config.go b/private/bufpkg/bufconfig/generate_external_config.go index b5a012fe97..4e0565a263 100644 --- a/private/bufpkg/bufconfig/generate_external_config.go +++ b/private/bufpkg/bufconfig/generate_external_config.go @@ -17,11 +17,33 @@ package bufconfig import "encoding/json" // TODO: this is a temporary file to avoid crowing other files. We can choose to move stuff from this file over. -// TODO: this is also completely copied over from bufgen.go, the only change made to it so far is unexporting the type. -// TODO: update struct type names to externalXYZFileV1/2/1Beta1 -// TODO: update GODOCs to the style of '// externalBufLockFileV2 represents the v2 buf.lock file.' -// externalBufGenYAMLFileV1 is a v1 external generate configuration. +// externalBufGenYAMLFileV1Beta1 represents the v1beta buf.gen.yaml file. +type externalBufGenYAMLFileV1Beta1 struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` + // Managed is whether managed mode is enabled. + Managed bool `json:"managed,omitempty" yaml:"managed,omitempty"` + Plugins []externalGeneratePluginConfigV1Beta1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Options externalGenerateManagedConfigV1Beta1 `json:"options,omitempty" yaml:"options,omitempty"` +} + +// externalGeneratePluginConfigV1Beta1 represents a single plugin conifg in a v1beta1 buf.gen.yaml file. +type externalGeneratePluginConfigV1Beta1 struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Out string `json:"out,omitempty" yaml:"out,omitempty"` + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + Path string `json:"path,omitempty" yaml:"path,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalGenerateManagedConfigV1Beta1 represents the options (for managed mode) config in a v1beta1 buf.gen.yaml file. +type externalGenerateManagedConfigV1Beta1 struct { + CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` + JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` + OptimizeFor string `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` +} + +// externalBufGenYAMLFileV1 represents the v1 buf.gen.yaml file. type externalBufGenYAMLFileV1 struct { Version string `json:"version,omitempty" yaml:"version,omitempty"` Plugins []externalGeneratePluginConfigV1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` @@ -29,22 +51,27 @@ type externalBufGenYAMLFileV1 struct { Types externalTypesConfigV1 `json:"types,omitempty" yaml:"types,omitempty"` } -// externalGeneratePluginConfigV1 is an external plugin configuration. +// externalGeneratePluginConfigV1 represents a single plugin config in a v1 buf.gen.yaml file. type externalGeneratePluginConfigV1 struct { - Plugin string `json:"plugin,omitempty" yaml:"plugin,omitempty"` - Revision int `json:"revision,omitempty" yaml:"revision,omitempty"` - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Remote string `json:"remote,omitempty" yaml:"remote,omitempty"` - Out string `json:"out,omitempty" yaml:"out,omitempty"` - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + // Exactly one of Plugin and Name is required. + // Plugin is the key for a local or remote plugin. + Plugin string `json:"plugin,omitempty" yaml:"plugin,omitempty"` + // Name is the key for a local plugin. + Name string `json:"name,omitempty" yaml:"name,omitempty"` + // Remote is the key for alpha remote plugin name, which is deprecated now. + Remote string `json:"remote,omitempty" yaml:"remote,omitempty"` + // Out is required. + Out string `json:"out,omitempty" yaml:"out,omitempty"` + Revision int `json:"revision,omitempty" yaml:"revision,omitempty"` + // Opt can be one string or multiple strings. + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + // Path can be one string or multiple strings. Path interface{} `json:"path,omitempty" yaml:"path,omitempty"` ProtocPath string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` } -// externalGenerateManagedConfigV1 is an external managed mode configuration. -// -// Only use outside of this package for testing. +// externalGenerateManagedConfigV1 represents the managed mode config in a v1 buf.gen.yaml file. type externalGenerateManagedConfigV1 struct { Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` @@ -56,51 +83,29 @@ type externalGenerateManagedConfigV1 struct { GoPackagePrefix externalGoPackagePrefixConfigV1 `json:"go_package_prefix,omitempty" yaml:"go_package_prefix,omitempty"` ObjcClassPrefix externalObjcClassPrefixConfigV1 `json:"objc_class_prefix,omitempty" yaml:"objc_class_prefix,omitempty"` RubyPackage externalRubyPackageConfigV1 `json:"ruby_package,omitempty" yaml:"ruby_package,omitempty"` - Override map[string]map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// isEmpty returns true if the config is empty, excluding the 'Enabled' setting. -func (e externalGenerateManagedConfigV1) isEmpty() bool { - return e.CcEnableArenas == nil && - e.JavaMultipleFiles == nil && - e.JavaStringCheckUtf8 == nil && - e.JavaPackagePrefix.isEmpty() && - e.CsharpNamespace.isEmpty() && - e.CsharpNamespace.isEmpty() && - e.OptimizeFor.isEmpty() && - e.GoPackagePrefix.isEmpty() && - e.ObjcClassPrefix.isEmpty() && - e.RubyPackage.isEmpty() && - len(e.Override) == 0 + // Override maps from a file option to a file path then to the value. + Override map[string]map[string]string `json:"override,omitempty" yaml:"override,omitempty"` } -// externalJavaPackagePrefixConfigV1 is the external java_package prefix configuration. +// externalJavaPackagePrefixConfigV1 represents the java_package_prefix config in a v1 buf.gen.yaml file. type externalJavaPackagePrefixConfigV1 struct { Default string `json:"default,omitempty" yaml:"default,omitempty"` Except []string `json:"except,omitempty" yaml:"except,omitempty"` Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` } -// isEmpty returns true if the config is empty. -func (e externalJavaPackagePrefixConfigV1) isEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// UnmarshalYAML satisfies the yaml.Unmarshaler interface. This is done to maintain backward compatibility +// UnmarshalYAML implements the yaml.Unmarshaler interface. This is done to maintain backward compatibility // of accepting a plain string value for java_package_prefix. func (e *externalJavaPackagePrefixConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { return e.unmarshalWith(unmarshal) } -// UnmarshalJSON satisfies the json.Unmarshaler interface. This is done to maintain backward compatibility +// UnmarshalJSON implements the json.Unmarshaler interface. This is done to maintain backward compatibility // of accepting a plain string value for java_package_prefix. func (e *externalJavaPackagePrefixConfigV1) UnmarshalJSON(data []byte) error { unmarshal := func(v interface{}) error { return json.Unmarshal(data, v) } - return e.unmarshalWith(unmarshal) } @@ -111,42 +116,39 @@ func (e *externalJavaPackagePrefixConfigV1) unmarshalWith(unmarshal func(interfa e.Default = prefix return nil } - type rawExternalJavaPackagePrefixConfigV1 externalJavaPackagePrefixConfigV1 if err := unmarshal((*rawExternalJavaPackagePrefixConfigV1)(e)); err != nil { return err } - return nil } -// externalOptimizeForConfigV1 is the external optimize_for configuration. +// isEmpty returns true if the config is empty. +func (e externalJavaPackagePrefixConfigV1) isEmpty() bool { + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalOptimizeForConfigV1 represents the optimize_for config in a v1 buf.gen.yaml file. type externalOptimizeForConfigV1 struct { Default string `json:"default,omitempty" yaml:"default,omitempty"` Except []string `json:"except,omitempty" yaml:"except,omitempty"` Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` } -// isEmpty returns true if the config is empty -func (e externalOptimizeForConfigV1) isEmpty() bool { // TODO: does it need to be public? - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// UnmarshalYAML satisfies the yaml.Unmarshaler interface. This is done to maintain backward compatibility +// UnmarshalYAML implements the yaml.Unmarshaler interface. This is done to maintain backward compatibility // of accepting a plain string value for optimize_for. func (e *externalOptimizeForConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { return e.unmarshalWith(unmarshal) } -// UnmarshalJSON satisfies the json.Unmarshaler interface. This is done to maintain backward compatibility +// UnmarshalJSON implements the json.Unmarshaler interface. This is done to maintain backward compatibility // of accepting a plain string value for optimize_for. func (e *externalOptimizeForConfigV1) UnmarshalJSON(data []byte) error { unmarshal := func(v interface{}) error { return json.Unmarshal(data, v) } - return e.unmarshalWith(unmarshal) } @@ -157,16 +159,21 @@ func (e *externalOptimizeForConfigV1) unmarshalWith(unmarshal func(interface{}) e.Default = optimizeFor return nil } - type rawExternalOptimizeForConfigV1 externalOptimizeForConfigV1 if err := unmarshal((*rawExternalOptimizeForConfigV1)(e)); err != nil { return err } - return nil } -// externalGoPackagePrefixConfigV1 is the external go_package prefix configuration. +// isEmpty returns true if the config is empty +func (e externalOptimizeForConfigV1) isEmpty() bool { // TODO: does it need to be public? + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalGoPackagePrefixConfigV1 represents the go_package_prefix config in a v1 buf.gen.yaml file. type externalGoPackagePrefixConfigV1 struct { Default string `json:"default,omitempty" yaml:"default,omitempty"` Except []string `json:"except,omitempty" yaml:"except,omitempty"` @@ -180,7 +187,7 @@ func (e externalGoPackagePrefixConfigV1) isEmpty() bool { len(e.Override) == 0 } -// externalCsharpNamespaceConfigV1 is the external csharp_namespace configuration. +// externalCsharpNamespaceConfigV1 represents the external csharp_namespace config in a v1 buf.gen.yaml file. type externalCsharpNamespaceConfigV1 struct { Except []string `json:"except,omitempty" yaml:"except,omitempty"` Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` @@ -192,140 +199,100 @@ func (e externalCsharpNamespaceConfigV1) isEmpty() bool { len(e.Override) == 0 } -// externalRubyPackageConfigV1 is the external ruby_package configuration +// externalRubyPackageConfigV1 represents the ruby_package config in a v1 buf.gen.yaml file. type externalRubyPackageConfigV1 struct { Except []string `json:"except,omitempty" yaml:"except,omitempty"` Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` } -// isEmpty returns true is the config is empty +// isEmpty returns true is the config is empty. func (e externalRubyPackageConfigV1) isEmpty() bool { // TODO: does this need to be public? same with other IsEmpty() return len(e.Except) == 0 && len(e.Override) == 0 } -// externalObjcClassPrefixConfigV1 is the external objc_class_prefix configuration. +// externalObjcClassPrefixConfigV1 represents the objc_class_prefix config in a v1 buf.gen.yaml file. type externalObjcClassPrefixConfigV1 struct { Default string `json:"default,omitempty" yaml:"default,omitempty"` Except []string `json:"except,omitempty" yaml:"except,omitempty"` Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` } +// isEmpty returns true is the config is empty. func (e externalObjcClassPrefixConfigV1) isEmpty() bool { return e.Default == "" && len(e.Except) == 0 && len(e.Override) == 0 } -// externalBufGenYAMLV1Beta1 is a v1 external generate configuration. -type externalBufGenYAMLV1Beta1 struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` - Managed bool `json:"managed,omitempty" yaml:"managed,omitempty"` - Plugins []externalGeneratePluginConfigV1Beta1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` - Options externalOptionsConfigV1Beta1 `json:"options,omitempty" yaml:"options,omitempty"` -} - -// externalGeneratePluginConfigV1Beta1 is an external plugin configuration. -type externalGeneratePluginConfigV1Beta1 struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Out string `json:"out,omitempty" yaml:"out,omitempty"` - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` - Path string `json:"path,omitempty" yaml:"path,omitempty"` - Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` -} - -// externalOptionsConfigV1Beta1 is an external options configuration. -type externalOptionsConfigV1Beta1 struct { - CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` - JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` - OptimizeFor string `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` -} - -// externalTypesConfigV1 is an external types configuration. +// externalTypesConfigV1 represents the types config in a v1 buf.gen.yaml file. type externalTypesConfigV1 struct { Include []string `json:"include,omitempty" yaml:"include"` } -// isEmpty returns true if e is empty. -func (e externalTypesConfigV1) isEmpty() bool { - return len(e.Include) == 0 -} - -// externalBufGenYAMLFileV2 is an external configuration. +// externalBufGenYAMLFileV2 represents the v2 buf.gen.yaml file. type externalBufGenYAMLFileV2 struct { Version string `json:"version,omitempty" yaml:"version,omitempty"` - Managed externalGenerateManagedConfigV2 `json:"managed,omitempty" yaml:"managed,omitempty"` Plugins []externalGeneratePluginConfigV2 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Managed externalGenerateManagedConfigV2 `json:"managed,omitempty" yaml:"managed,omitempty"` Inputs []externalInputConfigV2 `json:"inputs,omitempty" yaml:"inputs,omitempty"` } -// externalGenerateManagedConfigV2 is an external managed mode configuration. +// externalGeneratePluginConfigV2 represents a single plugin config in a v2 buf.gen.yaml file. +type externalGeneratePluginConfigV2 struct { + // Exactly one of Remote, Binary and ProtocBuiltin is required. + Remote *string `json:"remote,omitempty" yaml:"remote,omitempty"` + // Binary is the binary path, which can be one string or multiple strings. + Binary interface{} `json:"binary,omitempty" yaml:"binary,omitempty"` + // ProtocBuiltin is the protoc built-in plugin name, in the form of 'java' instead of 'protoc-gen-java'. + ProtocBuiltin *string `json:"protoc_builtin,omitempty" yaml:"protoc_builtin,omitempty"` + // Out is required. + Out string `json:"out,omitempty" yaml:"out,omitempty"` + // Revision is only valid with Remote set. + Revision *int `json:"revision,omitempty" yaml:"revision,omitempty"` + // ProtocPath is only valid with ProtocBuiltin + ProtocPath *string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` + // Opt can be one string or multiple strings. + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + IncludeImports bool `json:"include_imports,omitempty" yaml:"include_imports,omitempty"` + IncludeWKT bool `json:"include_wkt,omitempty" yaml:"include_wkt,omitempty"` + // Strategy 5s only valid with ProtoBuiltin and Binary + Strategy *string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalGenerateManagedConfigV2 represents the managed mode config in a v2 buf.gen.yaml file. type externalGenerateManagedConfigV2 struct { Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` Disable []externalManagedDisableConfigV2 `json:"disable,omitempty" yaml:"disable,omitempty"` Override []externalManagedOverrideConfigV2 `json:"override,omitempty" yaml:"override,omitempty"` } -// isEmpty returns true if the config is empty. -func (m externalGenerateManagedConfigV2) isEmpty() bool { - return !m.Enabled && len(m.Disable) == 0 && len(m.Override) == 0 -} - -// externalManagedDisableConfigV2 is an external configuration that disables file options in -// managed mode. +// externalManagedDisableConfigV2 represents a disable rule in managed mode in a v2 buf.gen.yaml file. type externalManagedDisableConfigV2 struct { + // At least one field must be set. // At most one of FileOption and FieldOption can be set - // Must be validated to be a valid FileOption - FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` - // Must be validated to be a valid FieldOption + FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` - // Must be validated to be a valid module path - Module string `json:"module,omitempty" yaml:"module,omitempty"` - // Must be normalized and validated + Module string `json:"module,omitempty" yaml:"module,omitempty"` + // Path must be normalized. Path string `json:"path,omitempty" yaml:"path,omitempty"` - // Must be validated to be a valid to be a valid field name. + // Field must not be set if FileOption is set. Field string `json:"field,omitempty" yaml:"field,omitempty"` } -// externalManagedOverrideConfigV2 is an external configuration that overrides file options in -// managed mode. +// externalManagedOverrideConfigV2 represents an override rule in managed mode in a v2 buf.gen.yaml file. type externalManagedOverrideConfigV2 struct { - // Must set exactly one of FileOption and FieldOption - // Must be validated to be a valid FileOption - FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` - // Must be validated to be a valid FieldOption + // Exactly one of FileOpion and FieldOption must be set. + FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` - // Must be validated to be a valid module path - Module string `json:"module,omitempty" yaml:"module,omitempty"` - // Must be normalized and validated + Module string `json:"module,omitempty" yaml:"module,omitempty"` + // Path must be normalized. Path string `json:"path,omitempty" yaml:"path,omitempty"` - // Must be validated to be a valid field name + // Field must not be set if FileOption is set. Field string `json:"field,omitempty" yaml:"field,omitempty"` - // Required + // Value is required Value interface{} `json:"value,omitempty" yaml:"value,omitempty"` } -// externalGeneratePluginConfigV2 is an external plugin configuration. -type externalGeneratePluginConfigV2 struct { - // Only one of Remote, Binary, Wasm, ProtocBuiltin can be set - Remote *string `json:"remote,omitempty" yaml:"remote,omitempty"` - // Can be multiple arguments - // All arguments must be strings - Binary interface{} `json:"binary,omitempty" yaml:"binary,omitempty"` - ProtocBuiltin *string `json:"protoc_builtin,omitempty" yaml:"protoc_builtin,omitempty"` - // Only valid with Remote - Revision *int `json:"revision,omitempty" yaml:"revision,omitempty"` - // Only valid with ProtocBuiltin - ProtocPath *string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` - // Required - Out string `json:"out,omitempty" yaml:"out,omitempty"` - // Can be one string or multiple strings - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` - IncludeImports bool `json:"include_imports,omitempty" yaml:"include_imports,omitempty"` - IncludeWKT bool `json:"include_wkt,omitempty" yaml:"include_wkt,omitempty"` - // Must be a valid Strategy, only valid with ProtoBuiltin and Binary - Strategy *string `json:"strategy,omitempty" yaml:"strategy,omitempty"` -} - // externalInputConfigV2 is an external input configuration. type externalInputConfigV2 struct { // One and only one of Module, Directory, ProtoFile, Tarball, ZipArchive, BinaryImage, @@ -339,8 +306,11 @@ type externalInputConfigV2 struct { JSONImage *string `json:"json_image,omitempty" yaml:"json_image,omitempty"` TextImage *string `json:"text_image,omitempty" yaml:"text_image,omitempty"` GitRepo *string `json:"git_repo,omitempty" yaml:"git_repo,omitempty"` - // Compression, StripComponents, Subdir, Branch, Tag, Ref, Depth, RecurseSubmodules - // and IncludePackageFils are available for only some formats. + // Types, IncludePaths and ExcludePaths are available for all formats. + Types []string `json:"types,omitempty" yaml:"types,omitempty"` + IncludePaths []string `json:"include_paths,omitempty" yaml:"include_paths,omitempty"` + ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` + // The following options are available depending on input format. Compression *string `json:"compression,omitempty" yaml:"compression,omitempty"` StripComponents *uint32 `json:"strip_components,omitempty" yaml:"strip_components,omitempty"` Subdir *string `json:"subdir,omitempty" yaml:"subdir,omitempty"` @@ -350,8 +320,4 @@ type externalInputConfigV2 struct { Depth *uint32 `json:"depth,omitempty" yaml:"depth,omitempty"` RecurseSubmodules *bool `json:"recurse_submodules,omitempty" yaml:"recurse_submodules,omitempty"` IncludePackageFiles *bool `json:"include_package_files,omitempty" yaml:"include_package_files,omitempty"` - // Types, IncludePaths and ExcludePaths are available for all formats. - Types []string `json:"types,omitempty" yaml:"types,omitempty"` - IncludePaths []string `json:"include_paths,omitempty" yaml:"include_paths,omitempty"` - ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` } diff --git a/private/bufpkg/bufconfig/generate_managed_config.go b/private/bufpkg/bufconfig/generate_managed_config.go index bdc370c96a..34cf6c848f 100644 --- a/private/bufpkg/bufconfig/generate_managed_config.go +++ b/private/bufpkg/bufconfig/generate_managed_config.go @@ -413,7 +413,7 @@ func newManagedConfigFromExternalV1( } func newManagedConfigFromExternalV1Beta1( - externalConfig externalOptionsConfigV1Beta1, + externalConfig externalGenerateManagedConfigV1Beta1, ) (GenerateManagedConfig, error) { var ( overrides []ManagedOverrideRule From 9ac1693818a6177ffbc0a73ddbdb5b5ef11e5845 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 17:13:19 -0500 Subject: [PATCH 52/66] generate_managed_config.go --- .../bufconfig/generate_managed_config.go | 355 ++++++++++-------- 1 file changed, 200 insertions(+), 155 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_managed_config.go b/private/bufpkg/bufconfig/generate_managed_config.go index 34cf6c848f..141a48170f 100644 --- a/private/bufpkg/bufconfig/generate_managed_config.go +++ b/private/bufpkg/bufconfig/generate_managed_config.go @@ -25,8 +25,6 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" ) -// TODO: check normalize(path) == path for disable and override paths. - // GenerateManagedConfig is a managed mode configuration. type GenerateManagedConfig interface { // Disables returns the disable rules in the configuration. @@ -86,33 +84,13 @@ func NewDisableRule( fileOption FileOption, fieldOption FieldOption, ) (ManagedDisableRule, error) { - if path != "" && normalpath.Normalize(path) != path { - // TODO: do we want to show words like 'normalized' to users? - return nil, fmt.Errorf("path must be normalized: %s", path) - } - if path == "" && moduleFullName == "" && fieldName == "" && fileOption == FileOptionUnspecified && fieldOption == FieldOptionUnspecified { - // This should never happen to parsing configs from provided by users. - return nil, errors.New("empty disable rule is not allowed") - } - if fieldName != "" && fileOption != FileOptionUnspecified { - return nil, errors.New("cannot disable a file option for a field") - } - if fileOption != FileOptionUnspecified && fieldOption != FieldOptionUnspecified { - return nil, errors.New("at most one of file_option and field_option can be specified") - } - // TODO: validate path here? Was it validated in v1/main? - if moduleFullName != "" { - if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { - return nil, err - } - } - return &managedDisableRule{ - path: path, - moduleFullName: moduleFullName, - fieldName: fieldName, - fileOption: fileOption, - fieldOption: fieldOption, - }, nil + return newDisableRule( + path, + moduleFullName, + fieldName, + fileOption, + fieldOption, + ) } // ManagedOverrideRule is an override rule. An override describes: @@ -150,33 +128,12 @@ func NewFileOptionOverrideRule( fileOption FileOption, value interface{}, ) (*managedOverrideRule, error) { - if path != "" && normalpath.Normalize(path) != path { - // TODO: do we want to show words like 'normalized' to users? - return nil, fmt.Errorf("path must be normalized: %s", path) - } - if moduleFullName != "" { - if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { - return nil, err - } - } - // All valid file options have a parse func. This lookup implicitly validates the option. - parseOverrideValueFunc, ok := fileOptionToParseOverrideValueFunc[fileOption] - if !ok { - return nil, fmt.Errorf("invalid fileOption: %v", fileOption) - } - if value == nil { - return nil, fmt.Errorf("value must be specified for override") - } - parsedValue, err := parseOverrideValueFunc(value) - if err != nil { - return nil, fmt.Errorf("invalid value %v for %v: %w", value, fileOption, err) - } - return &managedOverrideRule{ - path: path, - moduleFullName: moduleFullName, - fileOption: fileOption, - value: parsedValue, - }, nil + return newFileOptionOverrideRule( + path, + moduleFullName, + fileOption, + value, + ) } // NewFieldOptionOverrideRule returns an OverrideRule for a field option. @@ -187,42 +144,69 @@ func NewFieldOptionOverrideRule( fieldOption FieldOption, value interface{}, ) (ManagedOverrideRule, error) { - if path != "" && normalpath.Normalize(path) != path { - // TODO: do we want to show words like 'normalized' to users? - return nil, fmt.Errorf("path must be normalized: %s", path) - } - // TODO: validate path here? Was it validated in v1/main? - if moduleFullName != "" { - if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return newFieldOptionOverrideRule( + path, + moduleFullName, + fieldName, + fieldOption, + value, + ) +} + +// *** PRIVATE *** + +type generateManagedConfig struct { + disables []ManagedDisableRule + overrides []ManagedOverrideRule +} + +func newManagedConfigFromExternalV1Beta1( + externalConfig externalGenerateManagedConfigV1Beta1, +) (GenerateManagedConfig, error) { + var ( + overrides []ManagedOverrideRule + ) + if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { + override, err := NewFileOptionOverrideRule( + "", + "", + FileOptionCcEnableArenas, + *externalCCEnableArenas, + ) + if err != nil { return nil, err } + overrides = append(overrides, override) } - // All valid field options have a parse func. This lookup implicitly validates the option. - parseOverrideValueFunc, ok := fieldOptionToParseOverrideValueFunc[fieldOption] - if !ok { - return nil, fmt.Errorf("invalid fieldOption: %v", fieldOption) - } - if value == nil { - return nil, fmt.Errorf("value must be specified for override") + if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { + override, err := NewFileOptionOverrideRule( + "", + "", + FileOptionJavaMultipleFiles, + *externalJavaMultipleFiles, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, override) } - parsedValue, err := parseOverrideValueFunc(value) - if err != nil { - return nil, fmt.Errorf("invalid value %v for %v: %w", value, fieldOption, err) + if externalOptimizeFor := externalConfig.OptimizeFor; externalOptimizeFor != "" { + defaultOverride, err := NewFileOptionOverrideRule( + "", + "", + FileOptionOptimizeFor, + externalOptimizeFor, + ) + if err != nil { + return nil, err + } + overrides = append(overrides, defaultOverride) } - return &managedOverrideRule{ - path: path, - moduleFullName: moduleFullName, - fieldName: fieldName, - fieldOption: fieldOption, - value: parsedValue, + return &generateManagedConfig{ + overrides: overrides, }, nil } -type generateManagedConfig struct { - disables []ManagedDisableRule - overrides []ManagedOverrideRule -} - func newManagedConfigFromExternalV1( externalConfig externalGenerateManagedConfigV1, ) (GenerateManagedConfig, error) { @@ -271,9 +255,7 @@ func newManagedConfigFromExternalV1( } if externalJavaPackagePrefix := externalConfig.JavaPackagePrefix; !externalJavaPackagePrefix.isEmpty() { if externalJavaPackagePrefix.Default == "" { - // TODO: resolve this: this message has been updated, compared to the one in bufgen/config.go: - // "java_package_prefix setting requires a default value" - return nil, errors.New("java_package_prefix must have a default value") + return nil, errors.New("java_package_prefix requires a default value") } defaultOverride, err := NewFileOptionOverrideRule( "", @@ -312,7 +294,7 @@ func newManagedConfigFromExternalV1( } if externalOptimizeFor := externalConfig.OptimizeFor; !externalOptimizeFor.isEmpty() { if externalOptimizeFor.Default == "" { - return nil, errors.New("optimize_for must have a default value") + return nil, errors.New("optimize_for requires a default value") } defaultOverride, err := NewFileOptionOverrideRule( "", @@ -338,7 +320,7 @@ func newManagedConfigFromExternalV1( } if externalGoPackagePrefix := externalConfig.GoPackagePrefix; !externalGoPackagePrefix.isEmpty() { if externalGoPackagePrefix.Default == "" { - return nil, errors.New("go_package_prefix must have a default value") + return nil, errors.New("go_package_prefix requires a default value") } defaultOverride, err := NewFileOptionOverrideRule( "", @@ -412,61 +394,12 @@ func newManagedConfigFromExternalV1( }, nil } -func newManagedConfigFromExternalV1Beta1( - externalConfig externalGenerateManagedConfigV1Beta1, -) (GenerateManagedConfig, error) { - var ( - overrides []ManagedOverrideRule - ) - if externalCCEnableArenas := externalConfig.CcEnableArenas; externalCCEnableArenas != nil { - override, err := NewFileOptionOverrideRule( - "", - "", - FileOptionCcEnableArenas, - *externalCCEnableArenas, - ) - if err != nil { - return nil, err - } - overrides = append(overrides, override) - } - if externalJavaMultipleFiles := externalConfig.JavaMultipleFiles; externalJavaMultipleFiles != nil { - override, err := NewFileOptionOverrideRule( - "", - "", - FileOptionJavaMultipleFiles, - *externalJavaMultipleFiles, - ) - if err != nil { - return nil, err - } - overrides = append(overrides, override) - } - if externalOptimizeFor := externalConfig.OptimizeFor; externalOptimizeFor != "" { - defaultOverride, err := NewFileOptionOverrideRule( - "", - "", - FileOptionOptimizeFor, - externalOptimizeFor, - ) - if err != nil { - return nil, err - } - overrides = append(overrides, defaultOverride) - } - return &generateManagedConfig{ - overrides: overrides, - }, nil -} - func newManagedConfigFromExternalV2( externalConfig externalGenerateManagedConfigV2, ) (GenerateManagedConfig, error) { - // TODO: add test case for non-empty config but disabled if !externalConfig.Enabled { return nil, nil } - // TODO: log warning if disabled but non-empty var disables []ManagedDisableRule var overrides []ManagedOverrideRule for _, externalDisableConfig := range externalConfig.Disable { @@ -487,7 +420,7 @@ func newManagedConfigFromExternalV2( return nil, err } } - disable, err := NewDisableRule( + disable, err := newDisableRule( externalDisableConfig.Path, externalDisableConfig.Module, externalDisableConfig.Field, @@ -566,6 +499,41 @@ type managedDisableRule struct { fieldOption FieldOption } +func newDisableRule( + path string, + moduleFullName string, + fieldName string, + fileOption FileOption, + fieldOption FieldOption, +) (ManagedDisableRule, error) { + if path == "" && moduleFullName == "" && fieldName == "" && fileOption == FileOptionUnspecified && fieldOption == FieldOptionUnspecified { + return nil, errors.New("empty disable rule is not allowed") + } + if fieldName != "" && fileOption != FileOptionUnspecified { + return nil, errors.New("cannot disable a file option for a field") + } + if fileOption != FileOptionUnspecified && fieldOption != FieldOptionUnspecified { + return nil, errors.New("at most one of file_option and field_option can be specified") + } + if path != "" { + if err := validatePath(path); err != nil { + return nil, fmt.Errorf("invalid path for disable rule: %w", err) + } + } + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, err + } + } + return &managedDisableRule{ + path: path, + moduleFullName: moduleFullName, + fieldName: fieldName, + fileOption: fileOption, + fieldOption: fieldOption, + }, nil +} + func (m *managedDisableRule) Path() string { return m.path } @@ -597,6 +565,80 @@ type managedOverrideRule struct { value interface{} } +func newFileOptionOverrideRule( + path string, + moduleFullName string, + fileOption FileOption, + value interface{}, +) (*managedOverrideRule, error) { + // All valid file options have a parse func. This lookup implicitly validates the option. + parseOverrideValueFunc, ok := fileOptionToParseOverrideValueFunc[fileOption] + if !ok { + return nil, fmt.Errorf("invalid fileOption: %v", fileOption) + } + if value == nil { + return nil, fmt.Errorf("value must be specified for override") + } + parsedValue, err := parseOverrideValueFunc(value) + if err != nil { + return nil, fmt.Errorf("invalid value %v for %v: %w", value, fileOption, err) + } + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, fmt.Errorf("invalid module name for %v override: %w", fileOption, err) + } + } + if path != "" { + if err := validatePath(path); err != nil { + return nil, fmt.Errorf("invalid path for %v override: %w", fileOption, err) + } + } + return &managedOverrideRule{ + path: path, + moduleFullName: moduleFullName, + fileOption: fileOption, + value: parsedValue, + }, nil +} + +func newFieldOptionOverrideRule( + path string, + moduleFullName string, + fieldName string, + fieldOption FieldOption, + value interface{}, +) (ManagedOverrideRule, error) { + // All valid field options have a parse func. This lookup implicitly validates the option. + parseOverrideValueFunc, ok := fieldOptionToParseOverrideValueFunc[fieldOption] + if !ok { + return nil, fmt.Errorf("invalid fieldOption: %v", fieldOption) + } + if value == nil { + return nil, fmt.Errorf("value must be specified for override") + } + parsedValue, err := parseOverrideValueFunc(value) + if err != nil { + return nil, fmt.Errorf("invalid value %v for %v: %w", value, fieldOption, err) + } + if moduleFullName != "" { + if _, err := bufmodule.ParseModuleFullName(moduleFullName); err != nil { + return nil, fmt.Errorf("invalid module name for %v override: %w", fieldOption, err) + } + } + if path != "" { + if err := validatePath(path); err != nil { + return nil, fmt.Errorf("invalid path for %v override: %w", fieldOption, err) + } + } + return &managedOverrideRule{ + path: path, + moduleFullName: moduleFullName, + fieldName: fieldName, + fieldOption: fieldOption, + value: parsedValue, + }, nil +} + func (m *managedOverrideRule) Path() string { return m.path } @@ -642,7 +684,7 @@ func disablesAndOverridesFromExceptAndOverrideV1( return nil, nil, fmt.Errorf("%q is defined multiple times in except", exceptModuleFullName) } seenExceptModuleFullNames[exceptModuleFullName] = struct{}{} - disable, err := NewDisableRule( + disable, err := newDisableRule( "", exceptModuleFullName, "", @@ -690,22 +732,9 @@ func overrideRulesForPerFileOverridesV1( filePathToOverride := fileOptionToFilePathToOverride[fileOptionString] sortedFilePaths := slicesext.MapKeysToSortedSlice(filePathToOverride) for _, filePath := range sortedFilePaths { - normalizedFilePath, err := normalpath.NormalizeAndValidate(filePath) + err := validatePath(filePath) if err != nil { - return nil, fmt.Errorf( - "%s for override %s is not a valid import path: %w", - filePath, - fileOptionString, - err, - ) - } - if filePath != normalizedFilePath { - return nil, fmt.Errorf( - "import path %s for override %s is not normalized, use %s instead", - filePath, - fileOptionString, - normalizedFilePath, - ) + return nil, fmt.Errorf("invalid import path for override %s: %w", fileOptionString, err) } overrideString := filePathToOverride[filePath] var overrideValue interface{} = overrideString @@ -786,3 +815,19 @@ func newExternalManagedConfigV2FromGenerateManagedConfig( Override: externalOverrides, } } + +func validatePath(path string) error { + normalizedPath, err := normalpath.NormalizeAndValidate(path) + if err != nil { + return err + } + if path != normalizedPath { + return fmt.Errorf( + // TODO: do we want to show the word 'normalized' to users? + "%q is not normalized, use %q instead", + path, + normalizedPath, + ) + } + return nil +} From 1acc453d385b2f3a932302f9b4074ce2b79eb9c8 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 17:16:49 -0500 Subject: [PATCH 53/66] move external config to buf_gen_yaml_file.go --- private/bufpkg/bufconfig/buf_gen_yaml_file.go | 305 +++++++++++++++++ .../bufconfig/generate_external_config.go | 323 ------------------ 2 files changed, 305 insertions(+), 323 deletions(-) delete mode 100644 private/bufpkg/bufconfig/generate_external_config.go diff --git a/private/bufpkg/bufconfig/buf_gen_yaml_file.go b/private/bufpkg/bufconfig/buf_gen_yaml_file.go index abf48cf459..598edf50b8 100644 --- a/private/bufpkg/bufconfig/buf_gen_yaml_file.go +++ b/private/bufpkg/bufconfig/buf_gen_yaml_file.go @@ -16,6 +16,7 @@ package bufconfig import ( "context" + "encoding/json" "fmt" "io" @@ -236,3 +237,307 @@ func writeBufGenYAMLFile(writer io.Writer, bufGenYAMLFile BufGenYAMLFile) error _, err = writer.Write(data) return err } + +// externalBufGenYAMLFileV1Beta1 represents the v1beta buf.gen.yaml file. +type externalBufGenYAMLFileV1Beta1 struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` + // Managed is whether managed mode is enabled. + Managed bool `json:"managed,omitempty" yaml:"managed,omitempty"` + Plugins []externalGeneratePluginConfigV1Beta1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Options externalGenerateManagedConfigV1Beta1 `json:"options,omitempty" yaml:"options,omitempty"` +} + +// externalGeneratePluginConfigV1Beta1 represents a single plugin conifg in a v1beta1 buf.gen.yaml file. +type externalGeneratePluginConfigV1Beta1 struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + Out string `json:"out,omitempty" yaml:"out,omitempty"` + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + Path string `json:"path,omitempty" yaml:"path,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalGenerateManagedConfigV1Beta1 represents the options (for managed mode) config in a v1beta1 buf.gen.yaml file. +type externalGenerateManagedConfigV1Beta1 struct { + CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` + JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` + OptimizeFor string `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` +} + +// externalBufGenYAMLFileV1 represents the v1 buf.gen.yaml file. +type externalBufGenYAMLFileV1 struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Plugins []externalGeneratePluginConfigV1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Managed externalGenerateManagedConfigV1 `json:"managed,omitempty" yaml:"managed,omitempty"` + Types externalTypesConfigV1 `json:"types,omitempty" yaml:"types,omitempty"` +} + +// externalGeneratePluginConfigV1 represents a single plugin config in a v1 buf.gen.yaml file. +type externalGeneratePluginConfigV1 struct { + // Exactly one of Plugin and Name is required. + // Plugin is the key for a local or remote plugin. + Plugin string `json:"plugin,omitempty" yaml:"plugin,omitempty"` + // Name is the key for a local plugin. + Name string `json:"name,omitempty" yaml:"name,omitempty"` + // Remote is the key for alpha remote plugin name, which is deprecated now. + Remote string `json:"remote,omitempty" yaml:"remote,omitempty"` + // Out is required. + Out string `json:"out,omitempty" yaml:"out,omitempty"` + Revision int `json:"revision,omitempty" yaml:"revision,omitempty"` + // Opt can be one string or multiple strings. + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + // Path can be one string or multiple strings. + Path interface{} `json:"path,omitempty" yaml:"path,omitempty"` + ProtocPath string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` + Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalGenerateManagedConfigV1 represents the managed mode config in a v1 buf.gen.yaml file. +type externalGenerateManagedConfigV1 struct { + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` + JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` + JavaStringCheckUtf8 *bool `json:"java_string_check_utf8,omitempty" yaml:"java_string_check_utf8,omitempty"` + JavaPackagePrefix externalJavaPackagePrefixConfigV1 `json:"java_package_prefix,omitempty" yaml:"java_package_prefix,omitempty"` + CsharpNamespace externalCsharpNamespaceConfigV1 `json:"csharp_namespace,omitempty" yaml:"csharp_namespace,omitempty"` + OptimizeFor externalOptimizeForConfigV1 `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` + GoPackagePrefix externalGoPackagePrefixConfigV1 `json:"go_package_prefix,omitempty" yaml:"go_package_prefix,omitempty"` + ObjcClassPrefix externalObjcClassPrefixConfigV1 `json:"objc_class_prefix,omitempty" yaml:"objc_class_prefix,omitempty"` + RubyPackage externalRubyPackageConfigV1 `json:"ruby_package,omitempty" yaml:"ruby_package,omitempty"` + // Override maps from a file option to a file path then to the value. + Override map[string]map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// externalJavaPackagePrefixConfigV1 represents the java_package_prefix config in a v1 buf.gen.yaml file. +type externalJavaPackagePrefixConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for java_package_prefix. +func (e *externalJavaPackagePrefixConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { + return e.unmarshalWith(unmarshal) +} + +// UnmarshalJSON implements the json.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for java_package_prefix. +func (e *externalJavaPackagePrefixConfigV1) UnmarshalJSON(data []byte) error { + unmarshal := func(v interface{}) error { + return json.Unmarshal(data, v) + } + return e.unmarshalWith(unmarshal) +} + +// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. +func (e *externalJavaPackagePrefixConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { + var prefix string + if err := unmarshal(&prefix); err == nil { + e.Default = prefix + return nil + } + type rawExternalJavaPackagePrefixConfigV1 externalJavaPackagePrefixConfigV1 + if err := unmarshal((*rawExternalJavaPackagePrefixConfigV1)(e)); err != nil { + return err + } + return nil +} + +// isEmpty returns true if the config is empty. +func (e externalJavaPackagePrefixConfigV1) isEmpty() bool { + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalOptimizeForConfigV1 represents the optimize_for config in a v1 buf.gen.yaml file. +type externalOptimizeForConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for optimize_for. +func (e *externalOptimizeForConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { + return e.unmarshalWith(unmarshal) +} + +// UnmarshalJSON implements the json.Unmarshaler interface. This is done to maintain backward compatibility +// of accepting a plain string value for optimize_for. +func (e *externalOptimizeForConfigV1) UnmarshalJSON(data []byte) error { + unmarshal := func(v interface{}) error { + return json.Unmarshal(data, v) + } + return e.unmarshalWith(unmarshal) +} + +// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. +func (e *externalOptimizeForConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { + var optimizeFor string + if err := unmarshal(&optimizeFor); err == nil { + e.Default = optimizeFor + return nil + } + type rawExternalOptimizeForConfigV1 externalOptimizeForConfigV1 + if err := unmarshal((*rawExternalOptimizeForConfigV1)(e)); err != nil { + return err + } + return nil +} + +// isEmpty returns true if the config is empty +func (e externalOptimizeForConfigV1) isEmpty() bool { // TODO: does it need to be public? + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalGoPackagePrefixConfigV1 represents the go_package_prefix config in a v1 buf.gen.yaml file. +type externalGoPackagePrefixConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty. +func (e externalGoPackagePrefixConfigV1) isEmpty() bool { + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalCsharpNamespaceConfigV1 represents the external csharp_namespace config in a v1 buf.gen.yaml file. +type externalCsharpNamespaceConfigV1 struct { + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true if the config is empty. +func (e externalCsharpNamespaceConfigV1) isEmpty() bool { + return len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalRubyPackageConfigV1 represents the ruby_package config in a v1 buf.gen.yaml file. +type externalRubyPackageConfigV1 struct { + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true is the config is empty. +func (e externalRubyPackageConfigV1) isEmpty() bool { // TODO: does this need to be public? same with other IsEmpty() + return len(e.Except) == 0 && len(e.Override) == 0 +} + +// externalObjcClassPrefixConfigV1 represents the objc_class_prefix config in a v1 buf.gen.yaml file. +type externalObjcClassPrefixConfigV1 struct { + Default string `json:"default,omitempty" yaml:"default,omitempty"` + Except []string `json:"except,omitempty" yaml:"except,omitempty"` + Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` +} + +// isEmpty returns true is the config is empty. +func (e externalObjcClassPrefixConfigV1) isEmpty() bool { + return e.Default == "" && + len(e.Except) == 0 && + len(e.Override) == 0 +} + +// externalTypesConfigV1 represents the types config in a v1 buf.gen.yaml file. +type externalTypesConfigV1 struct { + Include []string `json:"include,omitempty" yaml:"include"` +} + +// externalBufGenYAMLFileV2 represents the v2 buf.gen.yaml file. +type externalBufGenYAMLFileV2 struct { + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Plugins []externalGeneratePluginConfigV2 `json:"plugins,omitempty" yaml:"plugins,omitempty"` + Managed externalGenerateManagedConfigV2 `json:"managed,omitempty" yaml:"managed,omitempty"` + Inputs []externalInputConfigV2 `json:"inputs,omitempty" yaml:"inputs,omitempty"` +} + +// externalGeneratePluginConfigV2 represents a single plugin config in a v2 buf.gen.yaml file. +type externalGeneratePluginConfigV2 struct { + // Exactly one of Remote, Binary and ProtocBuiltin is required. + Remote *string `json:"remote,omitempty" yaml:"remote,omitempty"` + // Binary is the binary path, which can be one string or multiple strings. + Binary interface{} `json:"binary,omitempty" yaml:"binary,omitempty"` + // ProtocBuiltin is the protoc built-in plugin name, in the form of 'java' instead of 'protoc-gen-java'. + ProtocBuiltin *string `json:"protoc_builtin,omitempty" yaml:"protoc_builtin,omitempty"` + // Out is required. + Out string `json:"out,omitempty" yaml:"out,omitempty"` + // Revision is only valid with Remote set. + Revision *int `json:"revision,omitempty" yaml:"revision,omitempty"` + // ProtocPath is only valid with ProtocBuiltin + ProtocPath *string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` + // Opt can be one string or multiple strings. + Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` + IncludeImports bool `json:"include_imports,omitempty" yaml:"include_imports,omitempty"` + IncludeWKT bool `json:"include_wkt,omitempty" yaml:"include_wkt,omitempty"` + // Strategy 5s only valid with ProtoBuiltin and Binary + Strategy *string `json:"strategy,omitempty" yaml:"strategy,omitempty"` +} + +// externalGenerateManagedConfigV2 represents the managed mode config in a v2 buf.gen.yaml file. +type externalGenerateManagedConfigV2 struct { + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` + Disable []externalManagedDisableConfigV2 `json:"disable,omitempty" yaml:"disable,omitempty"` + Override []externalManagedOverrideConfigV2 `json:"override,omitempty" yaml:"override,omitempty"` +} + +// externalManagedDisableConfigV2 represents a disable rule in managed mode in a v2 buf.gen.yaml file. +type externalManagedDisableConfigV2 struct { + // At least one field must be set. + // At most one of FileOption and FieldOption can be set + FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` + FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` + Module string `json:"module,omitempty" yaml:"module,omitempty"` + // Path must be normalized. + Path string `json:"path,omitempty" yaml:"path,omitempty"` + // Field must not be set if FileOption is set. + Field string `json:"field,omitempty" yaml:"field,omitempty"` +} + +// externalManagedOverrideConfigV2 represents an override rule in managed mode in a v2 buf.gen.yaml file. +type externalManagedOverrideConfigV2 struct { + // Exactly one of FileOpion and FieldOption must be set. + FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` + FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` + Module string `json:"module,omitempty" yaml:"module,omitempty"` + // Path must be normalized. + Path string `json:"path,omitempty" yaml:"path,omitempty"` + // Field must not be set if FileOption is set. + Field string `json:"field,omitempty" yaml:"field,omitempty"` + // Value is required + Value interface{} `json:"value,omitempty" yaml:"value,omitempty"` +} + +// externalInputConfigV2 is an external input configuration. +type externalInputConfigV2 struct { + // One and only one of Module, Directory, ProtoFile, Tarball, ZipArchive, BinaryImage, + // JSONImage and GitRepo must be specified as the format. + Module *string `json:"module,omitempty" yaml:"module,omitempty"` + Directory *string `json:"directory,omitempty" yaml:"directory,omitempty"` + ProtoFile *string `json:"proto_file,omitempty" yaml:"proto_file,omitempty"` + Tarball *string `json:"tarball,omitempty" yaml:"tarball,omitempty"` + ZipArchive *string `json:"zip_archive,omitempty" yaml:"zip_archive,omitempty"` + BinaryImage *string `json:"binary_image,omitempty" yaml:"binary_image,omitempty"` + JSONImage *string `json:"json_image,omitempty" yaml:"json_image,omitempty"` + TextImage *string `json:"text_image,omitempty" yaml:"text_image,omitempty"` + GitRepo *string `json:"git_repo,omitempty" yaml:"git_repo,omitempty"` + // Types, IncludePaths and ExcludePaths are available for all formats. + Types []string `json:"types,omitempty" yaml:"types,omitempty"` + IncludePaths []string `json:"include_paths,omitempty" yaml:"include_paths,omitempty"` + ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` + // The following options are available depending on input format. + Compression *string `json:"compression,omitempty" yaml:"compression,omitempty"` + StripComponents *uint32 `json:"strip_components,omitempty" yaml:"strip_components,omitempty"` + Subdir *string `json:"subdir,omitempty" yaml:"subdir,omitempty"` + Branch *string `json:"branch,omitempty" yaml:"branch,omitempty"` + Tag *string `json:"tag,omitempty" yaml:"tag,omitempty"` + Ref *string `json:"ref,omitempty" yaml:"ref,omitempty"` + Depth *uint32 `json:"depth,omitempty" yaml:"depth,omitempty"` + RecurseSubmodules *bool `json:"recurse_submodules,omitempty" yaml:"recurse_submodules,omitempty"` + IncludePackageFiles *bool `json:"include_package_files,omitempty" yaml:"include_package_files,omitempty"` +} diff --git a/private/bufpkg/bufconfig/generate_external_config.go b/private/bufpkg/bufconfig/generate_external_config.go deleted file mode 100644 index 4e0565a263..0000000000 --- a/private/bufpkg/bufconfig/generate_external_config.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufconfig - -import "encoding/json" - -// TODO: this is a temporary file to avoid crowing other files. We can choose to move stuff from this file over. - -// externalBufGenYAMLFileV1Beta1 represents the v1beta buf.gen.yaml file. -type externalBufGenYAMLFileV1Beta1 struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` - // Managed is whether managed mode is enabled. - Managed bool `json:"managed,omitempty" yaml:"managed,omitempty"` - Plugins []externalGeneratePluginConfigV1Beta1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` - Options externalGenerateManagedConfigV1Beta1 `json:"options,omitempty" yaml:"options,omitempty"` -} - -// externalGeneratePluginConfigV1Beta1 represents a single plugin conifg in a v1beta1 buf.gen.yaml file. -type externalGeneratePluginConfigV1Beta1 struct { - Name string `json:"name,omitempty" yaml:"name,omitempty"` - Out string `json:"out,omitempty" yaml:"out,omitempty"` - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` - Path string `json:"path,omitempty" yaml:"path,omitempty"` - Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` -} - -// externalGenerateManagedConfigV1Beta1 represents the options (for managed mode) config in a v1beta1 buf.gen.yaml file. -type externalGenerateManagedConfigV1Beta1 struct { - CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` - JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` - OptimizeFor string `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` -} - -// externalBufGenYAMLFileV1 represents the v1 buf.gen.yaml file. -type externalBufGenYAMLFileV1 struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` - Plugins []externalGeneratePluginConfigV1 `json:"plugins,omitempty" yaml:"plugins,omitempty"` - Managed externalGenerateManagedConfigV1 `json:"managed,omitempty" yaml:"managed,omitempty"` - Types externalTypesConfigV1 `json:"types,omitempty" yaml:"types,omitempty"` -} - -// externalGeneratePluginConfigV1 represents a single plugin config in a v1 buf.gen.yaml file. -type externalGeneratePluginConfigV1 struct { - // Exactly one of Plugin and Name is required. - // Plugin is the key for a local or remote plugin. - Plugin string `json:"plugin,omitempty" yaml:"plugin,omitempty"` - // Name is the key for a local plugin. - Name string `json:"name,omitempty" yaml:"name,omitempty"` - // Remote is the key for alpha remote plugin name, which is deprecated now. - Remote string `json:"remote,omitempty" yaml:"remote,omitempty"` - // Out is required. - Out string `json:"out,omitempty" yaml:"out,omitempty"` - Revision int `json:"revision,omitempty" yaml:"revision,omitempty"` - // Opt can be one string or multiple strings. - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` - // Path can be one string or multiple strings. - Path interface{} `json:"path,omitempty" yaml:"path,omitempty"` - ProtocPath string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` - Strategy string `json:"strategy,omitempty" yaml:"strategy,omitempty"` -} - -// externalGenerateManagedConfigV1 represents the managed mode config in a v1 buf.gen.yaml file. -type externalGenerateManagedConfigV1 struct { - Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` - CcEnableArenas *bool `json:"cc_enable_arenas,omitempty" yaml:"cc_enable_arenas,omitempty"` - JavaMultipleFiles *bool `json:"java_multiple_files,omitempty" yaml:"java_multiple_files,omitempty"` - JavaStringCheckUtf8 *bool `json:"java_string_check_utf8,omitempty" yaml:"java_string_check_utf8,omitempty"` - JavaPackagePrefix externalJavaPackagePrefixConfigV1 `json:"java_package_prefix,omitempty" yaml:"java_package_prefix,omitempty"` - CsharpNamespace externalCsharpNamespaceConfigV1 `json:"csharp_namespace,omitempty" yaml:"csharp_namespace,omitempty"` - OptimizeFor externalOptimizeForConfigV1 `json:"optimize_for,omitempty" yaml:"optimize_for,omitempty"` - GoPackagePrefix externalGoPackagePrefixConfigV1 `json:"go_package_prefix,omitempty" yaml:"go_package_prefix,omitempty"` - ObjcClassPrefix externalObjcClassPrefixConfigV1 `json:"objc_class_prefix,omitempty" yaml:"objc_class_prefix,omitempty"` - RubyPackage externalRubyPackageConfigV1 `json:"ruby_package,omitempty" yaml:"ruby_package,omitempty"` - // Override maps from a file option to a file path then to the value. - Override map[string]map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// externalJavaPackagePrefixConfigV1 represents the java_package_prefix config in a v1 buf.gen.yaml file. -type externalJavaPackagePrefixConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for java_package_prefix. -func (e *externalJavaPackagePrefixConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { - return e.unmarshalWith(unmarshal) -} - -// UnmarshalJSON implements the json.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for java_package_prefix. -func (e *externalJavaPackagePrefixConfigV1) UnmarshalJSON(data []byte) error { - unmarshal := func(v interface{}) error { - return json.Unmarshal(data, v) - } - return e.unmarshalWith(unmarshal) -} - -// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. -func (e *externalJavaPackagePrefixConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { - var prefix string - if err := unmarshal(&prefix); err == nil { - e.Default = prefix - return nil - } - type rawExternalJavaPackagePrefixConfigV1 externalJavaPackagePrefixConfigV1 - if err := unmarshal((*rawExternalJavaPackagePrefixConfigV1)(e)); err != nil { - return err - } - return nil -} - -// isEmpty returns true if the config is empty. -func (e externalJavaPackagePrefixConfigV1) isEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// externalOptimizeForConfigV1 represents the optimize_for config in a v1 buf.gen.yaml file. -type externalOptimizeForConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for optimize_for. -func (e *externalOptimizeForConfigV1) UnmarshalYAML(unmarshal func(interface{}) error) error { - return e.unmarshalWith(unmarshal) -} - -// UnmarshalJSON implements the json.Unmarshaler interface. This is done to maintain backward compatibility -// of accepting a plain string value for optimize_for. -func (e *externalOptimizeForConfigV1) UnmarshalJSON(data []byte) error { - unmarshal := func(v interface{}) error { - return json.Unmarshal(data, v) - } - return e.unmarshalWith(unmarshal) -} - -// unmarshalWith is used to unmarshal into json/yaml. See https://abhinavg.net/posts/flexible-yaml for details. -func (e *externalOptimizeForConfigV1) unmarshalWith(unmarshal func(interface{}) error) error { - var optimizeFor string - if err := unmarshal(&optimizeFor); err == nil { - e.Default = optimizeFor - return nil - } - type rawExternalOptimizeForConfigV1 externalOptimizeForConfigV1 - if err := unmarshal((*rawExternalOptimizeForConfigV1)(e)); err != nil { - return err - } - return nil -} - -// isEmpty returns true if the config is empty -func (e externalOptimizeForConfigV1) isEmpty() bool { // TODO: does it need to be public? - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// externalGoPackagePrefixConfigV1 represents the go_package_prefix config in a v1 buf.gen.yaml file. -type externalGoPackagePrefixConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// isEmpty returns true if the config is empty. -func (e externalGoPackagePrefixConfigV1) isEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// externalCsharpNamespaceConfigV1 represents the external csharp_namespace config in a v1 buf.gen.yaml file. -type externalCsharpNamespaceConfigV1 struct { - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// isEmpty returns true if the config is empty. -func (e externalCsharpNamespaceConfigV1) isEmpty() bool { - return len(e.Except) == 0 && - len(e.Override) == 0 -} - -// externalRubyPackageConfigV1 represents the ruby_package config in a v1 buf.gen.yaml file. -type externalRubyPackageConfigV1 struct { - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// isEmpty returns true is the config is empty. -func (e externalRubyPackageConfigV1) isEmpty() bool { // TODO: does this need to be public? same with other IsEmpty() - return len(e.Except) == 0 && len(e.Override) == 0 -} - -// externalObjcClassPrefixConfigV1 represents the objc_class_prefix config in a v1 buf.gen.yaml file. -type externalObjcClassPrefixConfigV1 struct { - Default string `json:"default,omitempty" yaml:"default,omitempty"` - Except []string `json:"except,omitempty" yaml:"except,omitempty"` - Override map[string]string `json:"override,omitempty" yaml:"override,omitempty"` -} - -// isEmpty returns true is the config is empty. -func (e externalObjcClassPrefixConfigV1) isEmpty() bool { - return e.Default == "" && - len(e.Except) == 0 && - len(e.Override) == 0 -} - -// externalTypesConfigV1 represents the types config in a v1 buf.gen.yaml file. -type externalTypesConfigV1 struct { - Include []string `json:"include,omitempty" yaml:"include"` -} - -// externalBufGenYAMLFileV2 represents the v2 buf.gen.yaml file. -type externalBufGenYAMLFileV2 struct { - Version string `json:"version,omitempty" yaml:"version,omitempty"` - Plugins []externalGeneratePluginConfigV2 `json:"plugins,omitempty" yaml:"plugins,omitempty"` - Managed externalGenerateManagedConfigV2 `json:"managed,omitempty" yaml:"managed,omitempty"` - Inputs []externalInputConfigV2 `json:"inputs,omitempty" yaml:"inputs,omitempty"` -} - -// externalGeneratePluginConfigV2 represents a single plugin config in a v2 buf.gen.yaml file. -type externalGeneratePluginConfigV2 struct { - // Exactly one of Remote, Binary and ProtocBuiltin is required. - Remote *string `json:"remote,omitempty" yaml:"remote,omitempty"` - // Binary is the binary path, which can be one string or multiple strings. - Binary interface{} `json:"binary,omitempty" yaml:"binary,omitempty"` - // ProtocBuiltin is the protoc built-in plugin name, in the form of 'java' instead of 'protoc-gen-java'. - ProtocBuiltin *string `json:"protoc_builtin,omitempty" yaml:"protoc_builtin,omitempty"` - // Out is required. - Out string `json:"out,omitempty" yaml:"out,omitempty"` - // Revision is only valid with Remote set. - Revision *int `json:"revision,omitempty" yaml:"revision,omitempty"` - // ProtocPath is only valid with ProtocBuiltin - ProtocPath *string `json:"protoc_path,omitempty" yaml:"protoc_path,omitempty"` - // Opt can be one string or multiple strings. - Opt interface{} `json:"opt,omitempty" yaml:"opt,omitempty"` - IncludeImports bool `json:"include_imports,omitempty" yaml:"include_imports,omitempty"` - IncludeWKT bool `json:"include_wkt,omitempty" yaml:"include_wkt,omitempty"` - // Strategy 5s only valid with ProtoBuiltin and Binary - Strategy *string `json:"strategy,omitempty" yaml:"strategy,omitempty"` -} - -// externalGenerateManagedConfigV2 represents the managed mode config in a v2 buf.gen.yaml file. -type externalGenerateManagedConfigV2 struct { - Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` - Disable []externalManagedDisableConfigV2 `json:"disable,omitempty" yaml:"disable,omitempty"` - Override []externalManagedOverrideConfigV2 `json:"override,omitempty" yaml:"override,omitempty"` -} - -// externalManagedDisableConfigV2 represents a disable rule in managed mode in a v2 buf.gen.yaml file. -type externalManagedDisableConfigV2 struct { - // At least one field must be set. - // At most one of FileOption and FieldOption can be set - FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` - FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` - Module string `json:"module,omitempty" yaml:"module,omitempty"` - // Path must be normalized. - Path string `json:"path,omitempty" yaml:"path,omitempty"` - // Field must not be set if FileOption is set. - Field string `json:"field,omitempty" yaml:"field,omitempty"` -} - -// externalManagedOverrideConfigV2 represents an override rule in managed mode in a v2 buf.gen.yaml file. -type externalManagedOverrideConfigV2 struct { - // Exactly one of FileOpion and FieldOption must be set. - FileOption string `json:"file_option,omitempty" yaml:"file_option,omitempty"` - FieldOption string `json:"field_option,omitempty" yaml:"field_option,omitempty"` - Module string `json:"module,omitempty" yaml:"module,omitempty"` - // Path must be normalized. - Path string `json:"path,omitempty" yaml:"path,omitempty"` - // Field must not be set if FileOption is set. - Field string `json:"field,omitempty" yaml:"field,omitempty"` - // Value is required - Value interface{} `json:"value,omitempty" yaml:"value,omitempty"` -} - -// externalInputConfigV2 is an external input configuration. -type externalInputConfigV2 struct { - // One and only one of Module, Directory, ProtoFile, Tarball, ZipArchive, BinaryImage, - // JSONImage and GitRepo must be specified as the format. - Module *string `json:"module,omitempty" yaml:"module,omitempty"` - Directory *string `json:"directory,omitempty" yaml:"directory,omitempty"` - ProtoFile *string `json:"proto_file,omitempty" yaml:"proto_file,omitempty"` - Tarball *string `json:"tarball,omitempty" yaml:"tarball,omitempty"` - ZipArchive *string `json:"zip_archive,omitempty" yaml:"zip_archive,omitempty"` - BinaryImage *string `json:"binary_image,omitempty" yaml:"binary_image,omitempty"` - JSONImage *string `json:"json_image,omitempty" yaml:"json_image,omitempty"` - TextImage *string `json:"text_image,omitempty" yaml:"text_image,omitempty"` - GitRepo *string `json:"git_repo,omitempty" yaml:"git_repo,omitempty"` - // Types, IncludePaths and ExcludePaths are available for all formats. - Types []string `json:"types,omitempty" yaml:"types,omitempty"` - IncludePaths []string `json:"include_paths,omitempty" yaml:"include_paths,omitempty"` - ExcludePaths []string `json:"exclude_paths,omitempty" yaml:"exclude_paths,omitempty"` - // The following options are available depending on input format. - Compression *string `json:"compression,omitempty" yaml:"compression,omitempty"` - StripComponents *uint32 `json:"strip_components,omitempty" yaml:"strip_components,omitempty"` - Subdir *string `json:"subdir,omitempty" yaml:"subdir,omitempty"` - Branch *string `json:"branch,omitempty" yaml:"branch,omitempty"` - Tag *string `json:"tag,omitempty" yaml:"tag,omitempty"` - Ref *string `json:"ref,omitempty" yaml:"ref,omitempty"` - Depth *uint32 `json:"depth,omitempty" yaml:"depth,omitempty"` - RecurseSubmodules *bool `json:"recurse_submodules,omitempty" yaml:"recurse_submodules,omitempty"` - IncludePackageFiles *bool `json:"include_package_files,omitempty" yaml:"include_package_files,omitempty"` -} From 4005a08f328667e3d9abd5c6869996ac62a6f33d Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 17:19:33 -0500 Subject: [PATCH 54/66] generate_managed_option.go --- .../bufconfig/generate_managed_option.go | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_managed_option.go b/private/bufpkg/bufconfig/generate_managed_option.go index e08e13ad34..7ddd2a9794 100644 --- a/private/bufpkg/bufconfig/generate_managed_option.go +++ b/private/bufpkg/bufconfig/generate_managed_option.go @@ -67,6 +67,15 @@ const ( FileOptionRubyPackageSuffix ) +// String implements fmt.Stringer. +func (f FileOption) String() string { + s, ok := fileOptionToString[f] + if !ok { + return strconv.Itoa(int(f)) + } + return s +} + // FieldOption is a field option. type FieldOption int @@ -77,6 +86,17 @@ const ( FieldOptionJSType ) +// String implements fmt.Stringer. +func (f FieldOption) String() string { + s, ok := fieldOptionToString[f] + if !ok { + return strconv.Itoa(int(f)) + } + return s +} + +// *** PRIVATE *** + var ( fileOptionToString = map[FileOption]string{ FileOptionJavaPackage: "java_package", @@ -138,9 +158,6 @@ var ( FileOptionRubyPackage: parseOverrideValue[string], FileOptionRubyPackageSuffix: parseOverrideValue[string], } - allFieldOptions = []FieldOption{ - FieldOptionJSType, - } fieldOptionToString = map[FieldOption]string{ FieldOptionJSType: "jstype", } @@ -152,15 +169,6 @@ var ( } ) -// String implements fmt.Stringer. -func (f FileOption) String() string { - s, ok := fileOptionToString[f] - if !ok { - return strconv.Itoa(int(f)) - } - return s -} - func parseFileOption(s string) (FileOption, error) { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { @@ -173,15 +181,6 @@ func parseFileOption(s string) (FileOption, error) { return 0, fmt.Errorf("unknown fileOption: %q", s) } -// String implements fmt.Stringer. -func (f FieldOption) String() string { - s, ok := fieldOptionToString[f] - if !ok { - return strconv.Itoa(int(f)) - } - return s -} - func parseFieldOption(s string) (FieldOption, error) { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { From f346e9617fa4d03e7402f129246f23d144295558 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 17:27:31 -0500 Subject: [PATCH 55/66] generate manged option --- buf.gen.yaml | 9 +++++++++ private/bufpkg/bufconfig/generate_managed_option.go | 11 +++++------ 2 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 buf.gen.yaml diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 0000000000..10e0fe4eb0 --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,9 @@ +version: v2 +plugins: + - binary: protoc-gen-go + out: gogogo +managed: + enabled: true + override: + - file_option: go_package + value: 20 \ No newline at end of file diff --git a/private/bufpkg/bufconfig/generate_managed_option.go b/private/bufpkg/bufconfig/generate_managed_option.go index 7ddd2a9794..492528e6cd 100644 --- a/private/bufpkg/bufconfig/generate_managed_option.go +++ b/private/bufpkg/bufconfig/generate_managed_option.go @@ -172,32 +172,31 @@ var ( func parseFileOption(s string) (FileOption, error) { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { - return 0, errors.New("empty fileOption") + return 0, errors.New("empty file_option") } f, ok := stringToFileOption[s] if ok { return f, nil } - return 0, fmt.Errorf("unknown fileOption: %q", s) + return 0, fmt.Errorf("unknown file_option: %q", s) } func parseFieldOption(s string) (FieldOption, error) { s = strings.ToLower(strings.TrimSpace(s)) if s == "" { - return 0, errors.New("empty field option") + return 0, errors.New("empty field_option") } f, ok := stringToFieldOption[s] if ok { return f, nil } - return 0, fmt.Errorf("unknown field option: %q", s) + return 0, fmt.Errorf("unknown field_option: %q", s) } func parseOverrideValue[T string | bool](overrideValue interface{}) (interface{}, error) { parsedValue, ok := overrideValue.(T) if !ok { - // TODO: test out this message - return nil, fmt.Errorf("must be a %T", overrideValue) + return nil, fmt.Errorf("expected a %T, got %T", parsedValue, overrideValue) } return parsedValue, nil } From 27fc46ad79048d33197c87c78cd80b1a4df0cc5f Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 17:29:56 -0500 Subject: [PATCH 56/66] commit --- buf.gen.yaml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 buf.gen.yaml diff --git a/buf.gen.yaml b/buf.gen.yaml deleted file mode 100644 index 10e0fe4eb0..0000000000 --- a/buf.gen.yaml +++ /dev/null @@ -1,9 +0,0 @@ -version: v2 -plugins: - - binary: protoc-gen-go - out: gogogo -managed: - enabled: true - override: - - file_option: go_package - value: 20 \ No newline at end of file From 521aec18963a0c6453c3ccb274ec88128b2d3432 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 18:17:22 -0500 Subject: [PATCH 57/66] add Enabled to managed mode config --- private/bufpkg/bufconfig/generate_config.go | 28 +++++++++---------- .../bufpkg/bufconfig/generate_config_test.go | 20 ++++++++++++- .../bufconfig/generate_managed_config.go | 19 +++++++++---- .../bufimage/bufimagemodify/new_modify.go | 2 +- .../bufimagemodify/new_modify_test.go | 21 +++++++++++++- 5 files changed, 67 insertions(+), 23 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_config.go b/private/bufpkg/bufconfig/generate_config.go index 398b5db5bc..e3cd35c7b7 100644 --- a/private/bufpkg/bufconfig/generate_config.go +++ b/private/bufpkg/bufconfig/generate_config.go @@ -26,7 +26,7 @@ type GenerateConfig interface { // non-empty. Zero plugin configs will cause an error at construction time. GeneratePluginConfigs() []GeneratePluginConfig // GenerateManagedConfig returns the managed mode configuration. - // This may be nil. + // This may will never be nil. GenerateManagedConfig() GenerateManagedConfig // GenerateTypeConfig returns the types to generate code for. This overrides other type // filters from input configurations, which exist in v2. @@ -60,23 +60,26 @@ type generateConfig struct { typeConfig GenerateTypeConfig } -func newGenerateConfigFromExternalFileV2( - externalFile externalBufGenYAMLFileV2, +func newGenerateConfigFromExternalFileV1Beta1( + externalFile externalBufGenYAMLFileV1Beta1, ) (GenerateConfig, error) { - managedConfig, err := newManagedConfigFromExternalV2(externalFile.Managed) + managedConfig, err := newManagedConfigFromExternalV1Beta1(externalFile.Managed, externalFile.Options) if err != nil { return nil, err } + if len(externalFile.Plugins) == 0 { + return nil, newNoPluginsError() + } pluginConfigs, err := slicesext.MapError( externalFile.Plugins, - newPluginConfigFromExternalV2, + newPluginConfigFromExternalV1Beta1, ) if err != nil { return nil, err } return &generateConfig{ - managedConfig: managedConfig, pluginConfigs: pluginConfigs, + managedConfig: managedConfig, }, nil } @@ -104,26 +107,23 @@ func newGenerateConfigFromExternalFileV1( }, nil } -func newGenerateConfigFromExternalFileV1Beta1( - externalFile externalBufGenYAMLFileV1Beta1, +func newGenerateConfigFromExternalFileV2( + externalFile externalBufGenYAMLFileV2, ) (GenerateConfig, error) { - managedConfig, err := newManagedConfigFromExternalV1Beta1(externalFile.Options) + managedConfig, err := newManagedConfigFromExternalV2(externalFile.Managed) if err != nil { return nil, err } - if len(externalFile.Plugins) == 0 { - return nil, newNoPluginsError() - } pluginConfigs, err := slicesext.MapError( externalFile.Plugins, - newPluginConfigFromExternalV1Beta1, + newPluginConfigFromExternalV2, ) if err != nil { return nil, err } return &generateConfig{ - pluginConfigs: pluginConfigs, managedConfig: managedConfig, + pluginConfigs: pluginConfigs, }, nil } diff --git a/private/bufpkg/bufconfig/generate_config_test.go b/private/bufpkg/bufconfig/generate_config_test.go index 3f8d9dc7f3..cf0bfb8eb6 100644 --- a/private/bufpkg/bufconfig/generate_config_test.go +++ b/private/bufpkg/bufconfig/generate_config_test.go @@ -44,6 +44,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeLocal, @@ -70,6 +71,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeLocal, @@ -96,6 +98,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeBinary, @@ -123,6 +126,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeBinary, @@ -148,6 +152,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeBinary, @@ -171,6 +176,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeBinary, @@ -194,6 +200,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeProtocBuiltin, @@ -217,6 +224,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeProtocBuiltin, @@ -240,6 +248,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeRemote, @@ -264,6 +273,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, expectedConfig: &generateConfig{ + managedConfig: &generateManagedConfig{enabled: false}, pluginConfigs: []GeneratePluginConfig{ &pluginConfig{ pluginConfigType: PluginConfigTypeRemote, @@ -297,7 +307,9 @@ func TestParseConfigFromExternalV1(t *testing.T) { out: "go/out", }, }, - managedConfig: &generateManagedConfig{}, + managedConfig: &generateManagedConfig{ + enabled: true, + }, }, }, { @@ -335,6 +347,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, managedConfig: &generateManagedConfig{ + enabled: true, disables: []ManagedDisableRule{ &managedDisableRule{ fileOption: FileOptionJavaPackage, @@ -413,6 +426,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, managedConfig: &generateManagedConfig{ + enabled: true, disables: []ManagedDisableRule{ &managedDisableRule{ fileOption: FileOptionOptimizeFor, @@ -468,6 +482,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, managedConfig: &generateManagedConfig{ + enabled: true, disables: []ManagedDisableRule{ &managedDisableRule{ fileOption: FileOptionGoPackage, @@ -518,6 +533,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, managedConfig: &generateManagedConfig{ + enabled: true, disables: []ManagedDisableRule{ &managedDisableRule{ fileOption: FileOptionObjcClassPrefix, @@ -567,6 +583,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, managedConfig: &generateManagedConfig{ + enabled: true, disables: []ManagedDisableRule{ &managedDisableRule{ fileOption: FileOptionRubyPackage, @@ -622,6 +639,7 @@ func TestParseConfigFromExternalV1(t *testing.T) { }, }, managedConfig: &generateManagedConfig{ + enabled: true, overrides: []ManagedOverrideRule{ // ordered by file option names and then by file paths &managedOverrideRule{ diff --git a/private/bufpkg/bufconfig/generate_managed_config.go b/private/bufpkg/bufconfig/generate_managed_config.go index 141a48170f..31ad9985d5 100644 --- a/private/bufpkg/bufconfig/generate_managed_config.go +++ b/private/bufpkg/bufconfig/generate_managed_config.go @@ -27,6 +27,8 @@ import ( // GenerateManagedConfig is a managed mode configuration. type GenerateManagedConfig interface { + // Enabled returns whether managed mode is enabled. + Enabled() bool // Disables returns the disable rules in the configuration. Disables() []ManagedDisableRule // Overrides returns the override rules in the configuration. @@ -37,10 +39,12 @@ type GenerateManagedConfig interface { // NewGenerateManagedConfig returns a new GenerateManagedConfig. func NewGenerateManagedConfig( + enabled bool, disables []ManagedDisableRule, overrides []ManagedOverrideRule, ) GenerateManagedConfig { return &generateManagedConfig{ + enabled: enabled, disables: disables, overrides: overrides, } @@ -156,11 +160,13 @@ func NewFieldOptionOverrideRule( // *** PRIVATE *** type generateManagedConfig struct { + enabled bool disables []ManagedDisableRule overrides []ManagedOverrideRule } func newManagedConfigFromExternalV1Beta1( + enabled bool, externalConfig externalGenerateManagedConfigV1Beta1, ) (GenerateManagedConfig, error) { var ( @@ -203,6 +209,7 @@ func newManagedConfigFromExternalV1Beta1( overrides = append(overrides, defaultOverride) } return &generateManagedConfig{ + enabled: enabled, overrides: overrides, }, nil } @@ -210,9 +217,6 @@ func newManagedConfigFromExternalV1Beta1( func newManagedConfigFromExternalV1( externalConfig externalGenerateManagedConfigV1, ) (GenerateManagedConfig, error) { - if !externalConfig.Enabled { - return nil, nil - } var ( disables []ManagedDisableRule overrides []ManagedOverrideRule @@ -389,6 +393,7 @@ func newManagedConfigFromExternalV1( } overrides = append(overrides, perFileOverrides...) return &generateManagedConfig{ + enabled: externalConfig.Enabled, disables: disables, overrides: overrides, }, nil @@ -397,9 +402,6 @@ func newManagedConfigFromExternalV1( func newManagedConfigFromExternalV2( externalConfig externalGenerateManagedConfigV2, ) (GenerateManagedConfig, error) { - if !externalConfig.Enabled { - return nil, nil - } var disables []ManagedDisableRule var overrides []ManagedOverrideRule for _, externalDisableConfig := range externalConfig.Disable { @@ -476,11 +478,16 @@ func newManagedConfigFromExternalV2( overrides = append(overrides, override) } return &generateManagedConfig{ + enabled: externalConfig.Enabled, disables: disables, overrides: overrides, }, nil } +func (g *generateManagedConfig) Enabled() bool { + return g.enabled +} + func (g *generateManagedConfig) Disables() []ManagedDisableRule { return g.disables } diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go index 8bd5ddd871..52f181e97f 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -30,7 +30,7 @@ func Modify( image bufimage.Image, config bufconfig.GenerateManagedConfig, ) error { - if config == nil { + if !config.Enabled() { return nil } // TODO: in v2 modify field options as well diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go index 8acf6fd864..b83ca613e6 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go @@ -46,7 +46,7 @@ func TestModifyImage(t *testing.T) { filepath.Join("testdata", "foo"): "buf.build/acme/foo", filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, - config: nil, + config: bufconfig.NewGenerateManagedConfig(false, nil, nil), filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ "foo_empty/with_package.proto": nil, "bar_all/with_package.proto": { @@ -78,6 +78,7 @@ func TestModifyImage(t *testing.T) { filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{}, ), @@ -194,6 +195,7 @@ func TestModifyImageFile( filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionCcEnableArenas, false), @@ -217,6 +219,7 @@ func TestModifyImageFile( filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{ bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionCsharpNamespacePrefix, bufconfig.FieldOptionUnspecified), }, @@ -253,6 +256,7 @@ func TestModifyImageFile( filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{ bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionGoPackagePrefix, bufconfig.FieldOptionUnspecified), }, @@ -292,6 +296,7 @@ func TestModifyImageFile( filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{ bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackagePrefix, bufconfig.FieldOptionUnspecified), }, @@ -388,6 +393,7 @@ func TestModifyImageFile( filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{ bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackageSuffix, bufconfig.FieldOptionUnspecified), }, @@ -442,6 +448,7 @@ func TestModifyImageFile( filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{ bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackage, bufconfig.FieldOptionUnspecified), }, @@ -495,6 +502,7 @@ func TestModifyImageFile( filepath.Join("testdata", "bar"): "buf.build/acme/bar", }, config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{ bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionObjcClassPrefix, bufconfig.FieldOptionUnspecified), }, @@ -598,6 +606,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "only_value", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), @@ -609,6 +618,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "only_prefix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), @@ -620,6 +630,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "only_suffix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), @@ -631,6 +642,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "prefix_then_value", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), @@ -643,6 +655,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "value_then_prefix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), @@ -655,6 +668,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "prefix_then_suffix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), @@ -670,6 +684,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "value_prefix_then_suffix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), @@ -686,6 +701,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "prefix_value_then_suffix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), @@ -699,6 +715,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "prefix_then_prefix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), @@ -711,6 +728,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "suffix_then_suffix", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), @@ -723,6 +741,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { { description: "value_then_value", config: bufconfig.NewGenerateManagedConfig( + true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), From 62b07b09b78499e19c59583a42c731f5fbb330f5 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Fri, 1 Dec 2023 22:58:09 -0500 Subject: [PATCH 58/66] generate_plugin_config.go --- .../bufconfig/generate_plugin_config.go | 365 ++++++++---------- 1 file changed, 152 insertions(+), 213 deletions(-) diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index daca439046..a3b84d1e63 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -28,16 +28,13 @@ import ( "github.com/bufbuild/buf/private/pkg/syserror" ) -// TODO: move private methods below /* private */ - -const remoteAlphaPluginDeprecationMessage = "the remote field no longer works as " + - "the remote generation alpha has been deprecated, see the migration guide to " + - "now-stable remote plugins: https://buf.build/docs/migration-guides/migrate-remote-generation-alpha/#migrate-to-remote-plugins" +const ( + remoteAlphaPluginDeprecationMessage = "the remote field no longer works as " + + "the remote generation alpha has been deprecated, see the migration guide to " + + "now-stable remote plugins: https://buf.build/docs/migration-guides/migrate-remote-generation-alpha/#migrate-to-remote-plugins" +) // GenerateStrategy is the generation strategy for a protoc plugin. -// -// TODO: Should this type live in this package? Perhaps it should live in the package that handles generation? -// TODO: The same question can be asked for FieldOption and FileOption. type GenerateStrategy int const ( @@ -133,11 +130,6 @@ func NewGeneratePluginWithIncludeImportsAndWKT( // *** PRIVATE *** type pluginConfig struct { - // TODO: perhaps make some of these pointers so that whether a field is - // specified in external config is preserved. This way, we can migrate more - // accurately. - // But what do we do about opt and path? Opt() and Path() could then return an error. - // Or define Migrate(templateOverride) error, which probably works better. pluginConfigType PluginConfigType name string out string @@ -151,56 +143,6 @@ type pluginConfig struct { revision int } -func (p *pluginConfig) Type() PluginConfigType { - return p.pluginConfigType -} - -func (p *pluginConfig) Name() string { - return p.name -} - -func (p *pluginConfig) Out() string { - return p.out -} - -func (p *pluginConfig) Opt() string { - return strings.Join(p.opts, ",") -} - -func (p *pluginConfig) IncludeImports() bool { - return p.includeImports -} - -func (p *pluginConfig) IncludeWKT() bool { - return p.includeWKT -} - -func (p *pluginConfig) Strategy() GenerateStrategy { - if p.strategy == nil { - return GenerateStrategyDirectory - } - return *p.strategy -} - -func (p *pluginConfig) Path() []string { - return p.path -} - -func (p *pluginConfig) ProtocPath() string { - return p.protocPath -} - -func (p *pluginConfig) RemoteHost() string { - return p.remoteHost -} - -func (p *pluginConfig) Revision() int { - return p.revision -} - -func (p *pluginConfig) isGeneratePluginConfig() {} - -// TODO: compare with the old implementation func newPluginConfigFromExternalV1Beta1( externalConfig externalGeneratePluginConfigV1Beta1, ) (GeneratePluginConfig, error) { @@ -221,25 +163,24 @@ func newPluginConfigFromExternalV1Beta1( if externalConfig.Path != "" { return newBinaryPluginConfig( externalConfig.Name, - []string{externalConfig.Path}, - strategy, externalConfig.Out, opt, false, false, + strategy, + []string{externalConfig.Path}, ) } return newLocalPluginConfig( externalConfig.Name, - strategy, externalConfig.Out, opt, false, false, + strategy, ) } -// TODO: figure out where is the best place to do parameter validation, here or in new*plugin. func newPluginConfigFromExternalV1( externalConfig externalGeneratePluginConfigV1, ) (GeneratePluginConfig, error) { @@ -288,17 +229,22 @@ func newPluginConfigFromExternalV1( return nil, err } if externalConfig.Plugin != "" && bufpluginref.IsPluginReferenceOrIdentity(pluginIdentifier) { - // TODO: Is checkPathAndStrategyUnset the best way to validate this? - if err := checkPathAndStrategyUnset(externalConfig, pluginIdentifier); err != nil { - return nil, err + if externalConfig.Path != nil { + return nil, fmt.Errorf("cannot specify path for remote plugin %s", externalConfig.Plugin) + } + if externalConfig.Strategy != "" { + return nil, fmt.Errorf("cannot specify strategy for remote plugin %s", externalConfig.Plugin) + } + if externalConfig.ProtocPath != "" { + return nil, fmt.Errorf("cannot specify protoc_path for remote plugin %s", externalConfig.Plugin) } return newRemotePluginConfig( externalConfig.Plugin, - externalConfig.Revision, externalConfig.Out, opt, false, false, + externalConfig.Revision, ) } // At this point the plugin must be local, regardless whehter it's specified @@ -306,127 +252,55 @@ func newPluginConfigFromExternalV1( if len(path) > 0 { return newBinaryPluginConfig( pluginIdentifier, - path, - strategy, externalConfig.Out, opt, false, false, + strategy, + path, ) } if externalConfig.ProtocPath != "" { return newProtocBuiltinPluginConfig( pluginIdentifier, - externalConfig.ProtocPath, externalConfig.Out, opt, false, false, strategy, + externalConfig.ProtocPath, ) } // It could be either binary or protoc built-in. We defer to the plugin executor // to decide whether the plugin is protoc-builtin or binary. return newLocalPluginConfig( pluginIdentifier, - strategy, externalConfig.Out, opt, false, false, + strategy, ) } -func parseStrategy(s string) (*GenerateStrategy, error) { - var strategy GenerateStrategy - switch s { - case "": - return nil, nil - case "directory": - strategy = GenerateStrategyDirectory - case "all": - strategy = GenerateStrategyAll - default: - return nil, fmt.Errorf("unknown strategy: %s", s) - } - return &strategy, nil -} - -// TODO: move this up, maybe let v1 use this as well -const ( - typeRemote = "remote" - typeBinary = "binary" - typeProtocBuiltin = "protoc_builtin" -) - -const ( - optionRevision = "revision" - optionProtocPath = "protoc_path" - optionStrategy = "strategy" -) - -var allowedOptionsForType = map[string](map[string]bool){ - typeRemote: { - optionRevision: true, - }, - typeBinary: { - optionStrategy: true, - }, - typeProtocBuiltin: { - optionProtocPath: true, - optionStrategy: true, - }, -} - -func getTypesAndOptions(externalConfig externalGeneratePluginConfigV2) ([]string, []string, error) { - var ( - types []string - options []string - ) - if externalConfig.Remote != nil { - types = append(types, typeRemote) - } - path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Binary) - if err != nil { - return nil, nil, err - } - if len(path) > 0 { - types = append(types, typeBinary) - } - if externalConfig.ProtocBuiltin != nil { - types = append(types, typeProtocBuiltin) - } - if externalConfig.Revision != nil { - options = append(options, optionRevision) - } - if externalConfig.ProtocPath != nil { - options = append(options, optionProtocPath) - } - if externalConfig.Strategy != nil { - options = append(options, optionStrategy) - } - return types, options, nil -} - func newPluginConfigFromExternalV2( externalConfig externalGeneratePluginConfigV2, ) (GeneratePluginConfig, error) { - pluginTypes, options, err := getTypesAndOptions(externalConfig) - if err != nil { - return nil, err + var pluginTypeCount int + if externalConfig.Remote != nil { + pluginTypeCount++ + } + if externalConfig.Binary != nil { + pluginTypeCount++ } - if len(pluginTypes) == 0 { - return nil, fmt.Errorf("must specify one of %s, %s and %s", typeRemote, typeBinary, typeProtocBuiltin) + if externalConfig.ProtocBuiltin != nil { + pluginTypeCount++ } - if len(pluginTypes) > 1 { - return nil, fmt.Errorf("only one of %s, %s and %s is allowed", typeRemote, typeBinary, typeProtocBuiltin) + if pluginTypeCount == 0 { + return nil, errors.New("must specify one of remote, binary and protoc_builtin") } - pluginType := pluginTypes[0] - allowedOptions := allowedOptionsForType[pluginType] - for _, option := range options { - if !allowedOptions[option] { - return nil, fmt.Errorf("%s is not allowed for %s plugin", option, pluginType) - } + if pluginTypeCount > 1 { + return nil, errors.New("only one of remote, binary and protoc_builtin") } var strategy string if externalConfig.Strategy != nil { @@ -440,73 +314,92 @@ func newPluginConfigFromExternalV2( if err != nil { return nil, err } - switch pluginType { - case typeRemote: + switch { + case externalConfig.Remote != nil: var revision int if externalConfig.Revision != nil { revision = *externalConfig.Revision } - if revision < 0 || revision > math.MaxInt32 { - return nil, fmt.Errorf("revision %d is out of accepted range %d-%d", revision, 0, math.MaxInt32) + if externalConfig.Strategy != nil { + return nil, fmt.Errorf("cannot specify strategy for remote plugin %s", *externalConfig.Remote) + } + if externalConfig.ProtocPath != nil { + return nil, fmt.Errorf("cannot specify protoc_path for remote plugin %s", *externalConfig.Remote) } return newRemotePluginConfig( *externalConfig.Remote, - revision, externalConfig.Out, opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + revision, ) - case typeBinary: + case externalConfig.Binary != nil: path, err := encoding.InterfaceSliceOrStringToStringSlice(externalConfig.Binary) if err != nil { return nil, err } + binaryPluginName := strings.Join(path, " ") + if externalConfig.Revision != nil { + return nil, fmt.Errorf("cannot specify revision for binary plugin %s", binaryPluginName) + } + if externalConfig.ProtocPath != nil { + return nil, fmt.Errorf("cannot specify protoc_path for binary plugin %s", binaryPluginName) + } return newBinaryPluginConfig( - strings.Join(path, ""), - path, - parsedStrategy, + strings.Join(path, " "), externalConfig.Out, opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, + parsedStrategy, + path, ) - case typeProtocBuiltin: + case externalConfig.ProtocBuiltin != nil: var protocPath string if externalConfig.ProtocPath != nil { protocPath = *externalConfig.ProtocPath } + if externalConfig.Revision != nil { + return nil, fmt.Errorf("cannot specify revision for protoc built-in plugin %s", *externalConfig.ProtocBuiltin) + } return newProtocBuiltinPluginConfig( *externalConfig.ProtocBuiltin, - protocPath, externalConfig.Out, opt, externalConfig.IncludeImports, externalConfig.IncludeWKT, parsedStrategy, + protocPath, ) default: - // this should not happen - return nil, fmt.Errorf("must specify one of %s, %s and %s", typeRemote, typeBinary, typeProtocBuiltin) + return nil, syserror.Newf("must specify one of remote, binary and protoc_builtin") } } -// TODO: unify parameter order -func newLocalPluginConfig( +func newRemotePluginConfig( name string, - strategy *GenerateStrategy, out string, opt []string, includeImports bool, includeWKT bool, + revision int, ) (*pluginConfig, error) { if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } + remoteHost, err := parseRemoteHostName(name) + if err != nil { + return nil, err + } + if revision < 0 || revision > math.MaxInt32 { + return nil, fmt.Errorf("revision %d is out of accepted range %d-%d", revision, 0, math.MaxInt32) + } return &pluginConfig{ - pluginConfigType: PluginConfigTypeLocal, + pluginConfigType: PluginConfigTypeRemote, name: name, - strategy: strategy, + remoteHost: remoteHost, + revision: revision, out: out, opts: opt, includeImports: includeImports, @@ -514,25 +407,20 @@ func newLocalPluginConfig( }, nil } -func newBinaryPluginConfig( +func newLocalPluginConfig( name string, - path []string, - strategy *GenerateStrategy, out string, opt []string, includeImports bool, includeWKT bool, + strategy *GenerateStrategy, ) (*pluginConfig, error) { - if len(path) == 0 { - return nil, errors.New("must specify a path to the plugin") - } if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } return &pluginConfig{ - pluginConfigType: PluginConfigTypeBinary, + pluginConfigType: PluginConfigTypeLocal, name: name, - path: path, strategy: strategy, out: out, opts: opt, @@ -541,58 +429,106 @@ func newBinaryPluginConfig( }, nil } -func newProtocBuiltinPluginConfig( +func newBinaryPluginConfig( name string, - protocPath string, out string, opt []string, includeImports bool, includeWKT bool, strategy *GenerateStrategy, + path []string, ) (*pluginConfig, error) { + if len(path) == 0 { + return nil, errors.New("must specify a path to the plugin") + } if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } return &pluginConfig{ - pluginConfigType: PluginConfigTypeProtocBuiltin, + pluginConfigType: PluginConfigTypeBinary, name: name, - protocPath: protocPath, + path: path, + strategy: strategy, out: out, opts: opt, - strategy: strategy, includeImports: includeImports, includeWKT: includeWKT, }, nil } -func newRemotePluginConfig( +func newProtocBuiltinPluginConfig( name string, - revision int, out string, opt []string, includeImports bool, includeWKT bool, + strategy *GenerateStrategy, + protocPath string, ) (*pluginConfig, error) { if includeWKT && !includeImports { return nil, errors.New("cannot include well-known types without including imports") } - remoteHost, err := parseRemoteHostName(name) - if err != nil { - return nil, err - } - // TODO: validate revision return &pluginConfig{ - pluginConfigType: PluginConfigTypeRemote, + pluginConfigType: PluginConfigTypeProtocBuiltin, name: name, - remoteHost: remoteHost, - revision: revision, + protocPath: protocPath, out: out, opts: opt, + strategy: strategy, includeImports: includeImports, includeWKT: includeWKT, }, nil } +func (p *pluginConfig) Type() PluginConfigType { + return p.pluginConfigType +} + +func (p *pluginConfig) Name() string { + return p.name +} + +func (p *pluginConfig) Out() string { + return p.out +} + +func (p *pluginConfig) Opt() string { + return strings.Join(p.opts, ",") +} + +func (p *pluginConfig) IncludeImports() bool { + return p.includeImports +} + +func (p *pluginConfig) IncludeWKT() bool { + return p.includeWKT +} + +func (p *pluginConfig) Strategy() GenerateStrategy { + if p.strategy == nil { + return GenerateStrategyDirectory + } + return *p.strategy +} + +func (p *pluginConfig) Path() []string { + return p.path +} + +func (p *pluginConfig) ProtocPath() string { + return p.protocPath +} + +func (p *pluginConfig) RemoteHost() string { + return p.remoteHost +} + +func (p *pluginConfig) Revision() int { + return p.revision +} + +func (p *pluginConfig) isGeneratePluginConfig() {} + func newExternalGeneratePluginConfigV2FromPluginConfig( generatePluginConfig GeneratePluginConfig, ) (externalGeneratePluginConfigV2, error) { @@ -654,6 +590,21 @@ func newExternalGeneratePluginConfigV2FromPluginConfig( return externalPluginConfigV2, nil } +func parseStrategy(s string) (*GenerateStrategy, error) { + var strategy GenerateStrategy + switch s { + case "": + return nil, nil + case "directory": + strategy = GenerateStrategyDirectory + case "all": + strategy = GenerateStrategyAll + default: + return nil, fmt.Errorf("unknown strategy: %s", s) + } + return &strategy, nil +} + func parseRemoteHostName(fullName string) (string, error) { if identity, err := bufpluginref.PluginIdentityForString(fullName); err == nil { return identity.Remote(), nil @@ -665,19 +616,7 @@ func parseRemoteHostName(fullName string) (string, error) { return "", err } -func checkPathAndStrategyUnset(plugin externalGeneratePluginConfigV1, pluginIdentifier string) error { - if plugin.Path != nil { - return fmt.Errorf("remote plugin %s cannot specify a path", pluginIdentifier) - } - if plugin.Strategy != "" { - return fmt.Errorf("remote plugin %s cannot specify a strategy", pluginIdentifier) - } - if plugin.ProtocPath != "" { - return fmt.Errorf("remote plugin %s cannot specify a protoc path", pluginIdentifier) - } - return nil -} - +// TODO: where to put this? func toPointer[T any](value T) *T { return &value } From e6908751a0cd75bbcdc7872348ce141fb8608a90 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Sat, 2 Dec 2023 00:05:27 -0500 Subject: [PATCH 59/66] input_config.go --- private/buf/buffetch/buffetch.go | 6 +- private/buf/buffetch/internal/internal.go | 30 +- .../bufpkg/bufconfig/generate_type_config.go | 3 +- private/bufpkg/bufconfig/input_config.go | 456 +++++++++--------- private/pkg/slicesext/slicesext.go | 16 + 5 files changed, 263 insertions(+), 248 deletions(-) diff --git a/private/buf/buffetch/buffetch.go b/private/buf/buffetch/buffetch.go index da6b324d8a..7d9c6343ae 100644 --- a/private/buf/buffetch/buffetch.go +++ b/private/buf/buffetch/buffetch.go @@ -419,17 +419,17 @@ func GetInputConfigForString( return bufconfig.NewBinaryImageInputConfig( t.Path(), t.internalSingleRef().CompressionType().String(), - ), nil + ) case MessageEncodingJSON: return bufconfig.NewJSONImageInputConfig( t.Path(), t.internalSingleRef().CompressionType().String(), - ), nil + ) case MessageEncodingTxtpb: return bufconfig.NewBinaryImageInputConfig( t.Path(), t.internalSingleRef().CompressionType().String(), - ), nil + ) default: // TODO: handle refs with YAML type return nil, fmt.Errorf("unknown encoding: %v", t.MessageEncoding()) diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index 5c927861bd..96c40cdec4 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -869,6 +869,10 @@ type GetModuleOption func(*getModuleOptions) // passed because if the ref is a git ref, it would only have a git.Name, instead // of a git branch, a git ref and a git tag. Therefore the original string is passed. func GetInputConfigForRef(ref Ref, value string) (bufconfig.InputConfig, error) { + _, options, err := getRawPathAndOptionsForInputString(value) + if err != nil { + return nil, err + } switch t := ref.(type) { case ArchiveRef: switch t.ArchiveType() { @@ -876,49 +880,47 @@ func GetInputConfigForRef(ref Ref, value string) (bufconfig.InputConfig, error) return bufconfig.NewZipArchiveInputConfig( t.Path(), t.SubDirPath(), - uint32ToPointer(t.StripComponents()), - ), nil + t.StripComponents(), + ) case ArchiveTypeTar: return bufconfig.NewTarballInputConfig( t.Path(), t.SubDirPath(), t.CompressionType().String(), - uint32ToPointer(t.StripComponents()), - ), nil + t.StripComponents(), + ) default: return nil, fmt.Errorf("invalid archive type: %v", t.ArchiveType()) } case DirRef: return bufconfig.NewDirectoryInputConfig( t.Path(), - ), nil + ) case ModuleRef: return bufconfig.NewModuleInputConfig( t.ModuleRef().String(), - ), nil + ) case ProtoFileRef: return bufconfig.NewProtoFileInputConfig( t.Path(), - ), nil + t.IncludePackageFiles(), + ) case GitRef: - _, options, err := getRawPathAndOptionsForInputString(value) - if err != nil { - return nil, err - } + return bufconfig.NewGitRepoInputConfig( t.Path(), t.SubDirPath(), options["branch"], options["tag"], options["ref"], - uint32ToPointer(t.Depth()), + toPointer(t.Depth()), t.RecurseSubmodules(), - ), nil + ) default: return nil, fmt.Errorf("unexpected Ref of type %T", ref) } } -func uint32ToPointer(value uint32) *uint32 { +func toPointer[T any](value T) *T { return &value } diff --git a/private/bufpkg/bufconfig/generate_type_config.go b/private/bufpkg/bufconfig/generate_type_config.go index ccf0ded554..541178e522 100644 --- a/private/bufpkg/bufconfig/generate_type_config.go +++ b/private/bufpkg/bufconfig/generate_type_config.go @@ -23,11 +23,12 @@ type GenerateTypeConfig interface { isGenerateTypeConfig() } +// *** PRIVATE *** + type generateTypeConfig struct { includeTypes []string } -// TODO: it seems like this isn't validated in main, but we should do some validation func newGenerateTypeConfig(includeTypes []string) GenerateTypeConfig { if len(includeTypes) == 0 { return nil diff --git a/private/bufpkg/bufconfig/input_config.go b/private/bufpkg/bufconfig/input_config.go index bcf66842e4..9bcafccac1 100644 --- a/private/bufpkg/bufconfig/input_config.go +++ b/private/bufpkg/bufconfig/input_config.go @@ -20,25 +20,36 @@ import ( "fmt" "strconv" + "github.com/bufbuild/buf/private/pkg/slicesext" + "github.com/bufbuild/buf/private/pkg/stringutil" "github.com/bufbuild/buf/private/pkg/syserror" ) -// TODO: input type? +// InputConfigType is an input config's type. type InputConfigType int const ( + // InputConfigTypeModule is the module input type. InputConfigTypeModule InputConfigType = iota + 1 + // InputConfigTypeDirectory is the directory input type. InputConfigTypeDirectory + // InputConfigTypeGitRepo is the git repository input type. InputConfigTypeGitRepo + // InputConfigTypeProtoFile is the proto file input type. InputConfigTypeProtoFile + // InputConfigTypeTarball is the tarball input type. InputConfigTypeTarball + // InputConfigTypeZipArchive is the zip archive input type. InputConfigTypeZipArchive + // InputConfigTypeBinaryImage is the binary image input type. InputConfigTypeBinaryImage + // InputConfigTypeJSONImage is the JSON image input type. InputConfigTypeJSONImage + // InputConfigTypeTextImage is the text image input type. InputConfigTypeTextImage ) -// Implements fmt.Stringer +// String implements fmt.Stringer. func (i InputConfigType) String() string { s, ok := inputConfigTypeToString[i] if !ok { @@ -48,85 +59,79 @@ func (i InputConfigType) String() string { } const ( - // TODO: move string literal to maps - formatGitRepo = "git_repo" - formatModule = "module" - formatDirectory = "directory" - formatProtoFile = "proto_file" - formatBinaryImage = "binary_image" - formatTarball = "tarball" - formatZipArchive = "zip_archive" - formatJSONImage = "json_image" - formatTextImage = "text_image" - optionCompression = "compression" - optionBranch = "branch" - optionTag = "tag" - optionRef = "ref" - optionDepth = "depth" - optionRecurseSubmodules = "recurse_submodules" - optionStripComponents = "strip_components" - optionSubdir = "subdir" - optionIncludePackageFiles = "include_package_files" + compressionKey = "compression" + branchKey = "branch" + tagKey = "tag" + refKey = "ref" + depthKey = "depth" + recurseSubmodulesKey = "recurse_submodules" + stripComponentsKey = "strip_components" + subDirKey = "subdir" + includePackageFilesKey = "include_package_files" ) -var allowedOptionsForFormat = map[InputConfigType](map[string]bool){ - InputConfigTypeGitRepo: { - optionBranch: true, - optionTag: true, - optionRef: true, - optionDepth: true, - optionRecurseSubmodules: true, - optionSubdir: true, - }, - InputConfigTypeModule: {}, - InputConfigTypeDirectory: {}, - InputConfigTypeProtoFile: { - optionIncludePackageFiles: true, - }, - InputConfigTypeTarball: { - optionCompression: true, - optionStripComponents: true, - optionSubdir: true, - }, - InputConfigTypeZipArchive: { - optionStripComponents: true, - optionSubdir: true, - }, - InputConfigTypeBinaryImage: { - optionCompression: true, - }, - InputConfigTypeJSONImage: { - optionCompression: true, - }, - InputConfigTypeTextImage: { - optionCompression: true, - }, -} - -var inputConfigTypeToString = map[InputConfigType]string{ - InputConfigTypeGitRepo: formatGitRepo, - InputConfigTypeModule: formatModule, - InputConfigTypeDirectory: formatDirectory, - InputConfigTypeProtoFile: formatProtoFile, - InputConfigTypeTarball: formatTarball, - InputConfigTypeZipArchive: formatZipArchive, - InputConfigTypeBinaryImage: formatBinaryImage, - InputConfigTypeJSONImage: formatJSONImage, - InputConfigTypeTextImage: formatTextImage, -} +var ( + allowedOptionsForFormat = map[InputConfigType](map[string]struct{}){ + InputConfigTypeGitRepo: { + branchKey: {}, + tagKey: {}, + refKey: {}, + depthKey: {}, + recurseSubmodulesKey: {}, + subDirKey: {}, + }, + InputConfigTypeModule: {}, + InputConfigTypeDirectory: {}, + InputConfigTypeProtoFile: { + includePackageFilesKey: {}, + }, + InputConfigTypeTarball: { + compressionKey: {}, + stripComponentsKey: {}, + subDirKey: {}, + }, + InputConfigTypeZipArchive: { + stripComponentsKey: {}, + subDirKey: {}, + }, + InputConfigTypeBinaryImage: { + compressionKey: {}, + }, + InputConfigTypeJSONImage: { + compressionKey: {}, + }, + InputConfigTypeTextImage: { + compressionKey: {}, + }, + } + inputConfigTypeToString = map[InputConfigType]string{ + InputConfigTypeGitRepo: "git_repo", + InputConfigTypeModule: "module", + InputConfigTypeDirectory: "directory", + InputConfigTypeProtoFile: "proto_file", + InputConfigTypeTarball: "tarball", + InputConfigTypeZipArchive: "zip_archive", + InputConfigTypeBinaryImage: "binary_image", + InputConfigTypeJSONImage: "json_image", + InputConfigTypeTextImage: "text_image", + } + allInputConfigTypeString = stringutil.SliceToHumanString( + slicesext.MapValuesToSortedSlice(inputConfigTypeToString), + ) +) // InputConfig is an input configuration for code generation. type InputConfig interface { - // Type returns the input type. + // Type returns the input type. This is never the zero value. Type() InputConfigType - // Location returns the location for the input. + // Location returns the location for the input. This is never empty. Location() string // Compression returns the compression scheme, not empty only if format is // one of tarball, binary image, json image or text image. Compression() string // StripComponents returns the number of directories to strip for tar or zip // inputs, not empty only if format is tarball or zip archive. - StripComponents() *uint32 + StripComponents() uint32 // SubDir returns the subdirectory to use, not empty only if format is one // git repo, tarball and zip archive. SubDir() string @@ -155,67 +160,6 @@ type InputConfig interface { isInputConfig() } -// NewInputConfig returns a new input config. -func NewInputConfig( - inputType InputConfigType, - location string, - compression string, - stripComponents *uint32, - subDir string, - branch string, - tag string, - ref string, - depth *uint32, - recurseSubmodules bool, - includePackageFiles bool, - includePaths []string, - excludePaths []string, - includeTypes []string, -) InputConfig { - return &inputConfig{ - inputType: inputType, - location: location, - compression: compression, - stripComponents: stripComponents, - subDir: subDir, - branch: branch, - tag: tag, - ref: ref, - depth: depth, - recurseSubmodules: recurseSubmodules, - includePackageFiles: includePackageFiles, - includePaths: includePaths, - excludePaths: excludePaths, - includeTypes: includeTypes, - } -} - -// NewInputConfigWithTargets returns an input config with include paths, exclude -// paths and include types overriden. -func NewInputConfigWithTargets( - config InputConfig, - includePaths []string, - excludePaths []string, - includeTypes []string, -) (InputConfig, error) { - originalConfig, ok := config.(*inputConfig) - if !ok { - return nil, syserror.Newf("unknown implementation of InputConfig: %T", config) - } - // targetConfig is a copy of the original config. - targetConfig := *originalConfig - if len(includePaths) > 0 { - targetConfig.includePaths = includePaths - } - if len(excludePaths) > 0 { - targetConfig.excludePaths = excludePaths - } - if len(includeTypes) > 0 { - targetConfig.includeTypes = includeTypes - } - return &targetConfig, nil -} - // NewGitRepoInputConfig returns an input config for a git repo. func NewGitRepoInputConfig( location string, @@ -225,7 +169,10 @@ func NewGitRepoInputConfig( ref string, depth *uint32, recurseSubModules bool, -) InputConfig { +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for git repository") + } return &inputConfig{ inputType: InputConfigTypeGitRepo, location: location, @@ -235,37 +182,48 @@ func NewGitRepoInputConfig( ref: ref, depth: depth, recurseSubmodules: recurseSubModules, - } + }, nil } // NewModuleInputConfig returns an input config for a module. func NewModuleInputConfig( location string, -) InputConfig { +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for module") + } return &inputConfig{ inputType: InputConfigTypeModule, location: location, - } + }, nil } // NewDirectoryInputConfig returns an input config for a directory. func NewDirectoryInputConfig( location string, -) InputConfig { +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for directory") + } return &inputConfig{ inputType: InputConfigTypeDirectory, location: location, - } + }, nil } // NewProtoFileInputConfig returns an input config for a proto file. func NewProtoFileInputConfig( location string, -) InputConfig { - return &inputConfig{ - inputType: InputConfigTypeProtoFile, - location: location, + includePackageFiles bool, +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for proto file") } + return &inputConfig{ + inputType: InputConfigTypeProtoFile, + location: location, + includePackageFiles: includePackageFiles, + }, nil } // NewTarballInputConfig returns an input config for a tarball. @@ -273,75 +231,115 @@ func NewTarballInputConfig( location string, subDir string, compression string, - stripComponents *uint32, -) InputConfig { + stripComponents uint32, +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for tarball") + } return &inputConfig{ inputType: InputConfigTypeTarball, location: location, subDir: subDir, compression: compression, stripComponents: stripComponents, - } + }, nil } // NewZipArchiveInputConfig returns an input config for a zip archive. func NewZipArchiveInputConfig( location string, subDir string, - stripComponents *uint32, -) InputConfig { + stripComponents uint32, +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for zip archive") + } return &inputConfig{ inputType: InputConfigTypeZipArchive, location: location, subDir: subDir, stripComponents: stripComponents, - } + }, nil } // NewBinaryImageInputConfig returns an input config for a binary image. func NewBinaryImageInputConfig( location string, compression string, -) InputConfig { +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for binary image") + } return &inputConfig{ inputType: InputConfigTypeBinaryImage, location: location, compression: compression, - } + }, nil } // NewJSONImageInputConfig returns an input config for a JSON image. func NewJSONImageInputConfig( location string, compression string, -) InputConfig { +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for JSON image") + } return &inputConfig{ inputType: InputConfigTypeJSONImage, location: location, compression: compression, - } + }, nil } // NewTextImageInputConfig returns an input config for a text image. func NewTextImageInputConfig( location string, compression string, -) InputConfig { +) (InputConfig, error) { + if location == "" { + return nil, errors.New("empty location for binary image") + } return &inputConfig{ inputType: InputConfigTypeTextImage, location: location, compression: compression, + }, nil +} + +// NewInputConfigWithTargets returns an input config the same as the passed config, +// but with include paths, exclude paths and include types overriden. +func NewInputConfigWithTargets( + config InputConfig, + includePaths []string, + excludePaths []string, + includeTypes []string, +) (InputConfig, error) { + originalConfig, ok := config.(*inputConfig) + if !ok { + return nil, syserror.Newf("unknown implementation of InputConfig: %T", config) + } + // targetConfig is a copy of the original config. + targetConfig := *originalConfig + if len(includePaths) > 0 { + targetConfig.includePaths = includePaths } + if len(excludePaths) > 0 { + targetConfig.excludePaths = excludePaths + } + if len(includeTypes) > 0 { + targetConfig.includeTypes = includeTypes + } + return &targetConfig, nil } // *** PRIVATE *** type inputConfig struct { - inputType InputConfigType - location string - compression string - // TODO: does it make sense to be a pointer? - stripComponents *uint32 + inputType InputConfigType + location string + compression string + stripComponents uint32 subDir string branch string tag string @@ -395,119 +393,60 @@ func newInputConfigFromExternalV2(externalConfig externalInputConfigV2) (InputCo inputConfig.location = *externalConfig.GitRepo } if externalConfig.Compression != nil { - options = append(options, optionCompression) + options = append(options, compressionKey) inputConfig.compression = *externalConfig.Compression } if externalConfig.StripComponents != nil { - options = append(options, optionStripComponents) - inputConfig.stripComponents = externalConfig.StripComponents + options = append(options, stripComponentsKey) + inputConfig.stripComponents = *externalConfig.StripComponents } if externalConfig.Subdir != nil { - options = append(options, optionSubdir) + options = append(options, subDirKey) inputConfig.subDir = *externalConfig.Subdir } if externalConfig.Branch != nil { - options = append(options, optionBranch) + options = append(options, branchKey) inputConfig.branch = *externalConfig.Branch } if externalConfig.Tag != nil { - options = append(options, optionTag) + options = append(options, tagKey) inputConfig.tag = *externalConfig.Tag } if externalConfig.Ref != nil { - options = append(options, optionRef) + options = append(options, refKey) inputConfig.ref = *externalConfig.Ref } if externalConfig.Depth != nil { - options = append(options, optionDepth) + options = append(options, depthKey) inputConfig.depth = externalConfig.Depth } if externalConfig.RecurseSubmodules != nil { - options = append(options, optionRecurseSubmodules) + options = append(options, recurseSubmodulesKey) inputConfig.recurseSubmodules = *externalConfig.RecurseSubmodules } if externalConfig.IncludePackageFiles != nil { - options = append(options, optionIncludePackageFiles) + options = append(options, includePackageFilesKey) inputConfig.includePackageFiles = *externalConfig.IncludePackageFiles } if len(inputTypes) == 0 { - return nil, errors.New("must specify input type") + return nil, fmt.Errorf("must specify one of %s", allInputConfigTypeString) } if len(inputTypes) > 1 { - // TODO: print out all types allowed - return nil, fmt.Errorf("exactly one input type can be specified") + return nil, fmt.Errorf("exactly one of %s must be specified", allInputConfigTypeString) } format := inputTypes[0] allowedOptions, ok := allowedOptionsForFormat[format] if !ok { - // this should not happen - return nil, fmt.Errorf("unable to find allowed options for format %v", format) + return nil, syserror.Newf("unable to find allowed options for format %v", format) } for _, option := range options { - if !allowedOptions[option] { + if _, ok := allowedOptions[option]; !ok { return nil, fmt.Errorf("option %s is not allowed for format %v", option, format) } } return inputConfig, nil } -func newExternalInputConfigV2FromInputConfig( - inputConfig InputConfig, -) (externalInputConfigV2, error) { - externalInputConfigV2 := externalInputConfigV2{} - switch inputConfig.Type() { - case InputConfigTypeGitRepo: - externalInputConfigV2.GitRepo = toPointer(inputConfig.Location()) - case InputConfigTypeDirectory: - externalInputConfigV2.Directory = toPointer(inputConfig.Location()) - case InputConfigTypeModule: - externalInputConfigV2.Module = toPointer(inputConfig.Location()) - case InputConfigTypeProtoFile: - externalInputConfigV2.ProtoFile = toPointer(inputConfig.Location()) - case InputConfigTypeZipArchive: - externalInputConfigV2.ZipArchive = toPointer(inputConfig.Location()) - case InputConfigTypeTarball: - externalInputConfigV2.Tarball = toPointer(inputConfig.Location()) - case InputConfigTypeBinaryImage: - externalInputConfigV2.BinaryImage = toPointer(inputConfig.Location()) - case InputConfigTypeJSONImage: - externalInputConfigV2.JSONImage = toPointer(inputConfig.Location()) - case InputConfigTypeTextImage: - externalInputConfigV2.TextImage = toPointer(inputConfig.Location()) - default: - return externalInputConfigV2, syserror.Newf("unknown input config type: %v", inputConfig.Type()) - } - if inputConfig.Branch() != "" { - externalInputConfigV2.Branch = toPointer(inputConfig.Branch()) - } - if inputConfig.Ref() != "" { - externalInputConfigV2.Ref = toPointer(inputConfig.Ref()) - } - if inputConfig.Tag() != "" { - externalInputConfigV2.Tag = toPointer(inputConfig.Tag()) - } - externalInputConfigV2.Depth = inputConfig.Depth() - // TODO: make RecurseSubmodules return a pointer for more accurate representation - if inputConfig.RecurseSubmodules() { - externalInputConfigV2.RecurseSubmodules = toPointer(inputConfig.RecurseSubmodules()) - } - if inputConfig.Compression() != "" { - externalInputConfigV2.Compression = toPointer(inputConfig.Compression()) - } - externalInputConfigV2.StripComponents = inputConfig.StripComponents() - if inputConfig.SubDir() != "" { - externalInputConfigV2.Subdir = toPointer(inputConfig.SubDir()) - } - // TODO: make IncludePackageFiles return a pointer for more accurate representation - if inputConfig.IncludePackageFiles() { - externalInputConfigV2.IncludePackageFiles = toPointer(inputConfig.IncludePackageFiles()) - } - externalInputConfigV2.IncludePaths = inputConfig.IncludePaths() - externalInputConfigV2.ExcludePaths = inputConfig.ExcludePaths() - externalInputConfigV2.Types = inputConfig.IncludeTypes() - return externalInputConfigV2, nil -} - func (i *inputConfig) Type() InputConfigType { return i.inputType } @@ -520,7 +459,7 @@ func (i *inputConfig) Compression() string { return i.compression } -func (i *inputConfig) StripComponents() *uint32 { +func (i *inputConfig) StripComponents() uint32 { return i.stripComponents } @@ -565,3 +504,60 @@ func (i *inputConfig) IncludeTypes() []string { } func (i *inputConfig) isInputConfig() {} + +func newExternalInputConfigV2FromInputConfig( + inputConfig InputConfig, +) (externalInputConfigV2, error) { + externalInputConfigV2 := externalInputConfigV2{} + switch inputConfig.Type() { + case InputConfigTypeGitRepo: + externalInputConfigV2.GitRepo = toPointer(inputConfig.Location()) + case InputConfigTypeDirectory: + externalInputConfigV2.Directory = toPointer(inputConfig.Location()) + case InputConfigTypeModule: + externalInputConfigV2.Module = toPointer(inputConfig.Location()) + case InputConfigTypeProtoFile: + externalInputConfigV2.ProtoFile = toPointer(inputConfig.Location()) + case InputConfigTypeZipArchive: + externalInputConfigV2.ZipArchive = toPointer(inputConfig.Location()) + case InputConfigTypeTarball: + externalInputConfigV2.Tarball = toPointer(inputConfig.Location()) + case InputConfigTypeBinaryImage: + externalInputConfigV2.BinaryImage = toPointer(inputConfig.Location()) + case InputConfigTypeJSONImage: + externalInputConfigV2.JSONImage = toPointer(inputConfig.Location()) + case InputConfigTypeTextImage: + externalInputConfigV2.TextImage = toPointer(inputConfig.Location()) + default: + return externalInputConfigV2, syserror.Newf("unknown input config type: %v", inputConfig.Type()) + } + if inputConfig.Branch() != "" { + externalInputConfigV2.Branch = toPointer(inputConfig.Branch()) + } + if inputConfig.Ref() != "" { + externalInputConfigV2.Ref = toPointer(inputConfig.Ref()) + } + if inputConfig.Tag() != "" { + externalInputConfigV2.Tag = toPointer(inputConfig.Tag()) + } + externalInputConfigV2.Depth = inputConfig.Depth() + if inputConfig.RecurseSubmodules() { + externalInputConfigV2.RecurseSubmodules = toPointer(inputConfig.RecurseSubmodules()) + } + if inputConfig.Compression() != "" { + externalInputConfigV2.Compression = toPointer(inputConfig.Compression()) + } + if inputConfig.StripComponents() != 0 { + externalInputConfigV2.StripComponents = toPointer(inputConfig.StripComponents()) + } + if inputConfig.SubDir() != "" { + externalInputConfigV2.Subdir = toPointer(inputConfig.SubDir()) + } + if inputConfig.IncludePackageFiles() { + externalInputConfigV2.IncludePackageFiles = toPointer(inputConfig.IncludePackageFiles()) + } + externalInputConfigV2.IncludePaths = inputConfig.IncludePaths() + externalInputConfigV2.ExcludePaths = inputConfig.ExcludePaths() + externalInputConfigV2.Types = inputConfig.IncludeTypes() + return externalInputConfigV2, nil +} diff --git a/private/pkg/slicesext/slicesext.go b/private/pkg/slicesext/slicesext.go index 339238cc46..cd7c6479ba 100644 --- a/private/pkg/slicesext/slicesext.go +++ b/private/pkg/slicesext/slicesext.go @@ -186,6 +186,22 @@ func MapKeysToSlice[K comparable, V any](m map[K]V) []K { return s } +// MapValuesToSlice converts the map's values to a sorted slice. +// +// Duplicate values will be added. This should generally be used +// in cases where you know there is a 1-1 mapping from K to V. +func MapValuesToSortedSlice[K comparable, V Ordered](m map[K]V) []V { + s := MapValuesToSlice(m) + // TODO: Replace with slices.Sort when we only support Go versions >= 1.21. + sort.Slice( + s, + func(i int, j int) bool { + return s[i] < s[j] + }, + ) + return s +} + // MapValuesToSlice converts the map's values to a slice. // // Duplicate values will be added. This should generally be used From ca3cec8afe7f53fc62a5a58bf5597b103265613b Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Sat, 2 Dec 2023 00:25:37 -0500 Subject: [PATCH 60/66] some bufimagemodify --- .../bufimage/bufimagemodify/bufimagemodify.go | 291 +------ .../bufimagemodify/cc_enable_arenas.go | 90 --- .../bufimagemodify/cc_enable_arenas_test.go | 385 ---------- .../bufimagemodify/csharp_namespace.go | 151 ---- .../bufimagemodify/csharp_namespace_test.go | 627 --------------- .../bufimage/bufimagemodify/field_option.go | 3 +- .../bufimagemodify/file_option_sweeper.go | 115 --- .../bufimage/bufimagemodify/go_package.go | 137 ---- .../bufimagemodify/go_package_test.go | 687 ----------------- .../bufimagemodify/java_multiple_files.go | 103 --- .../java_multiple_files_test.go | 428 ----------- .../bufimagemodify/java_outer_classname.go | 97 --- .../java_outer_classname_test.go | 409 ---------- .../bufimage/bufimagemodify/java_package.go | 153 ---- .../bufimagemodify/java_package_test.go | 632 --------------- .../bufimagemodify/java_string_check_utf8.go | 90 --- .../java_string_check_utf8_test.go | 346 --------- .../bufimage/bufimagemodify/marksweeper.go | 14 + .../bufimage/bufimagemodify/multi_modifier.go | 45 -- .../bufimage/bufimagemodify/new_modify.go | 3 +- .../bufimagemodify/objc_class_prefix.go | 151 ---- .../bufimagemodify/objc_class_prefix_test.go | 718 ------------------ .../bufimage/bufimagemodify/optimize_for.go | 130 ---- .../bufimagemodify/optimize_for_test.go | 664 ---------------- .../bufimagemodify/php_metadata_namespace.go | 94 --- .../php_metadata_namespace_test.go | 358 --------- .../bufimage/bufimagemodify/php_namespace.go | 203 ----- .../bufimagemodify/php_namespace_test.go | 359 --------- .../bufimage/bufimagemodify/ruby_package.go | 136 ---- .../bufimagemodify/ruby_package_test.go | 587 -------------- .../bufpkg/bufimage/bufimagemodify/vars.go | 284 +++++++ 31 files changed, 301 insertions(+), 8189 deletions(-) delete mode 100644 private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/csharp_namespace.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/csharp_namespace_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/file_option_sweeper.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/go_package.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/go_package_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_multiple_files.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_multiple_files_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_outer_classname.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_outer_classname_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_package.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_package_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/multi_modifier.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/objc_class_prefix.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/objc_class_prefix_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/optimize_for.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/optimize_for_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/php_namespace.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/php_namespace_test.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/ruby_package.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/ruby_package_test.go create mode 100644 private/bufpkg/bufimage/bufimagemodify/vars.go diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go index 43a14fc840..f0160f12d4 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go @@ -15,307 +15,18 @@ package bufimagemodify import ( - "context" "fmt" - "path" "strconv" - "strings" "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/gen/data/datawkt" - "github.com/bufbuild/buf/private/pkg/protoversion" - "go.uber.org/zap" "google.golang.org/protobuf/types/descriptorpb" ) // TODO: remove code dealing with the old config (maps) -// Modifier modifies Images. -type Modifier interface { - // Modify modifies the Image. - Modify(context.Context, bufimage.Image) error -} - -// NewMultiModifier returns a new Modifier for the given Modifiers. -func NewMultiModifier(modifiers ...Modifier) Modifier { - switch len(modifiers) { - case 0: - return nil - case 1: - return modifiers[0] - default: - return newMultiModifier(modifiers) - } -} - -// ModifierFunc is a convenience type that implements the Modifier interface. -type ModifierFunc func(context.Context, bufimage.Image) error - -// Modify invokes the ModifierFunc with the given context and image. -func (m ModifierFunc) Modify(ctx context.Context, image bufimage.Image) error { - return m(ctx, image) -} - -// Sweeper is used to mark-and-sweep SourceCodeInfo_Locations from images. -type Sweeper interface { - // Sweep implements the ModifierFunc signature so that the Sweeper - // can be used as a Modifier. - Sweep(context.Context, bufimage.Image) error - - // mark is un-exported so that the Sweeper cannot be implemented - // outside of this package. - mark(string, []int32) -} - -// NewFileOptionSweeper constructs a new file option Sweeper that removes -// the SourceCodeInfo_Locations associated with the marks. -func NewFileOptionSweeper() Sweeper { - return newFileOptionSweeper() -} - -// Merge merges the given modifiers together so that they are run in the order -// they are provided. This is particularly useful for constructing a modifier -// from its initial 'nil' value. -// -// var modifier Modifier -// if config.JavaMultipleFiles { -// modifier = Merge(modifier, JavaMultipleFiles) -// } -func Merge(left Modifier, right Modifier) Modifier { - if left == nil { - return right - } - if right == nil { - return left - } - return NewMultiModifier(left, right) -} - -// CcEnableArenas returns a Modifier that sets the cc_enable_arenas -// file option to the given value in all of the files contained in -// the Image. -func CcEnableArenas( - logger *zap.Logger, - sweeper Sweeper, - value bool, - overrides map[string]string, -) (Modifier, error) { - validatedOverrides, err := stringOverridesToBoolOverrides(overrides) - if err != nil { - return nil, fmt.Errorf("invalid override for %s: %w", CcEnableArenasID, err) - } - return ccEnableArenas(logger, sweeper, value, validatedOverrides), nil -} - -// GoPackage returns a Modifier that sets the go_package file option -// according to the given defaultImportPathPrefix, exceptions, and -// overrides. -func GoPackage( - logger *zap.Logger, - sweeper Sweeper, - defaultImportPathPrefix string, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) (Modifier, error) { - return goPackage( - logger, - sweeper, - defaultImportPathPrefix, - except, - moduleOverrides, - overrides, - ) -} - -// JavaMultipleFiles returns a Modifier that sets the java_multiple_files -// file option to the given value in all of the files contained in -// the Image. If the preserveExistingValue is set to true, then the -// java_multiple_files option will be preserved if set. -func JavaMultipleFiles( - logger *zap.Logger, - sweeper Sweeper, - value bool, - overrides map[string]string, - preserveExistingValue bool, -) (Modifier, error) { - validatedOverrides, err := stringOverridesToBoolOverrides(overrides) - if err != nil { - return nil, fmt.Errorf("invalid override for %s: %w", JavaMultipleFilesID, err) - } - return javaMultipleFiles(logger, sweeper, value, validatedOverrides, preserveExistingValue), nil -} - -// JavaOuterClassname returns a Modifier that sets the java_outer_classname file option -// in all of the files contained in the Image based on the PascalCase of their filename. -// If the preserveExistingValue is set to true, then the java_outer_classname option will -// be preserved if set. -func JavaOuterClassname( - logger *zap.Logger, - sweeper Sweeper, - overrides map[string]string, - preserveExistingValue bool, -) Modifier { - return javaOuterClassname(logger, sweeper, overrides, preserveExistingValue) -} - -// JavaPackage returns a Modifier that sets the java_package file option -// according to the given packagePrefix. -func JavaPackage( - logger *zap.Logger, - sweeper Sweeper, - defaultPrefix string, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) (Modifier, error) { - return javaPackage( - logger, - sweeper, - defaultPrefix, - except, - moduleOverrides, - overrides, - ) -} - -// JavaStringCheckUtf8 returns a Modifier that sets the java_string_check_utf8 file option according -// to the given value. -func JavaStringCheckUtf8( - logger *zap.Logger, - sweeper Sweeper, - value bool, - overrides map[string]string, -) (Modifier, error) { - validatedOverrides, err := stringOverridesToBoolOverrides(overrides) - if err != nil { - return nil, fmt.Errorf("invalid override for %s: %w", JavaStringCheckUtf8ID, err) - } - return javaStringCheckUtf8(logger, sweeper, value, validatedOverrides), nil -} - -// OptimizeFor returns a Modifier that sets the optimize_for file -// option to the given value in all of the files contained in -// the Image. -func OptimizeFor( - logger *zap.Logger, - sweeper Sweeper, - defaultOptimizeFor descriptorpb.FileOptions_OptimizeMode, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode, - overrides map[string]string, -) (Modifier, error) { - validatedOverrides, err := stringOverridesToOptimizeModeOverrides(overrides) - if err != nil { - return nil, fmt.Errorf("invalid override for %s: %w", OptimizeForID, err) - } - return optimizeFor(logger, sweeper, defaultOptimizeFor, except, moduleOverrides, validatedOverrides), nil -} - -// GoPackageImportPathForFile returns the go_package import path for the given -// ImageFile. If the package contains a version suffix, and if there are more -// than two components, concatenate the final two components. Otherwise, we -// exclude the ';' separator and adopt the default behavior from the import path. -// -// For example, an ImageFile with `package acme.weather.v1;` will include `;weatherv1` -// in the `go_package` declaration so that the generated package is named as such. -func GoPackageImportPathForFile(imageFile bufimage.ImageFile, importPathPrefix string) string { - goPackageImportPath := path.Join(importPathPrefix, path.Dir(imageFile.Path())) - packageName := imageFile.FileDescriptorProto().GetPackage() - if _, ok := protoversion.NewPackageVersionForPackage(packageName); ok { - parts := strings.Split(packageName, ".") - if len(parts) >= 2 { - goPackageImportPath += ";" + parts[len(parts)-2] + parts[len(parts)-1] - } - } - return goPackageImportPath -} - -// ObjcClassPrefix returns a Modifier that sets the objc_class_prefix file option -// according to the package name. It is set to the uppercase first letter of each package sub-name, -// not including the package version, with the following rules: -// - If the resulting abbreviation is 2 characters, add "X". -// - If the resulting abbreviation is 1 character, add "XX". -// - If the resulting abbreviation is "GPB", change it to "GPX". -// "GPB" is reserved by Google for the Protocol Buffers implementation. -func ObjcClassPrefix( - logger *zap.Logger, - sweeper Sweeper, - defaultPrefix string, - except []bufmodule.ModuleFullName, - moduleOverride map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) Modifier { - return objcClassPrefix(logger, sweeper, defaultPrefix, except, moduleOverride, overrides) -} - -// CsharpNamespace returns a Modifier that sets the csharp_namespace file option -// according to the package name. It is set to the package name with each package sub-name capitalized. -func CsharpNamespace( - logger *zap.Logger, - sweeper Sweeper, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) Modifier { - return csharpNamespace( - logger, - sweeper, - except, - moduleOverrides, - overrides, - ) -} - -// PhpNamespace returns a Modifier that sets the php_namespace file option -// according to the package name. It is set to the package name with each package sub-name capitalized -// and each "." replaced with "\\". -func PhpNamespace( - logger *zap.Logger, - sweeper Sweeper, - overrides map[string]string, -) Modifier { - return phpNamespace(logger, sweeper, overrides) -} - -// PhpMetadataNamespace returns a Modifier that sets the php_metadata_namespace file option -// according to the package name. It appends "\\GPBMetadata" to the heuristic used by PhpNamespace. -func PhpMetadataNamespace( - logger *zap.Logger, - sweeper Sweeper, - overrides map[string]string, -) Modifier { - return phpMetadataNamespace(logger, sweeper, overrides) -} - -// RubyPackage returns a Modifier that sets the ruby_package file option -// according to the given packagePrefix. It is set to the package name with each package sub-name capitalized -// and each "." replaced with "::". -func RubyPackage( - logger *zap.Logger, - sweeper Sweeper, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) Modifier { - return rubyPackage( - logger, - sweeper, - except, - moduleOverrides, - overrides, - ) -} - -// TODO: delete this one -// isWellKnownType returns true if the given path is one of the well-known types. -func isWellKnownType(ctx context.Context, imageFile bufimage.ImageFile) bool { - return datawkt.Exists(imageFile.Path()) -} - // TODO: rename this one -func isWellKnownTypeNoCtx(imageFile bufimage.ImageFile) bool { +func isWellKnownType(imageFile bufimage.ImageFile) bool { return datawkt.Exists(imageFile.Path()) } diff --git a/private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas.go b/private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas.go deleted file mode 100644 index d4e52ac1d2..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// CcEnableArenasID is the ID of the cc_enable_arenas modifier. -const CcEnableArenasID = "CC_ENABLE_ARENAS" - -// ccEnableArenas is the SourceCodeInfo path for the cc_enable_arenas option. -// https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L420 -var ccEnableArenasPath = []int32{8, 31} - -func ccEnableArenas( - logger *zap.Logger, - sweeper Sweeper, - value bool, - overrides map[string]bool, -) Modifier { - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - modifierValue := value - if overrideValue, ok := overrides[imageFile.Path()]; ok { - modifierValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := ccEnableArenasForFile(ctx, sweeper, imageFile, modifierValue); err != nil { - return err - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", CcEnableArenasID, overrideFile) - } - } - return nil - }, - ) -} - -func ccEnableArenasForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - value bool, -) error { - descriptor := imageFile.FileDescriptorProto() - options := descriptor.GetOptions() - switch { - case isWellKnownType(ctx, imageFile): - // The file is a well-known type, don't do anything. - return nil - case options != nil && options.GetCcEnableArenas() == value: - // The option is already set to the same value, don't do anything. - return nil - case options == nil && descriptorpb.Default_FileOptions_CcEnableArenas == value: - // The option is not set, but the value we want to set is the - // same as the default, don't do anything. - return nil - } - if options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.CcEnableArenas = proto.Bool(value) - if sweeper != nil { - sweeper.mark(imageFile.Path(), ccEnableArenasPath) - } - return nil -} diff --git a/private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas_test.go b/private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas_test.go deleted file mode 100644 index 5bcc60e885..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/cc_enable_arenas_test.go +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestCcEnableArenasEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, true) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, false, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - ccEnableArenasModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, false, nil) - require.NoError(t, err) - err = ccEnableArenasModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, true) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, false, map[string]string{"a.proto": "true"}) - require.NoError(t, err) - modifier := NewMultiModifier( - ccEnableArenasModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - continue - } - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, false, map[string]string{"a.proto": "true"}) - require.NoError(t, err) - err = ccEnableArenasModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - continue - } - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - }) -} - -func TestCcEnableArenasAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, ccEnableArenasPath) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - ccEnableArenasModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - err = ccEnableArenasModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, ccEnableArenasPath) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}) - require.NoError(t, err) - modifier := NewMultiModifier( - ccEnableArenasModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - continue - } - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, ccEnableArenasPath) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}) - require.NoError(t, err) - err = ccEnableArenasModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - continue - } - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - }) -} - -func TestCcEnableArenasCcOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "ccoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, ccEnableArenasPath) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - ccEnableArenasModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - err = ccEnableArenasModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, ccEnableArenasPath) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}) - require.NoError(t, err) - modifier := NewMultiModifier( - ccEnableArenasModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - continue - } - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}) - require.NoError(t, err) - err = ccEnableArenasModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.False(t, descriptor.GetOptions().GetCcEnableArenas()) - continue - } - assert.True(t, descriptor.GetOptions().GetCcEnableArenas()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, ccEnableArenasPath, false) - }) -} - -func TestCcEnableArenasWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - ccEnableArenasModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - assert.True(t, imageFile.FileDescriptorProto().GetOptions().GetCcEnableArenas()) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - ccEnableArenasModifier, err := CcEnableArenas(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - err = ccEnableArenasModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - assert.True(t, imageFile.FileDescriptorProto().GetOptions().GetCcEnableArenas()) - } - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/csharp_namespace.go b/private/bufpkg/bufimage/bufimagemodify/csharp_namespace.go deleted file mode 100644 index 6bdca80e0b..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/csharp_namespace.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "strings" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/stringutil" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// CsharpNamespaceID is the ID of the csharp_namespace modifier. -const CsharpNamespaceID = "CSHARP_NAMESPACE" - -// csharpNamespacePath is the SourceCodeInfo path for the csharp_namespace option. -// https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L428 -var csharpNamespacePath = []int32{8, 37} - -func csharpNamespace( - logger *zap.Logger, - sweeper Sweeper, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) Modifier { - // Convert the bufmodule.ModuleFullName types into - // strings so that they're comparable. - exceptModuleFullNameStrings := make(map[string]struct{}, len(except)) - for _, moduleFullName := range except { - exceptModuleFullNameStrings[moduleFullName.String()] = struct{}{} - } - overrideModuleFullNameStrings := make(map[string]string, len(moduleOverrides)) - for moduleFullName, csharpNamespace := range moduleOverrides { - overrideModuleFullNameStrings[moduleFullName.String()] = csharpNamespace - } - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenModuleFullNameStrings := make(map[string]struct{}, len(overrideModuleFullNameStrings)) - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - csharpNamespaceValue := csharpNamespaceValue(imageFile) - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - moduleFullNameString := moduleFullName.String() - if moduleNamespaceOverride, ok := overrideModuleFullNameStrings[moduleFullNameString]; ok { - seenModuleFullNameStrings[moduleFullNameString] = struct{}{} - csharpNamespaceValue = moduleNamespaceOverride - } - } - if overrideValue, ok := overrides[imageFile.Path()]; ok { - csharpNamespaceValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := csharpNamespaceForFile( - ctx, - sweeper, - imageFile, - csharpNamespaceValue, - exceptModuleFullNameStrings, - ); err != nil { - return err - } - } - for moduleFullNameString := range overrideModuleFullNameStrings { - if _, ok := seenModuleFullNameStrings[moduleFullNameString]; !ok { - logger.Sugar().Warnf("csharp_namespace_prefix override for %q was unused", moduleFullNameString) - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", CsharpNamespaceID, overrideFile) - } - } - return nil - }, - ) -} - -func csharpNamespaceForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - csharpNamespaceValue string, - exceptModuleFullNameStrings map[string]struct{}, -) error { - if shouldSkipCsharpNamespaceForFile(ctx, imageFile, csharpNamespaceValue, exceptModuleFullNameStrings) { - // This is a well-known type or we could not resolve a non-empty csharp_namespace - // value, so this is a no-op. - return nil - } - descriptor := imageFile.FileDescriptorProto() - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.CsharpNamespace = proto.String(csharpNamespaceValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), csharpNamespacePath) - } - return nil -} - -func shouldSkipCsharpNamespaceForFile( - ctx context.Context, - imageFile bufimage.ImageFile, - csharpNamespaceValue string, - exceptModuleFullNameStrings map[string]struct{}, -) bool { - if isWellKnownType(ctx, imageFile) || csharpNamespaceValue == "" { - // This is a well-known type or we could not resolve a non-empty csharp_namespace - // value, so this is a no-op. - return true - } - - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - if _, ok := exceptModuleFullNameStrings[moduleFullName.String()]; ok { - return true - } - } - return false -} - -// csharpNamespaceValue returns the csharp_namespace for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func csharpNamespaceValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - packageParts := strings.Split(pkg, ".") - for i, part := range packageParts { - packageParts[i] = stringutil.ToPascalCase(part) - } - return strings.Join(packageParts, ".") -} diff --git a/private/bufpkg/bufimage/bufimagemodify/csharp_namespace_test.go b/private/bufpkg/bufimage/bufimagemodify/csharp_namespace_test.go deleted file mode 100644 index 27a9921fbd..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/csharp_namespace_test.go +++ /dev/null @@ -1,627 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "strings" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/pkg/normalpath" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestCsharpNamespaceEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "foo"}) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - // Overwritten with "foo" in the namespace - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetCsharpNamespace()) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "foo"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - // Overwritten with "foo" in the namespace - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetCsharpNamespace()) - }) -} - -func TestCsharpNamespaceAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, csharpNamespacePath) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, csharpNamespacePath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, csharpNamespacePath) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "bar"}) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - // Overwritten with "bar" in the namespace - assert.Equal(t, "bar", descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "bar"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - // Overwritten with "bar" in the namespace - assert.Equal(t, "bar", descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - }) -} - -func TestCsharpNamespaceOptions(t *testing.T) { - t.Parallel() - testCsharpNamespaceOptions(t, filepath.Join("testdata", "csharpoptions", "single"), "Acme.V1") - testCsharpNamespaceOptions(t, filepath.Join("testdata", "csharpoptions", "double"), "Acme.Weather.V1") - testCsharpNamespaceOptions(t, filepath.Join("testdata", "csharpoptions", "triple"), "Acme.Weather.Data.V1") - testCsharpNamespaceOptions(t, filepath.Join("testdata", "csharpoptions", "underscore"), "Acme.Weather.FooBar.V1") -} - -func testCsharpNamespaceOptions(t *testing.T, dirPath string, classPrefix string) { - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, csharpNamespacePath) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - }) - - t.Run("with SourceCodeInfo and per-file options", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, csharpNamespacePath) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, map[string]string{"override.proto": "Acme.Override.V1"}) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "Acme.Override.V1", descriptor.GetOptions().GetCsharpNamespace()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, map[string]string{"override.proto": "Acme.Override.V1"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "Acme.Override.V1", descriptor.GetOptions().GetCsharpNamespace()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - }) -} - -func TestCsharpNamespaceWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - modifiedCsharpNamespace := "Acme.Weather.V1alpha1" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetCsharpNamespace()) - assert.NotEqual(t, modifiedCsharpNamespace, descriptor.GetOptions().GetCsharpNamespace()) - continue - } - assert.Equal(t, - modifiedCsharpNamespace, - descriptor.GetOptions().GetCsharpNamespace(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier := CsharpNamespace(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetCsharpNamespace()) - assert.NotEqual(t, modifiedCsharpNamespace, descriptor.GetOptions().GetCsharpNamespace()) - continue - } - assert.Equal(t, - modifiedCsharpNamespace, - descriptor.GetOptions().GetCsharpNamespace(), - ) - } - }) -} - -func TestCsharpNamespaceWithExcept(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"a.proto": "override"}, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "", descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"a.proto": "override"}, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "", descriptor.GetOptions().GetCsharpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} - -func TestCsharpNamespaceWithOverride(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - overrideCsharpNamespacePrefix := "x.y.z" - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideCsharpNamespacePrefix, - }, - nil, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - strings.ReplaceAll(normalpath.Dir(overrideCsharpNamespacePrefix+"/"+imageFile.Path()), "/", "."), - descriptor.GetOptions().GetCsharpNamespace(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideCsharpNamespacePrefix, - }, - nil, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - strings.ReplaceAll(normalpath.Dir(overrideCsharpNamespacePrefix+"/"+imageFile.Path()), "/", "."), - descriptor.GetOptions().GetCsharpNamespace(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideCsharpNamespacePrefix, - }, - map[string]string{"a.proto": "override"}, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.FileDescriptorProto().Name != nil && *imageFile.FileDescriptorProto().Name == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetCsharpNamespace()) - continue - } - assert.Equal(t, - strings.ReplaceAll(normalpath.Dir(imageFile.Path()), "/", "."), - descriptor.GetOptions().GetCsharpNamespace(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, csharpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - csharpNamespaceModifier := CsharpNamespace( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideCsharpNamespacePrefix, - }, - map[string]string{"a.proto": "override"}, - ) - - modifier := NewMultiModifier(csharpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.FileDescriptorProto().Name != nil && *imageFile.FileDescriptorProto().Name == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetCsharpNamespace()) - continue - } - assert.Equal(t, - strings.ReplaceAll(normalpath.Dir(imageFile.Path()), "/", "."), - descriptor.GetOptions().GetCsharpNamespace(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/field_option.go b/private/bufpkg/bufimage/bufimagemodify/field_option.go index 4e981ce30b..bc9547d821 100644 --- a/private/bufpkg/bufimage/bufimagemodify/field_option.go +++ b/private/bufpkg/bufimage/bufimagemodify/field_option.go @@ -53,8 +53,7 @@ func modifyJsType( return nil } } - // TODO: this check might not be needed - if isWellKnownTypeNoCtx(imageFile) { + if isWellKnownType(imageFile) { return nil } return walk.DescriptorProtosWithPath( diff --git a/private/bufpkg/bufimage/bufimagemodify/file_option_sweeper.go b/private/bufpkg/bufimage/bufimagemodify/file_option_sweeper.go deleted file mode 100644 index 9844af3952..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/file_option_sweeper.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "fmt" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "google.golang.org/protobuf/types/descriptorpb" -) - -// fileOptionPath is the path prefix used for FileOptions. -// All file option locations are preceded by a location -// with a path set to the fileOptionPath. -// https://github.com/protocolbuffers/protobuf/blob/053966b4959bdd21e4a24e657bcb97cb9de9e8a4/src/google/protobuf/descriptor.proto#L80 -var fileOptionPath = []int32{8} - -type fileOptionSweeper struct { - // Filepath -> SourceCodeInfo_Location.Path keys. - sourceCodeInfoPaths map[string]map[string]struct{} -} - -func newFileOptionSweeper() *fileOptionSweeper { - return &fileOptionSweeper{ - sourceCodeInfoPaths: make(map[string]map[string]struct{}), - } -} - -// mark is used to mark the given SourceCodeInfo_Location indices for -// deletion. This method should be called in each of the file option -// modifiers. -func (s *fileOptionSweeper) mark(imageFilePath string, path []int32) { - paths, ok := s.sourceCodeInfoPaths[imageFilePath] - if !ok { - paths = make(map[string]struct{}) - s.sourceCodeInfoPaths[imageFilePath] = paths - } - paths[getPathKey(path)] = struct{}{} -} - -// Sweep applies all of the marks and sweeps the file option SourceCodeInfo_Locations. -func (s *fileOptionSweeper) Sweep(ctx context.Context, image bufimage.Image) error { - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if descriptor.SourceCodeInfo == nil { - continue - } - paths, ok := s.sourceCodeInfoPaths[imageFile.Path()] - if !ok { - continue - } - // We can't just match on an exact path match because the target - // file option's parent path elements would remain (i.e [8]). - // Instead, we perform an initial pass to validate that the paths - // are structured as expect, and collect all of the indices that - // we need to delete. - indices := make(map[int]struct{}, len(paths)*2) - for i, location := range descriptor.SourceCodeInfo.Location { - if _, ok := paths[getPathKey(location.Path)]; !ok { - continue - } - if i == 0 { - return fmt.Errorf("path %v must have a preceding parent path", location.Path) - } - if !int32SliceIsEqual(descriptor.SourceCodeInfo.Location[i-1].Path, fileOptionPath) { - return fmt.Errorf("path %v must have a preceding parent path equal to %v", location.Path, fileOptionPath) - } - // Add the target path and its parent. - indices[i-1] = struct{}{} - indices[i] = struct{}{} - } - // Now that we know exactly which indices to exclude, we can - // filter the SourceCodeInfo_Locations as needed. - locations := make( - []*descriptorpb.SourceCodeInfo_Location, - 0, - len(descriptor.SourceCodeInfo.Location)-len(indices), - ) - for i, location := range descriptor.SourceCodeInfo.Location { - if _, ok := indices[i]; ok { - continue - } - locations = append(locations, location) - } - descriptor.SourceCodeInfo.Location = locations - } - return nil -} - -// getPathKey returns a unique key for the given path. -func getPathKey(path []int32) string { - key := make([]byte, len(path)*4) - j := 0 - for _, elem := range path { - key[j] = byte(elem) - key[j+1] = byte(elem >> 8) - key[j+2] = byte(elem >> 16) - key[j+3] = byte(elem >> 24) - j += 4 - } - return string(key) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/go_package.go b/private/bufpkg/bufimage/bufimagemodify/go_package.go deleted file mode 100644 index 9bc9915d74..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/go_package.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "fmt" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// GoPackageID is the ID of the go_package modifier. -const GoPackageID = "GO_PACKAGE" - -// goPackagePath is the SourceCodeInfo path for the go_package option. -// https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L392 -var goPackagePath = []int32{8, 11} - -func goPackage( - logger *zap.Logger, - sweeper Sweeper, - defaultImportPathPrefix string, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) (Modifier, error) { - if defaultImportPathPrefix == "" { - return nil, fmt.Errorf("a non-empty import path prefix is required") - } - // Convert the bufmodule.ModuleFullName types into - // strings so that they're comparable. - exceptModuleFullNameStrings := make(map[string]struct{}, len(except)) - for _, moduleFullName := range except { - exceptModuleFullNameStrings[moduleFullName.String()] = struct{}{} - } - overrideModuleFullNameStrings := make(map[string]string, len(moduleOverrides)) - for moduleFullName, goPackagePrefix := range moduleOverrides { - overrideModuleFullNameStrings[moduleFullName.String()] = goPackagePrefix - } - seenModuleFullNameStrings := make(map[string]struct{}, len(overrideModuleFullNameStrings)) - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - for _, imageFile := range image.Files() { - importPathPrefix := defaultImportPathPrefix - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - moduleFullNameString := moduleFullName.String() - if modulePrefixOverride, ok := overrideModuleFullNameStrings[moduleFullNameString]; ok { - importPathPrefix = modulePrefixOverride - seenModuleFullNameStrings[moduleFullNameString] = struct{}{} - } - } - goPackageValue := GoPackageImportPathForFile(imageFile, importPathPrefix) - if overrideValue, ok := overrides[imageFile.Path()]; ok { - goPackageValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := goPackageForFile( - ctx, - sweeper, - imageFile, - goPackageValue, - exceptModuleFullNameStrings, - ); err != nil { - return err - } - } - for moduleFullNameString := range overrideModuleFullNameStrings { - if _, ok := seenModuleFullNameStrings[moduleFullNameString]; !ok { - logger.Sugar().Warnf("go_package_prefix override for %q was unused", moduleFullNameString) - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", GoPackageID, overrideFile) - } - } - return nil - }, - ), nil -} - -func goPackageForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - goPackageValue string, - exceptModuleFullNameStrings map[string]struct{}, -) error { - if shouldSkipGoPackageForFile(ctx, imageFile, exceptModuleFullNameStrings) { - return nil - } - descriptor := imageFile.FileDescriptorProto() - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.GoPackage = proto.String(goPackageValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), goPackagePath) - } - return nil -} - -func shouldSkipGoPackageForFile( - ctx context.Context, - imageFile bufimage.ImageFile, - exceptModuleFullNameStrings map[string]struct{}, -) bool { - if isWellKnownType(ctx, imageFile) && imageFile.FileDescriptorProto().GetOptions().GetGoPackage() != "" { - // The well-known type defines the go_package option, so this is a no-op. - // If a well-known type ever omits the go_package option, we make sure - // to include it. - return true - } - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - if _, ok := exceptModuleFullNameStrings[moduleFullName.String()]; ok { - return true - } - } - return false -} diff --git a/private/bufpkg/bufimage/bufimagemodify/go_package_test.go b/private/bufpkg/bufimage/bufimagemodify/go_package_test.go deleted file mode 100644 index 9d0ad9ddd0..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/go_package_test.go +++ /dev/null @@ -1,687 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "fmt" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/pkg/normalpath" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestGoPackageError(t *testing.T) { - t.Parallel() - _, err := GoPackage(zap.NewNop(), NewFileOptionSweeper(), "", nil, nil, nil) - require.Error(t, err) -} - -func TestGoPackageEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, nil) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} - -func TestGoPackageAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, goPackagePath) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, map[string]string{}) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, goPackagePath) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} - -func TestGoPackagePackageVersion(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "packageversion") - packageSuffix := "weatherv1alpha1" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, goPackagePath) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, nil) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - fmt.Sprintf("%s;%s", - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - packageSuffix, - ), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - fmt.Sprintf("%s;%s", - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - packageSuffix, - ), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, goPackagePath) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - continue - } - assert.Equal(t, - fmt.Sprintf("%s;%s", - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - packageSuffix, - ), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - continue - } - assert.Equal(t, - fmt.Sprintf("%s;%s", - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - packageSuffix, - ), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} - -func TestGoPackageWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - packageSuffix := "weatherv1alpha1" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, nil) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - modifiedGoPackage := fmt.Sprintf("%s;%s", - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - packageSuffix, - ) - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetGoPackage()) - assert.NotEqual(t, modifiedGoPackage, descriptor.GetOptions().GetGoPackage()) - continue - } - assert.Equal(t, - modifiedGoPackage, - descriptor.GetOptions().GetGoPackage(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage(zap.NewNop(), sweeper, testImportPathPrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - modifiedGoPackage := fmt.Sprintf("%s;%s", - normalpath.Dir(testImportPathPrefix+"/"+imageFile.Path()), - packageSuffix, - ) - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetGoPackage()) - assert.NotEqual(t, modifiedGoPackage, descriptor.GetOptions().GetGoPackage()) - continue - } - assert.Equal(t, - modifiedGoPackage, - descriptor.GetOptions().GetGoPackage(), - ) - } - }) -} - -func TestGoPackageWithExcept(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} - -func TestGoPackageWithOverride(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - overrideGoPackagePrefix := "github.com/foo/bar/private/private/gen/proto/go" - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideGoPackagePrefix, - }, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - normalpath.Dir(overrideGoPackagePrefix+"/"+imageFile.Path()), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideGoPackagePrefix, - }, - nil, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - normalpath.Dir(overrideGoPackagePrefix+"/"+imageFile.Path()), - descriptor.GetOptions().GetGoPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - goPackageModifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideGoPackagePrefix, - }, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(goPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := GoPackage( - zap.NewNop(), - sweeper, - testImportPathPrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideGoPackagePrefix, - }, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetGoPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_multiple_files.go b/private/bufpkg/bufimage/bufimagemodify/java_multiple_files.go deleted file mode 100644 index f0385ae109..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_multiple_files.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -const ( - // DefaultJavaMultipleFilesValue is the default value for the java_multiple_files modifier. - DefaultJavaMultipleFilesValue = true - - // JavaMultipleFilesID is the ID of the java_multiple_files modifier. - JavaMultipleFilesID = "JAVA_MULTIPLE_FILES" -) - -// javaMultipleFilesPath is the SourceCodeInfo path for the java_multiple_files option. -// https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L364 -var javaMultipleFilesPath = []int32{8, 10} - -func javaMultipleFiles( - logger *zap.Logger, - sweeper Sweeper, - value bool, - overrides map[string]bool, - preserveExistingValue bool, -) Modifier { - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - modifierValue := value - if overrideValue, ok := overrides[imageFile.Path()]; ok { - modifierValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := javaMultipleFilesForFile(ctx, sweeper, imageFile, modifierValue, preserveExistingValue); err != nil { - return err - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", JavaMultipleFilesID, overrideFile) - } - } - return nil - }, - ) -} - -func javaMultipleFilesForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - value bool, - preserveExistingValue bool, -) error { - descriptor := imageFile.FileDescriptorProto() - options := descriptor.GetOptions() - switch { - case isWellKnownType(ctx, imageFile): - // The file is a well-known type, don't do anything. - return nil - case options != nil: - if preserveExistingValue && options.JavaMultipleFiles != nil { - // This option is explicitly set to a value - preserve existing value. - return nil - } - if options.GetJavaMultipleFiles() == value { - // The option is already set to the same value, don't do anything. - return nil - } - case options == nil && descriptorpb.Default_FileOptions_JavaMultipleFiles == value: - // The option is not set, but the value we want to set is the - // same as the default, don't do anything. - return nil - } - if options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.JavaMultipleFiles = proto.Bool(value) - if sweeper != nil { - sweeper.mark(imageFile.Path(), javaMultipleFilesPath) - } - return nil -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_multiple_files_test.go b/private/bufpkg/bufimage/bufimagemodify/java_multiple_files_test.go deleted file mode 100644 index 059e8f78d3..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_multiple_files_test.go +++ /dev/null @@ -1,428 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestJavaMultipleFilesEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, true) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, nil, false) - require.NoError(t, err) - modifier := NewMultiModifier( - javaMultipleFilesModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, nil, false) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, true) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}, false) - require.NoError(t, err) - modifier := NewMultiModifier( - javaMultipleFilesModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}, false) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) -} - -func TestJavaMultipleFilesAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaMultipleFilesPath) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, nil, false) - require.NoError(t, err) - modifier := NewMultiModifier( - javaMultipleFilesModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, nil, false) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaMultipleFilesPath) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}, false) - require.NoError(t, err) - modifier := NewMultiModifier( - javaMultipleFilesModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - continue - } - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaMultipleFilesPath) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, map[string]string{"a.proto": "false"}, false) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - continue - } - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) - - t.Run("with preserveExistingValue", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, nil, true) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - continue - } - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) -} - -func TestJavaMultipleFilesJavaOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "javaoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaMultipleFilesPath) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, false, nil, false) - require.NoError(t, err) - modifier := NewMultiModifier( - javaMultipleFilesModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, false, nil, false) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - continue - } - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaMultipleFilesPath) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, false, map[string]string{"override.proto": "true"}, false) - require.NoError(t, err) - modifier := NewMultiModifier( - javaMultipleFilesModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - continue - } - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, false, map[string]string{"override.proto": "true"}, false) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - continue - } - assert.False(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) - - t.Run("with preserveExistingValue", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, false, nil, true) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaMultipleFiles()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaMultipleFilesPath, false) - }) -} - -func TestJavaMultipleFilesWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, nil, false) - require.NoError(t, err) - modifier := NewMultiModifier( - javaMultipleFilesModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - assert.True(t, imageFile.FileDescriptorProto().GetOptions().GetJavaMultipleFiles()) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - javaMultipleFilesModifier, err := JavaMultipleFiles(zap.NewNop(), sweeper, true, nil, false) - require.NoError(t, err) - err = javaMultipleFilesModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - assert.True(t, imageFile.FileDescriptorProto().GetOptions().GetJavaMultipleFiles()) - } - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_outer_classname.go b/private/bufpkg/bufimage/bufimagemodify/java_outer_classname.go deleted file mode 100644 index f46ae5a83d..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_outer_classname.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/normalpath" - "github.com/bufbuild/buf/private/pkg/stringutil" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// JavaOuterClassNameID is the ID for the java_outer_classname modifier. -const JavaOuterClassNameID = "JAVA_OUTER_CLASSNAME" - -// javaOuterClassnamePath is the SourceCodeInfo path for the java_outer_classname option. -// https://github.com/protocolbuffers/protobuf/blob/87d140f851131fb8a6e8a80449cf08e73e568259/src/google/protobuf/descriptor.proto#L356 -var javaOuterClassnamePath = []int32{8, 8} - -func javaOuterClassname( - logger *zap.Logger, - sweeper Sweeper, - overrides map[string]string, - preserveExistingValue bool, -) Modifier { - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - javaOuterClassnameValue := javaOuterClassnameValue(imageFile) - if overrideValue, ok := overrides[imageFile.Path()]; ok { - javaOuterClassnameValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := javaOuterClassnameForFile(ctx, sweeper, imageFile, javaOuterClassnameValue, preserveExistingValue); err != nil { - return err - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", JavaOuterClassNameID, overrideFile) - } - } - return nil - }, - ) -} - -func javaOuterClassnameForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - javaOuterClassnameValue string, - preserveExistingValue bool, -) error { - if isWellKnownType(ctx, imageFile) { - // The file is a well-known type - don't override the value. - return nil - } - descriptor := imageFile.FileDescriptorProto() - options := descriptor.GetOptions() - if options != nil && options.JavaOuterClassname != nil && preserveExistingValue { - // The option is explicitly set in the file - don't override it if we want to preserve the existing value. - return nil - } - if options.GetJavaOuterClassname() == javaOuterClassnameValue { - // The file already defines the java_outer_classname option with the given value, so this is a no-op. - return nil - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.JavaOuterClassname = proto.String(javaOuterClassnameValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), javaOuterClassnamePath) - } - return nil -} - -func javaOuterClassnameValue(imageFile bufimage.ImageFile) string { - return stringutil.ToPascalCase(normalpath.Base(imageFile.Path())) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_outer_classname_test.go b/private/bufpkg/bufimage/bufimagemodify/java_outer_classname_test.go deleted file mode 100644 index 1532177fd0..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_outer_classname_test.go +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/pkg/normalpath" - "github.com/bufbuild/buf/private/pkg/stringutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestJavaOuterClassnameEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, nil, false), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "AProto", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - - sweeper := NewFileOptionSweeper() - err := JavaOuterClassname(zap.NewNop(), sweeper, nil, false).Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "AProto", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}, false), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - - sweeper := NewFileOptionSweeper() - err := JavaOuterClassname(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}, false).Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - }) - - t.Run("with preserveExistingValue", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, nil, true), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "AProto", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - }) -} - -func TestJavaOuterClassnameAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaOuterClassnamePath) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, nil, false), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "AProto", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - - sweeper := NewFileOptionSweeper() - err := JavaOuterClassname(zap.NewNop(), sweeper, nil, false).Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "AProto", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaOuterClassnamePath) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}, false), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - - sweeper := NewFileOptionSweeper() - err := JavaOuterClassname(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}, false).Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - }) - - t.Run("with preserveExistingValue", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaOuterClassnamePath) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, nil, true), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaOuterClassnamePath) - }) -} - -func TestJavaOuterClassnameJavaOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "javaoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaOuterClassnamePath) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, nil, false), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, stringutil.ToPascalCase(normalpath.Base(imageFile.Path())), descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - - sweeper := NewFileOptionSweeper() - err := JavaOuterClassname(zap.NewNop(), sweeper, nil, false).Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, stringutil.ToPascalCase(normalpath.Base(imageFile.Path())), descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaOuterClassnamePath) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, map[string]string{"override.proto": "override"}, false), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetJavaOuterClassname()) - continue - } - assert.Equal(t, "JavaFileProto", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - - sweeper := NewFileOptionSweeper() - err := JavaOuterClassname(zap.NewNop(), sweeper, map[string]string{"override.proto": "override"}, false).Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetJavaOuterClassname()) - continue - } - assert.Equal(t, "JavaFileProto", descriptor.GetOptions().GetJavaOuterClassname()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaOuterClassnamePath, false) - }) -} - -func TestJavaOuterClassnameWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - modifier := NewMultiModifier( - JavaOuterClassname(zap.NewNop(), sweeper, nil, false), - ModifierFunc(sweeper.Sweep), - ) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.Equal(t, javaOuterClassnameValue(imageFile), descriptor.GetOptions().GetJavaOuterClassname()) - continue - } - assert.Equal(t, - "AProto", - descriptor.GetOptions().GetJavaOuterClassname(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - err := JavaOuterClassname(zap.NewNop(), sweeper, nil, false).Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.Equal(t, javaOuterClassnameValue(imageFile), descriptor.GetOptions().GetJavaOuterClassname()) - continue - } - assert.Equal(t, - "AProto", - descriptor.GetOptions().GetJavaOuterClassname(), - ) - } - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_package.go b/private/bufpkg/bufimage/bufimagemodify/java_package.go deleted file mode 100644 index 97820b9c9c..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_package.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "fmt" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -const ( - // DefaultJavaPackagePrefix is the default java_package prefix used in the java_package modifier. - DefaultJavaPackagePrefix = "com" - - // JavaPackageID is the ID of the java_package modifier. - JavaPackageID = "JAVA_PACKAGE" -) - -// javaPackagePath is the SourceCodeInfo path for the java_package option. -// https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L348 -var javaPackagePath = []int32{8, 1} - -func javaPackage( - logger *zap.Logger, - sweeper Sweeper, - defaultPackagePrefix string, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) (Modifier, error) { - if defaultPackagePrefix == "" { - return nil, fmt.Errorf("a non-empty package prefix is required") - } - // Convert the bufmodule.ModuleFullName types into - // strings so that they're comparable. - exceptModuleFullNameStrings := make(map[string]struct{}, len(except)) - for _, moduleFullName := range except { - exceptModuleFullNameStrings[moduleFullName.String()] = struct{}{} - } - overrideModuleFullNameStrings := make(map[string]string, len(moduleOverrides)) - for moduleFullName, javaPackagePrefix := range moduleOverrides { - overrideModuleFullNameStrings[moduleFullName.String()] = javaPackagePrefix - } - seenModuleFullNameStrings := make(map[string]struct{}, len(overrideModuleFullNameStrings)) - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - for _, imageFile := range image.Files() { - packagePrefix := defaultPackagePrefix - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - moduleFullNameString := moduleFullName.String() - if modulePrefixOverride, ok := overrideModuleFullNameStrings[moduleFullNameString]; ok { - packagePrefix = modulePrefixOverride - seenModuleFullNameStrings[moduleFullNameString] = struct{}{} - } - } - javaPackageValue := javaPackageValue(imageFile, packagePrefix) - if overridePackagePrefix, ok := overrides[imageFile.Path()]; ok { - javaPackageValue = overridePackagePrefix - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := javaPackageForFile( - ctx, - sweeper, - imageFile, - javaPackageValue, - exceptModuleFullNameStrings, - ); err != nil { - return err - } - } - for moduleFullNameString := range overrideModuleFullNameStrings { - if _, ok := seenModuleFullNameStrings[moduleFullNameString]; !ok { - logger.Sugar().Warnf("java_package_prefix override for %q was unused", moduleFullNameString) - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", JavaPackageID, overrideFile) - } - } - return nil - }, - ), nil -} - -func javaPackageForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - javaPackageValue string, - exceptModuleFullNameStrings map[string]struct{}, -) error { - if shouldSkipJavaPackageForFile(ctx, imageFile, javaPackageValue, exceptModuleFullNameStrings) { - return nil - } - descriptor := imageFile.FileDescriptorProto() - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.JavaPackage = proto.String(javaPackageValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), javaPackagePath) - } - return nil -} - -func shouldSkipJavaPackageForFile( - ctx context.Context, - imageFile bufimage.ImageFile, - javaPackageValue string, - exceptModuleFullNameStrings map[string]struct{}, -) bool { - if isWellKnownType(ctx, imageFile) || javaPackageValue == "" { - // This is a well-known type or we could not resolve a non-empty java_package - // value, so this is a no-op. - return true - } - - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - if _, ok := exceptModuleFullNameStrings[moduleFullName.String()]; ok { - return true - } - } - return false -} - -// javaPackageValue returns the java_package for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func javaPackageValue(imageFile bufimage.ImageFile, packagePrefix string) string { - if pkg := imageFile.FileDescriptorProto().GetPackage(); pkg != "" { - return packagePrefix + "." + pkg - } - return "" -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_package_test.go b/private/bufpkg/bufimage/bufimagemodify/java_package_test.go deleted file mode 100644 index b89d46b12e..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_package_test.go +++ /dev/null @@ -1,632 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -const testJavaPackagePrefix = "com" - -func TestJavaPackageError(t *testing.T) { - t.Parallel() - _, err := JavaPackage(zap.NewNop(), NewFileOptionSweeper(), "", nil, nil, nil) - require.Error(t, err) -} - -func TestJavaPackageEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - }) -} - -func TestJavaPackageAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaPackagePath) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaPackagePath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaPackagePath) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, map[string]string{"a.proto": "override"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) -} - -func TestJavaPackageJavaOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "javaoptions") - modifiedJavaPackage := "com.acme.weather" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaPackagePath) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, modifiedJavaPackage, descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, modifiedJavaPackage, descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaPackagePath) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, map[string]string{"override.proto": "override"}) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - continue - } - assert.Equal(t, modifiedJavaPackage, descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage(zap.NewNop(), sweeper, testJavaPackagePrefix, nil, nil, map[string]string{"override.proto": "override"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - continue - } - assert.Equal(t, modifiedJavaPackage, descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) -} - -func TestJavaPackageWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - javaPackagePrefix := "org" - modifiedJavaPackage := "org.acme.weather.v1alpha1" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage(zap.NewNop(), sweeper, javaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetJavaPackage()) - assert.NotEqual(t, modifiedJavaPackage, descriptor.GetOptions().GetJavaPackage()) - continue - } - assert.Equal(t, - modifiedJavaPackage, - descriptor.GetOptions().GetJavaPackage(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage(zap.NewNop(), sweeper, javaPackagePrefix, nil, nil, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetJavaPackage()) - assert.NotEqual(t, modifiedJavaPackage, descriptor.GetOptions().GetJavaPackage()) - continue - } - assert.Equal(t, - modifiedJavaPackage, - descriptor.GetOptions().GetJavaPackage(), - ) - } - }) -} - -func TestJavaPackageWithExcept(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "javaemptyoptions") - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) -} - -func TestJavaPackageWithOverride(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "javaemptyoptions") - overrideJavaPackagePrefix := "foo.bar" - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideJavaPackagePrefix, - }, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - overrideJavaPackagePrefix+"."+descriptor.GetPackage(), - descriptor.GetOptions().GetJavaPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideJavaPackagePrefix, - }, - nil, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - overrideJavaPackagePrefix+"."+descriptor.GetPackage(), - descriptor.GetOptions().GetJavaPackage(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - - sweeper := NewFileOptionSweeper() - javaPackageModifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideJavaPackagePrefix, - }, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(javaPackageModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaPackage( - zap.NewNop(), - sweeper, - testJavaPackagePrefix, - nil, - map[bufmodule.ModuleFullName]string{ - testModuleFullName: overrideJavaPackagePrefix, - }, - map[string]string{"a.proto": "override"}, - ) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetJavaPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaPackagePath, false) - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8.go b/private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8.go deleted file mode 100644 index 767455d69a..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// JavaStringCheckUtf8ID is the ID of the java_string_check_utf8 modifier. -const JavaStringCheckUtf8ID = "JAVA_STRING_CHECK_UTF8" - -// javaStringCheckUtf8Path is the SourceCodeInfo path for the java_string_check_utf8 option. -// https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L375 -var javaStringCheckUtf8Path = []int32{8, 27} - -func javaStringCheckUtf8( - logger *zap.Logger, - sweeper Sweeper, - value bool, - overrides map[string]bool, -) Modifier { - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - modifierValue := value - if overrideValue, ok := overrides[imageFile.Path()]; ok { - modifierValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := javaStringCheckUtf8ForFile(ctx, sweeper, imageFile, modifierValue); err != nil { - return err - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", JavaStringCheckUtf8ID, overrideFile) - } - } - return nil - }, - ) -} - -func javaStringCheckUtf8ForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - value bool, -) error { - descriptor := imageFile.FileDescriptorProto() - options := descriptor.GetOptions() - switch { - case isWellKnownType(ctx, imageFile): - // The file is a well-known type, don't do anything. - return nil - case options != nil && options.GetJavaStringCheckUtf8() == value: - // The option is already set to the same value, don't do anything. - return nil - case options == nil && descriptorpb.Default_FileOptions_JavaStringCheckUtf8 == value: - // The option is not set, but the value we want to set is the - // same as the default, don't do anything. - return nil - } - if options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.JavaStringCheckUtf8 = proto.Bool(value) - if sweeper != nil { - sweeper.mark(imageFile.Path(), javaStringCheckUtf8Path) - } - return nil -} diff --git a/private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8_test.go b/private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8_test.go deleted file mode 100644 index 63f3e99a2b..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/java_string_check_utf8_test.go +++ /dev/null @@ -1,346 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestJavaStringCheckUtf8EmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, true) - - sweeper := NewFileOptionSweeper() - JavaStringCheckUtf8Modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, nil) - require.NoError(t, err) - modifier := NewMultiModifier(JavaStringCheckUtf8Modifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, true) - - sweeper := NewFileOptionSweeper() - JavaStringCheckUtf8Modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, map[string]string{"a.proto": "true"}) - require.NoError(t, err) - modifier := NewMultiModifier(JavaStringCheckUtf8Modifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, map[string]string{"a.proto": "true"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - }) -} - -func TestJavaStringCheckUtf8AllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaStringCheckUtf8Path) - - sweeper := NewFileOptionSweeper() - JavaStringCheckUtf8Modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, nil) - require.NoError(t, err) - modifier := NewMultiModifier(JavaStringCheckUtf8Modifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaStringCheckUtf8Path) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaStringCheckUtf8Path) - - sweeper := NewFileOptionSweeper() - JavaStringCheckUtf8Modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, map[string]string{"a.proto": "true"}) - require.NoError(t, err) - modifier := NewMultiModifier(JavaStringCheckUtf8Modifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - continue - } - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, false, map[string]string{"a.proto": "true"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - continue - } - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - }) -} - -func TestJavaStringCheckUtf8JavaOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "javaoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaStringCheckUtf8Path) - - sweeper := NewFileOptionSweeper() - JavaStringCheckUtf8Modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - modifier := NewMultiModifier(JavaStringCheckUtf8Modifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, javaStringCheckUtf8Path) - - sweeper := NewFileOptionSweeper() - JavaStringCheckUtf8Modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, true, map[string]string{"override.proto": "false"}) - require.NoError(t, err) - modifier := NewMultiModifier(JavaStringCheckUtf8Modifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - continue - } - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, javaStringCheckUtf8Path, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, true, map[string]string{"override.proto": "false"}) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - continue - } - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - }) -} - -func TestJavaStringCheckUtf8WellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - JavaStringCheckUtf8Modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - modifier := NewMultiModifier(JavaStringCheckUtf8Modifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - continue - } - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier, err := JavaStringCheckUtf8(zap.NewNop(), sweeper, true, nil) - require.NoError(t, err) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.False(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - continue - } - assert.True(t, descriptor.GetOptions().GetJavaStringCheckUtf8()) - } - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/marksweeper.go b/private/bufpkg/bufimage/bufimagemodify/marksweeper.go index 946251a965..41f587d09f 100644 --- a/private/bufpkg/bufimage/bufimagemodify/marksweeper.go +++ b/private/bufpkg/bufimage/bufimagemodify/marksweeper.go @@ -172,3 +172,17 @@ func Int32SliceIsEqual(x []int32, y []int32) bool { } return true } + +// getPathKey returns a unique key for the given path. +func getPathKey(path []int32) string { + key := make([]byte, len(path)*4) + j := 0 + for _, elem := range path { + key[j] = byte(elem) + key[j+1] = byte(elem >> 8) + key[j+2] = byte(elem >> 16) + key[j+3] = byte(elem >> 24) + j += 4 + } + return string(key) +} diff --git a/private/bufpkg/bufimage/bufimagemodify/multi_modifier.go b/private/bufpkg/bufimage/bufimagemodify/multi_modifier.go deleted file mode 100644 index 4d1064ab04..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/multi_modifier.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" -) - -type multiModifier struct { - delegates []Modifier -} - -func newMultiModifier( - delegates []Modifier, -) *multiModifier { - return &multiModifier{ - delegates: delegates, - } -} - -func (m *multiModifier) Modify( - ctx context.Context, - image bufimage.Image, -) error { - for _, delegate := range m.delegates { - if err := delegate.Modify(ctx, image); err != nil { - return err - } - } - return nil -} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/new_modify.go index 52f181e97f..e6ec60bf1a 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/new_modify.go @@ -33,10 +33,9 @@ func Modify( if !config.Enabled() { return nil } - // TODO: in v2 modify field options as well sweeper := newMarkSweeper(image) for _, imageFile := range image.Files() { - if isWellKnownType(ctx, imageFile) { + if isWellKnownType(imageFile) { continue } modifyFuncs := []func(*markSweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error{ diff --git a/private/bufpkg/bufimage/bufimagemodify/objc_class_prefix.go b/private/bufpkg/bufimage/bufimagemodify/objc_class_prefix.go deleted file mode 100644 index be45df05e9..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/objc_class_prefix.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "strings" - "unicode" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/pkg/protoversion" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// ObjcClassPrefixID is the ID of the objc_class_prefix modifier. -const ObjcClassPrefixID = "OBJC_CLASS_PREFIX" - -// objcClassPrefixPath is the SourceCodeInfo path for the objc_class_prefix option. -// https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L425 -var objcClassPrefixPath = []int32{8, 36} - -func objcClassPrefix( - logger *zap.Logger, - sweeper Sweeper, - defaultPrefix string, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) Modifier { - // Convert the bufmodule.ModuleFullName types into - // strings so that they're comparable. - exceptModuleFullNameStrings := make(map[string]struct{}, len(except)) - for _, moduleFullName := range except { - exceptModuleFullNameStrings[moduleFullName.String()] = struct{}{} - } - overrideModuleFullNameStrings := make(map[string]string, len(moduleOverrides)) - for moduleFullName, goPackagePrefix := range moduleOverrides { - overrideModuleFullNameStrings[moduleFullName.String()] = goPackagePrefix - } - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenModuleFullNameStrings := make(map[string]struct{}, len(overrideModuleFullNameStrings)) - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - objcClassPrefixValue := objcClassPrefixValue(imageFile) - if defaultPrefix != "" { - objcClassPrefixValue = defaultPrefix - } - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - moduleFullNameString := moduleFullName.String() - if modulePrefixOverride, ok := overrideModuleFullNameStrings[moduleFullNameString]; ok { - objcClassPrefixValue = modulePrefixOverride - seenModuleFullNameStrings[moduleFullNameString] = struct{}{} - } - } - if overrideValue, ok := overrides[imageFile.Path()]; ok { - objcClassPrefixValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := objcClassPrefixForFile(ctx, sweeper, imageFile, objcClassPrefixValue, exceptModuleFullNameStrings); err != nil { - return err - } - } - for moduleFullNameString := range overrideModuleFullNameStrings { - if _, ok := seenModuleFullNameStrings[moduleFullNameString]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", ObjcClassPrefixID, moduleFullNameString) - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", ObjcClassPrefixID, overrideFile) - } - } - return nil - }, - ) -} - -func objcClassPrefixForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - objcClassPrefixValue string, - exceptModuleFullNameStrings map[string]struct{}, -) error { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(ctx, imageFile) || objcClassPrefixValue == "" { - // This is a well-known type or we could not resolve a non-empty objc_class_prefix - // value, so this is a no-op. - return nil - } - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - if _, ok := exceptModuleFullNameStrings[moduleFullName.String()]; ok { - return nil - } - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.ObjcClassPrefix = proto.String(objcClassPrefixValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), objcClassPrefixPath) - } - return nil -} - -// objcClassPrefixValue returns the objc_class_prefix for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func objcClassPrefixValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - _, hasPackageVersion := protoversion.NewPackageVersionForPackage(pkg) - packageParts := strings.Split(pkg, ".") - var prefixParts []rune - for i, part := range packageParts { - // Check if last part is a version before appending. - if i == len(packageParts)-1 && hasPackageVersion { - continue - } - // Probably should never be a non-ASCII character, - // but why not support it just in case? - runeSlice := []rune(part) - prefixParts = append(prefixParts, unicode.ToUpper(runeSlice[0])) - } - for len(prefixParts) < 3 { - prefixParts = append(prefixParts, 'X') - } - prefix := string(prefixParts) - if prefix == "GPB" { - prefix = "GPX" - } - return prefix -} diff --git a/private/bufpkg/bufimage/bufimagemodify/objc_class_prefix_test.go b/private/bufpkg/bufimage/bufimagemodify/objc_class_prefix_test.go deleted file mode 100644 index 49da4ab054..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/objc_class_prefix_test.go +++ /dev/null @@ -1,718 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestObjcClassPrefixEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - modifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, map[string]string{"a.proto": "override"}) - - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - require.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - modifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - require.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - }) -} - -func TestObjcClassPrefixAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - modifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, map[string]string{"a.proto": "override"}) - - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - modifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) -} - -func TestObjcClassPrefixObjcOptions(t *testing.T) { - t.Parallel() - testObjcClassPrefixOptions(t, filepath.Join("testdata", "objcoptions", "single"), "AXX") - testObjcClassPrefixOptions(t, filepath.Join("testdata", "objcoptions", "double"), "AWX") - testObjcClassPrefixOptions(t, filepath.Join("testdata", "objcoptions", "triple"), "AWD") - testObjcClassPrefixOptions(t, filepath.Join("testdata", "objcoptions", "unversioned"), "AWD") - testObjcClassPrefixOptions(t, filepath.Join("testdata", "objcoptions", "gpb"), "GPX") -} - -func testObjcClassPrefixOptions(t *testing.T, dirPath string, classPrefix string) { - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - modifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, map[string]string{"override.proto": "override"}) - - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - modifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, map[string]string{"override.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) -} - -func TestObjcClassPrefixWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - modifiedObjcClassPrefix := "AWX" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetObjcClassPrefix()) - assert.NotEqual(t, modifiedObjcClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, - modifiedObjcClassPrefix, - descriptor.GetOptions().GetObjcClassPrefix(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier := ObjcClassPrefix(zap.NewNop(), sweeper, "", nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - assert.NotEmpty(t, descriptor.GetOptions().GetObjcClassPrefix()) - assert.NotEqual(t, modifiedObjcClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, - modifiedObjcClassPrefix, - descriptor.GetOptions().GetObjcClassPrefix(), - ) - } - }) -} - -func TestObjcClassPrefixWithDefault(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "objcoptions", "single") - defaultClassPrefix := "DEFAULT" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - nil, - nil, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, defaultClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - nil, - nil, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, defaultClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - nil, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, defaultClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - nil, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, defaultClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) -} - -func TestObjcClassPrefixWithExcept(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "objcoptions", "single") - defaultClassPrefix := "DEFAULT" - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - // Should still be non-empty because the module is skipped. - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - // Should still be non-empty because the module is skipped. - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) -} - -func TestObjcClassPrefixWithOverride(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "objcoptions", "single") - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - defaultClassPrefix := "DEFAULT" - overrideClassPrefix := "MODULE_OVERRIDE" - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideClassPrefix}, - nil, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, overrideClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideClassPrefix}, - nil, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, overrideClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, objcClassPrefixPath) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideClassPrefix}, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, overrideClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - - sweeper := NewFileOptionSweeper() - objcClassPrefixModifier := ObjcClassPrefix( - zap.NewNop(), - sweeper, - defaultClassPrefix, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideClassPrefix}, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(objcClassPrefixModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetObjcClassPrefix()) - continue - } - assert.Equal(t, overrideClassPrefix, descriptor.GetOptions().GetObjcClassPrefix()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, objcClassPrefixPath, false) - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/optimize_for.go b/private/bufpkg/bufimage/bufimagemodify/optimize_for.go deleted file mode 100644 index b0e477b8e3..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/optimize_for.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "go.uber.org/zap" - "google.golang.org/protobuf/types/descriptorpb" -) - -// OptimizeForID is the ID for the optimize_for modifier. -const OptimizeForID = "OPTIMIZE_FOR" - -// optimizeFor is the SourceCodeInfo path for the optimize_for option. -// https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L385 -var optimizeForPath = []int32{8, 9} - -func optimizeFor( - logger *zap.Logger, - sweeper Sweeper, - defaultOptimizeFor descriptorpb.FileOptions_OptimizeMode, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode, - overrides map[string]descriptorpb.FileOptions_OptimizeMode, -) Modifier { - // Convert the bufmodule.ModuleFullName types into - // strings so that they're comparable. - exceptModuleFullNameStrings := make(map[string]struct{}, len(except)) - for _, moduleFullName := range except { - exceptModuleFullNameStrings[moduleFullName.String()] = struct{}{} - } - overrideModuleFullNameStrings := make( - map[string]descriptorpb.FileOptions_OptimizeMode, - len(moduleOverrides), - ) - for moduleFullName, optimizeFor := range moduleOverrides { - overrideModuleFullNameStrings[moduleFullName.String()] = optimizeFor - } - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenModuleFullNameStrings := make(map[string]struct{}, len(overrideModuleFullNameStrings)) - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - modifierValue := defaultOptimizeFor - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - moduleFullNameString := moduleFullName.String() - if optimizeForOverrdie, ok := overrideModuleFullNameStrings[moduleFullNameString]; ok { - modifierValue = optimizeForOverrdie - seenModuleFullNameStrings[moduleFullNameString] = struct{}{} - } - } - if overrideValue, ok := overrides[imageFile.Path()]; ok { - modifierValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := optimizeForForFile( - ctx, - sweeper, - imageFile, - modifierValue, - exceptModuleFullNameStrings, - ); err != nil { - return err - } - } - for moduleFullNameString := range overrideModuleFullNameStrings { - if _, ok := seenModuleFullNameStrings[moduleFullNameString]; !ok { - logger.Sugar().Warnf("optimize_for override for %q was unused", moduleFullNameString) - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", OptimizeForID, overrideFile) - } - } - return nil - }, - ) -} - -func optimizeForForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - value descriptorpb.FileOptions_OptimizeMode, - exceptModuleFullNameStrings map[string]struct{}, -) error { - descriptor := imageFile.FileDescriptorProto() - options := descriptor.GetOptions() - switch { - case isWellKnownType(ctx, imageFile): - // The file is a well-known type, don't do anything. - return nil - case options != nil && options.GetOptimizeFor() == value: - // The option is already set to the same value, don't do anything. - return nil - case options == nil && descriptorpb.Default_FileOptions_OptimizeFor == value: - // The option is not set, but the value we want to set is the - // same as the default, don't do anything. - return nil - } - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - if _, ok := exceptModuleFullNameStrings[moduleFullName.String()]; ok { - return nil - } - } - if options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.OptimizeFor = &value - if sweeper != nil { - sweeper.mark(imageFile.Path(), optimizeForPath) - } - return nil -} diff --git a/private/bufpkg/bufimage/bufimagemodify/optimize_for_test.go b/private/bufpkg/bufimage/bufimagemodify/optimize_for_test.go deleted file mode 100644 index 08793c2b42..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/optimize_for_test.go +++ /dev/null @@ -1,664 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "google.golang.org/protobuf/types/descriptorpb" -) - -func TestOptimizeForEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, true) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - optimizeForModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, nil) - require.NoError(t, err) - err = optimizeForModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, true) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, map[string]string{"a.proto": "SPEED"}) - require.NoError(t, err) - modifier := NewMultiModifier( - optimizeForModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - continue - } - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, map[string]string{"a.proto": "SPEED"}) - require.NoError(t, err) - err = optimizeForModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - continue - } - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - }) -} - -func TestOptimizeForAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, optimizeForPath) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - optimizeForModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, nil) - require.NoError(t, err) - err = optimizeForModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, optimizeForPath) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, map[string]string{"a.proto": "SPEED"}) - require.NoError(t, err) - modifier := NewMultiModifier( - optimizeForModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - continue - } - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, optimizeForPath) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_LITE_RUNTIME, nil, nil, map[string]string{"a.proto": "SPEED"}) - require.NoError(t, err) - err = optimizeForModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - continue - } - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - }) -} - -func TestOptimizeForCcOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "ccoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, optimizeForPath) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_SPEED, nil, nil, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - optimizeForModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_SPEED, nil, nil, nil) - require.NoError(t, err) - err = optimizeForModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, optimizeForPath) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_SPEED, nil, nil, map[string]string{"a.proto": "LITE_RUNTIME"}) - require.NoError(t, err) - modifier := NewMultiModifier( - optimizeForModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - continue - } - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - } - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_SPEED, nil, nil, map[string]string{"a.proto": "LITE_RUNTIME"}) - require.NoError(t, err) - err = optimizeForModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, descriptorpb.FileOptions_LITE_RUNTIME, descriptor.GetOptions().GetOptimizeFor()) - continue - } - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, optimizeForPath, false) - }) -} - -func TestOptimizeForWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_SPEED, nil, nil, nil) - require.NoError(t, err) - modifier := NewMultiModifier( - optimizeForModifier, - ModifierFunc(sweeper.Sweep), - ) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor(zap.NewNop(), sweeper, descriptorpb.FileOptions_SPEED, nil, nil, nil) - require.NoError(t, err) - err = optimizeForModifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, descriptorpb.FileOptions_SPEED, descriptor.GetOptions().GetOptimizeFor()) - } - }) -} - -func TestOptimizeForWithExcept(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{ - "a.proto": "LITE_RUNTIME", - }, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{ - "a.proto": "SPEED", - }, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} - -func TestOptimizeForWithOverride(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - overrideOptimizeFor := descriptorpb.FileOptions_LITE_RUNTIME - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - nil, - map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode{ - testModuleFullName: overrideOptimizeFor, - }, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - overrideOptimizeFor, - descriptor.GetOptions().GetOptimizeFor(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - nil, - map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode{ - testModuleFullName: overrideOptimizeFor, - }, - nil, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - overrideOptimizeFor, - descriptor.GetOptions().GetOptimizeFor(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - nil, - map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode{ - testModuleFullName: overrideOptimizeFor, - }, - map[string]string{ - "a.proto": "CODE_SIZE", - }, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - descriptorpb.FileOptions_CODE_SIZE, - descriptor.GetOptions().GetOptimizeFor(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - - sweeper := NewFileOptionSweeper() - optimizeForModifier, err := OptimizeFor( - zap.NewNop(), - sweeper, - descriptorpb.FileOptions_CODE_SIZE, - nil, - map[bufmodule.ModuleFullName]descriptorpb.FileOptions_OptimizeMode{ - testModuleFullName: overrideOptimizeFor, - }, - map[string]string{ - "a.proto": "CODE_SIZE", - }, - ) - require.NoError(t, err) - - modifier := NewMultiModifier(optimizeForModifier, ModifierFunc(sweeper.Sweep)) - err = modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, - descriptorpb.FileOptions_CODE_SIZE, - descriptor.GetOptions().GetOptimizeFor(), - ) - } - assertFileOptionSourceCodeInfoEmpty(t, image, goPackagePath, false) - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace.go b/private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace.go deleted file mode 100644 index 8f3b205687..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// PhpMetadataNamespaceID is the ID of the php_metadata_namespace modifier. -const PhpMetadataNamespaceID = "PHP_METADATA_NAMESPACE" - -var ( - // phpMetadataNamespacePath is the SourceCodeInfo path for the php_metadata_namespace option. - // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L448 - phpMetadataNamespacePath = []int32{8, 44} -) - -func phpMetadataNamespace( - logger *zap.Logger, - sweeper Sweeper, - overrides map[string]string, -) Modifier { - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - phpMetadataNamespaceValue := phpMetadataNamespaceValue(imageFile) - if overrideValue, ok := overrides[imageFile.Path()]; ok { - phpMetadataNamespaceValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := phpMetadataNamespaceForFile(ctx, sweeper, imageFile, phpMetadataNamespaceValue); err != nil { - return err - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", PhpMetadataNamespaceID, overrideFile) - } - } - return nil - }, - ) -} - -func phpMetadataNamespaceForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - phpMetadataNamespaceValue string, -) error { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(ctx, imageFile) || phpMetadataNamespaceValue == "" { - // This is a well-known type or we could not resolve a non-empty php_metadata_namespace - // value, so this is a no-op. - return nil - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.PhpMetadataNamespace = proto.String(phpMetadataNamespaceValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), phpMetadataNamespacePath) - } - return nil -} - -// phpMetadataNamespaceValue returns the php_metadata_namespace for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func phpMetadataNamespaceValue(imageFile bufimage.ImageFile) string { - phpNamespace := phpNamespaceValue(imageFile) - if phpNamespace == "" { - return "" - } - return phpNamespace + `\GPBMetadata` -} diff --git a/private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace_test.go b/private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace_test.go deleted file mode 100644 index 928415a13d..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/php_metadata_namespace_test.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestPhpMetadataNamespaceEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, true) - - sweeper := NewFileOptionSweeper() - phpMetadataNamespaceModifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpMetadataNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, true) - - sweeper := NewFileOptionSweeper() - phpMetadataNamespaceModifier := PhpMetadataNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - - modifier := NewMultiModifier(phpMetadataNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - require.Equal(t, "override", descriptor.GetOptions().GetPhpMetadataNamespace()) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpMetadataNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - require.Equal(t, "override", descriptor.GetOptions().GetPhpMetadataNamespace()) - }) -} - -func TestPhpMetadataNamespaceAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpMetadataNamespacePath) - - sweeper := NewFileOptionSweeper() - phpMetadataNamespaceModifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpMetadataNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpMetadataNamespacePath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpMetadataNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpMetadataNamespace()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpMetadataNamespacePath) - - sweeper := NewFileOptionSweeper() - phpMetadataNamespaceModifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpMetadataNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpMetadataNamespacePath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpMetadataNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpMetadataNamespace()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - }) -} - -func TestPhpMetadataNamespaceOptions(t *testing.T) { - t.Parallel() - testPhpMetadataNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "single"), `Acme\V1\GPBMetadata`) - testPhpMetadataNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "double"), `Acme\Weather\V1\GPBMetadata`) - testPhpMetadataNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "triple"), `Acme\Weather\Data\V1\GPBMetadata`) - testPhpMetadataNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "reserved"), `Acme\Error_\V1\GPBMetadata`) -} - -func testPhpMetadataNamespaceOptions(t *testing.T, dirPath string, classPrefix string) { - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpMetadataNamespacePath) - - sweeper := NewFileOptionSweeper() - phpMetadataNamespaceModifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpMetadataNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpMetadataNamespacePath) - - sweeper := NewFileOptionSweeper() - phpMetadataNamespaceModifier := PhpMetadataNamespace(zap.NewNop(), sweeper, map[string]string{"override.proto": "override"}) - - modifier := NewMultiModifier(phpMetadataNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpMetadataNamespace()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpMetadataNamespace(zap.NewNop(), sweeper, map[string]string{"override.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpMetadataNamespace()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpMetadataNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpMetadataNamespacePath, false) - }) -} - -func TestPhpMetadataNamespaceWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - modifiedPhpMetadataNamespace := `Acme\Weather\V1alpha1\GPBMetadata` - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - phpMetadataNamespaceModifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpMetadataNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - // php_namespace is unset for the well-known types - assert.Empty(t, descriptor.GetOptions().GetPhpMetadataNamespace()) - continue - } - assert.Equal(t, - modifiedPhpMetadataNamespace, - descriptor.GetOptions().GetPhpMetadataNamespace(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpMetadataNamespace(zap.NewNop(), sweeper, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - // php_namespace is unset for the well-known types - assert.Empty(t, descriptor.GetOptions().GetPhpMetadataNamespace()) - continue - } - assert.Equal(t, - modifiedPhpMetadataNamespace, - descriptor.GetOptions().GetPhpMetadataNamespace(), - ) - } - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/php_namespace.go b/private/bufpkg/bufimage/bufimagemodify/php_namespace.go deleted file mode 100644 index 2e5bf64e33..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/php_namespace.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "strings" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/stringutil" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// PhpNamespaceID is the ID of the php_namespace modifier. -const PhpNamespaceID = "PHP_NAMESPACE" - -var ( - // phpNamespacePath is the SourceCodeInfo path for the php_namespace option. - // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L443 - phpNamespacePath = []int32{8, 41} - - // Keywords and classes that could be produced by our heuristic. - // They must not be used in a php_namespace. - // Ref: https://www.php.net/manual/en/reserved.php - phpReservedKeywords = map[string]struct{}{ - // Reserved classes as per above. - "directory": {}, - "exception": {}, - "errorexception": {}, - "closure": {}, - "generator": {}, - "arithmeticerror": {}, - "assertionerror": {}, - "divisionbyzeroerror": {}, - "error": {}, - "throwable": {}, - "parseerror": {}, - "typeerror": {}, - // Keywords avoided by protoc. - // Ref: https://github.com/protocolbuffers/protobuf/blob/66d749188ff2a2e30e932110222d58da7c6a8d49/src/google/protobuf/compiler/php/php_generator.cc#L50-L66 - "abstract": {}, - "and": {}, - "array": {}, - "as": {}, - "break": {}, - "callable": {}, - "case": {}, - "catch": {}, - "class": {}, - "clone": {}, - "const": {}, - "continue": {}, - "declare": {}, - "default": {}, - "die": {}, - "do": {}, - "echo": {}, - "else": {}, - "elseif": {}, - "empty": {}, - "enddeclare": {}, - "endfor": {}, - "endforeach": {}, - "endif": {}, - "endswitch": {}, - "endwhile": {}, - "eval": {}, - "exit": {}, - "extends": {}, - "final": {}, - "finally": {}, - "fn": {}, - "for": {}, - "foreach": {}, - "function": {}, - "global": {}, - "goto": {}, - "if": {}, - "implements": {}, - "include": {}, - "include_once": {}, - "instanceof": {}, - "insteadof": {}, - "interface": {}, - "isset": {}, - "list": {}, - "match": {}, - "namespace": {}, - "new": {}, - "or": {}, - "print": {}, - "private": {}, - "protected": {}, - "public": {}, - "require": {}, - "require_once": {}, - "return": {}, - "static": {}, - "switch": {}, - "throw": {}, - "trait": {}, - "try": {}, - "unset": {}, - "use": {}, - "var": {}, - "while": {}, - "xor": {}, - "yield": {}, - "int": {}, - "float": {}, - "bool": {}, - "string": {}, - "true": {}, - "false": {}, - "null": {}, - "void": {}, - "iterable": {}, - } -) - -func phpNamespace( - logger *zap.Logger, - sweeper Sweeper, - overrides map[string]string, -) Modifier { - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - phpNamespaceValue := phpNamespaceValue(imageFile) - if overrideValue, ok := overrides[imageFile.Path()]; ok { - phpNamespaceValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := phpNamespaceForFile(ctx, sweeper, imageFile, phpNamespaceValue); err != nil { - return err - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", PhpNamespaceID, overrideFile) - } - } - return nil - }, - ) -} - -func phpNamespaceForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - phpNamespaceValue string, -) error { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(ctx, imageFile) || phpNamespaceValue == "" { - // This is a well-known type or we could not resolve a non-empty php_namespace - // value, so this is a no-op. - return nil - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.PhpNamespace = proto.String(phpNamespaceValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), phpNamespacePath) - } - return nil -} - -// phpNamespaceValue returns the php_namespace for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func phpNamespaceValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - packageParts := strings.Split(pkg, ".") - for i, part := range packageParts { - packagePart := stringutil.ToPascalCase(part) - if _, ok := phpReservedKeywords[strings.ToLower(part)]; ok { - // Append _ to the package part if it is a reserved keyword. - packagePart += "_" - } - packageParts[i] = packagePart - } - return strings.Join(packageParts, `\`) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/php_namespace_test.go b/private/bufpkg/bufimage/bufimagemodify/php_namespace_test.go deleted file mode 100644 index f1c6b9c7bb..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/php_namespace_test.go +++ /dev/null @@ -1,359 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestPhpNamespaceEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - phpNamespaceModifier := PhpNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpNamespace(zap.NewNop(), sweeper, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, true) - - sweeper := NewFileOptionSweeper() - phpNamespaceModifier := PhpNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - - modifier := NewMultiModifier(phpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetPhpNamespace()) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetPhpNamespace()) - }) -} - -func TestPhpNamespaceAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpNamespacePath) - - sweeper := NewFileOptionSweeper() - phpNamespaceModifier := PhpNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpNamespacePath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpNamespace(zap.NewNop(), sweeper, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpNamespacePath) - - sweeper := NewFileOptionSweeper() - phpNamespaceModifier := PhpNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - - modifier := NewMultiModifier(phpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpNamespace()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpNamespace(zap.NewNop(), sweeper, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpNamespace()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - }) -} - -func TestPhpNamespaceOptions(t *testing.T) { - t.Parallel() - testPhpNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "single"), `Acme\V1`) - testPhpNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "double"), `Acme\Weather\V1`) - testPhpNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "triple"), `Acme\Weather\Data\V1`) - testPhpNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "reserved"), `Acme\Error_\V1`) - testPhpNamespaceOptions(t, filepath.Join("testdata", "phpoptions", "underscore"), `Acme\Weather\FooBar\V1`) -} - -func testPhpNamespaceOptions(t *testing.T, dirPath string, classPrefix string) { - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpNamespacePath) - - sweeper := NewFileOptionSweeper() - phpNamespaceModifier := PhpNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpNamespace(zap.NewNop(), sweeper, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, phpNamespacePath) - - sweeper := NewFileOptionSweeper() - phpNamespaceModifier := PhpNamespace(zap.NewNop(), sweeper, map[string]string{"override.proto": "override"}) - - modifier := NewMultiModifier(phpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpNamespace()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpNamespace(zap.NewNop(), sweeper, map[string]string{"override.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetPhpNamespace()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetPhpNamespace()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, phpNamespacePath, false) - }) -} - -func TestPhpNamespaceWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - modifiedPhpNamespace := `Acme\Weather\V1alpha1` - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - phpNamespaceModifier := PhpNamespace(zap.NewNop(), sweeper, nil) - - modifier := NewMultiModifier(phpNamespaceModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - // php_namespace is unset for the well-known types - assert.Empty(t, descriptor.GetOptions().GetPhpNamespace()) - continue - } - assert.Equal(t, - modifiedPhpNamespace, - descriptor.GetOptions().GetPhpNamespace(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier := PhpNamespace(zap.NewNop(), sweeper, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - // php_namespace is unset for the well-known types - assert.Empty(t, descriptor.GetOptions().GetPhpNamespace()) - continue - } - assert.Equal(t, - modifiedPhpNamespace, - descriptor.GetOptions().GetPhpNamespace(), - ) - } - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/ruby_package.go b/private/bufpkg/bufimage/bufimagemodify/ruby_package.go deleted file mode 100644 index 3f813873fc..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/ruby_package.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "strings" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/pkg/stringutil" - "go.uber.org/zap" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/descriptorpb" -) - -// RubyPackageID is the ID of the ruby_package modifier. -const RubyPackageID = "RUBY_PACKAGE" - -// rubyPackagePath is the SourceCodeInfo path for the ruby_package option. -// https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L453 -var rubyPackagePath = []int32{8, 45} - -func rubyPackage( - logger *zap.Logger, - sweeper Sweeper, - except []bufmodule.ModuleFullName, - moduleOverrides map[bufmodule.ModuleFullName]string, - overrides map[string]string, -) Modifier { - // Convert the bufmodule.ModuleFullName types into - // strings so that they're comparable. - exceptModuleFullNameStrings := make(map[string]struct{}, len(except)) - for _, moduleFullName := range except { - exceptModuleFullNameStrings[moduleFullName.String()] = struct{}{} - } - overrideModuleFullNameStrings := make(map[string]string, len(moduleOverrides)) - for moduleFullName, rubyPackage := range moduleOverrides { - overrideModuleFullNameStrings[moduleFullName.String()] = rubyPackage - } - return ModifierFunc( - func(ctx context.Context, image bufimage.Image) error { - seenModuleFullNameStrings := make(map[string]struct{}, len(overrideModuleFullNameStrings)) - seenOverrideFiles := make(map[string]struct{}, len(overrides)) - for _, imageFile := range image.Files() { - rubyPackageValue := rubyPackageValue(imageFile) - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - moduleFullNameString := moduleFullName.String() - if moduleNamespaceOverride, ok := overrideModuleFullNameStrings[moduleFullNameString]; ok { - seenModuleFullNameStrings[moduleFullNameString] = struct{}{} - rubyPackageValue = moduleNamespaceOverride - } - } - if overrideValue, ok := overrides[imageFile.Path()]; ok { - rubyPackageValue = overrideValue - seenOverrideFiles[imageFile.Path()] = struct{}{} - } - if err := rubyPackageForFile( - ctx, - sweeper, - imageFile, - rubyPackageValue, - exceptModuleFullNameStrings, - ); err != nil { - return err - } - } - for moduleFullNameString := range overrideModuleFullNameStrings { - if _, ok := seenModuleFullNameStrings[moduleFullNameString]; !ok { - logger.Sugar().Warnf("ruby_package override for %q was unused", moduleFullNameString) - } - } - for overrideFile := range overrides { - if _, ok := seenOverrideFiles[overrideFile]; !ok { - logger.Sugar().Warnf("%s override for %q was unused", RubyPackageID, overrideFile) - } - } - return nil - }, - ) -} - -func rubyPackageForFile( - ctx context.Context, - sweeper Sweeper, - imageFile bufimage.ImageFile, - rubyPackageValue string, - exceptModuleFullNameStrings map[string]struct{}, -) error { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(ctx, imageFile) || rubyPackageValue == "" { - // This is a well-known type or we could not resolve a non-empty ruby_package - // value, so this is a no-op. - return nil - } - if moduleFullName := imageFile.ModuleFullName(); moduleFullName != nil { - if _, ok := exceptModuleFullNameStrings[moduleFullName.String()]; ok { - return nil - } - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - descriptor.Options.RubyPackage = proto.String(rubyPackageValue) - if sweeper != nil { - sweeper.mark(imageFile.Path(), rubyPackagePath) - } - return nil -} - -// rubyPackageValue returns the ruby_package for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func rubyPackageValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - packageParts := strings.Split(pkg, ".") - for i, part := range packageParts { - packageParts[i] = stringutil.ToPascalCase(part) - } - return strings.Join(packageParts, "::") -} diff --git a/private/bufpkg/bufimage/bufimagemodify/ruby_package_test.go b/private/bufpkg/bufimage/bufimagemodify/ruby_package_test.go deleted file mode 100644 index 177fa8d911..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/ruby_package_test.go +++ /dev/null @@ -1,587 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" -) - -func TestRubyPackageEmptyOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "emptyoptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, true) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, true) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "override"}) - - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - require.Equal(t, 1, len(image.Files())) - descriptor := image.Files()[0].FileDescriptorProto() - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - }) -} - -func TestRubyPackageAllOptions(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "alloptions") - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, "foo", descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "override"}) - - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, true) - }) - - t.Run("without SourceCodeInfo and with per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, map[string]string{"a.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "a.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, "foo", descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) -} - -func TestRubyPackageOptions(t *testing.T) { - t.Parallel() - testRubyPackageOptions(t, filepath.Join("testdata", "rubyoptions", "single"), `Acme::V1`) - testRubyPackageOptions(t, filepath.Join("testdata", "rubyoptions", "double"), `Acme::Weather::V1`) - testRubyPackageOptions(t, filepath.Join("testdata", "rubyoptions", "triple"), `Acme::Weather::Data::V1`) - testRubyPackageOptions(t, filepath.Join("testdata", "rubyoptions", "underscore"), `Acme::Weather::FooBar::V1`) -} - -func testRubyPackageOptions(t *testing.T, dirPath string, classPrefix string) { - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, classPrefix, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) - - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, map[string]string{"override.proto": "override"}) - - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, true) - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - modifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, map[string]string{"override.proto": "override"}) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, classPrefix, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) -} - -func TestRubyPackageWellKnownTypes(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "wktimport") - modifiedRubyPackage := `Acme::Weather::V1alpha1` - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - // php_namespace is unset for the well-known types - assert.Empty(t, descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, - modifiedRubyPackage, - descriptor.GetOptions().GetRubyPackage(), - ) - } - }) - - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - - sweeper := NewFileOptionSweeper() - modifier := RubyPackage(zap.NewNop(), sweeper, nil, nil, nil) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if isWellKnownType(context.Background(), imageFile) { - // php_namespace is unset for the well-known types - assert.Empty(t, descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, - modifiedRubyPackage, - descriptor.GetOptions().GetRubyPackage(), - ) - } - }) -} - -func TestRubyPackageExcept(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "rubyoptions", "single") - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, true), image) - // Still not empty, because the module is in except - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - }) - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - nil, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) - t.Run("with SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - // not modified even though a file has override, because the module is in except - assert.Equal(t, testGetImage(t, dirPath, true), image) - // Still not empty, because the module is in except - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - }) - t.Run("without SourceCodeInfo and per-file overrides", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - []bufmodule.ModuleFullName{testModuleFullName}, - nil, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.Equal(t, testGetImage(t, dirPath, false), image) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) -} - -func TestRubyPackageOverride(t *testing.T) { - t.Parallel() - dirPath := filepath.Join("testdata", "rubyoptions", "single") - overrideRubyPackage := "MODULE" - testModuleFullName, err := bufmodule.NewModuleFullName( - testRemote, - testRepositoryOwner, - testRepositoryName, - ) - require.NoError(t, err) - t.Run("with SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideRubyPackage}, - nil, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, overrideRubyPackage, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, true) - }) - t.Run("without SourceCodeInfo", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideRubyPackage}, - nil, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - assert.Equal(t, overrideRubyPackage, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) - t.Run("with SourceCodeInfo with per-file override", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, true) - assertFileOptionSourceCodeInfoNotEmpty(t, image, rubyPackagePath) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideRubyPackage}, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, true), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, overrideRubyPackage, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, true) - }) - t.Run("without SourceCodeInfo and per-file override", func(t *testing.T) { - t.Parallel() - image := testGetImage(t, dirPath, false) - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - - sweeper := NewFileOptionSweeper() - rubyPackageModifier := RubyPackage( - zap.NewNop(), - sweeper, - nil, - map[bufmodule.ModuleFullName]string{testModuleFullName: overrideRubyPackage}, - map[string]string{"override.proto": "override"}, - ) - modifier := NewMultiModifier(rubyPackageModifier, ModifierFunc(sweeper.Sweep)) - err := modifier.Modify( - context.Background(), - image, - ) - require.NoError(t, err) - assert.NotEqual(t, testGetImage(t, dirPath, false), image) - - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - if imageFile.Path() == "override.proto" { - assert.Equal(t, "override", descriptor.GetOptions().GetRubyPackage()) - continue - } - assert.Equal(t, overrideRubyPackage, descriptor.GetOptions().GetRubyPackage()) - } - assertFileOptionSourceCodeInfoEmpty(t, image, rubyPackagePath, false) - }) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/vars.go b/private/bufpkg/bufimage/bufimagemodify/vars.go new file mode 100644 index 0000000000..23d70b907e --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/vars.go @@ -0,0 +1,284 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "path" + "strings" + "unicode" + + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/normalpath" + "github.com/bufbuild/buf/private/pkg/protoversion" + "github.com/bufbuild/buf/private/pkg/stringutil" +) + +var ( + // fileOptionPath is the path prefix used for FileOptions. + // All file option locations are preceded by a location + // with a path set to the fileOptionPath. + // https://github.com/protocolbuffers/protobuf/blob/053966b4959bdd21e4a24e657bcb97cb9de9e8a4/src/google/protobuf/descriptor.proto#L80 + fileOptionPath = []int32{8} + // ccEnableArenas is the SourceCodeInfo path for the cc_enable_arenas option. + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L420 + ccEnableArenasPath = []int32{8, 31} + // csharpNamespacePath is the SourceCodeInfo path for the csharp_namespace option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L428 + csharpNamespacePath = []int32{8, 37} + // goPackagePath is the SourceCodeInfo path for the go_package option. + // https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L392 + goPackagePath = []int32{8, 11} + // javaMultipleFilesPath is the SourceCodeInfo path for the java_multiple_files option. + // https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L364 + javaMultipleFilesPath = []int32{8, 10} + // javaOuterClassnamePath is the SourceCodeInfo path for the java_outer_classname option. + // https://github.com/protocolbuffers/protobuf/blob/87d140f851131fb8a6e8a80449cf08e73e568259/src/google/protobuf/descriptor.proto#L356 + javaOuterClassnamePath = []int32{8, 8} + // javaPackagePath is the SourceCodeInfo path for the java_package option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L348 + javaPackagePath = []int32{8, 1} + // javaStringCheckUtf8Path is the SourceCodeInfo path for the java_string_check_utf8 option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L375 + javaStringCheckUtf8Path = []int32{8, 27} + // objcClassPrefixPath is the SourceCodeInfo path for the objc_class_prefix option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L425 + objcClassPrefixPath = []int32{8, 36} + // optimizeFor is the SourceCodeInfo path for the optimize_for option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L385 + optimizeForPath = []int32{8, 9} + // phpMetadataNamespacePath is the SourceCodeInfo path for the php_metadata_namespace option. + // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L448 + phpMetadataNamespacePath = []int32{8, 44} + // phpNamespacePath is the SourceCodeInfo path for the php_namespace option. + // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L443 + phpNamespacePath = []int32{8, 41} + // Keywords and classes that could be produced by our heuristic. + // They must not be used in a php_namespace. + // Ref: https://www.php.net/manual/en/reserved.php + phpReservedKeywords = map[string]struct{}{ + // Reserved classes as per above. + "directory": {}, + "exception": {}, + "errorexception": {}, + "closure": {}, + "generator": {}, + "arithmeticerror": {}, + "assertionerror": {}, + "divisionbyzeroerror": {}, + "error": {}, + "throwable": {}, + "parseerror": {}, + "typeerror": {}, + // Keywords avoided by protoc. + // Ref: https://github.com/protocolbuffers/protobuf/blob/66d749188ff2a2e30e932110222d58da7c6a8d49/src/google/protobuf/compiler/php/php_generator.cc#L50-L66 + "abstract": {}, + "and": {}, + "array": {}, + "as": {}, + "break": {}, + "callable": {}, + "case": {}, + "catch": {}, + "class": {}, + "clone": {}, + "const": {}, + "continue": {}, + "declare": {}, + "default": {}, + "die": {}, + "do": {}, + "echo": {}, + "else": {}, + "elseif": {}, + "empty": {}, + "enddeclare": {}, + "endfor": {}, + "endforeach": {}, + "endif": {}, + "endswitch": {}, + "endwhile": {}, + "eval": {}, + "exit": {}, + "extends": {}, + "final": {}, + "finally": {}, + "fn": {}, + "for": {}, + "foreach": {}, + "function": {}, + "global": {}, + "goto": {}, + "if": {}, + "implements": {}, + "include": {}, + "include_once": {}, + "instanceof": {}, + "insteadof": {}, + "interface": {}, + "isset": {}, + "list": {}, + "match": {}, + "namespace": {}, + "new": {}, + "or": {}, + "print": {}, + "private": {}, + "protected": {}, + "public": {}, + "require": {}, + "require_once": {}, + "return": {}, + "static": {}, + "switch": {}, + "throw": {}, + "trait": {}, + "try": {}, + "unset": {}, + "use": {}, + "var": {}, + "while": {}, + "xor": {}, + "yield": {}, + "int": {}, + "float": {}, + "bool": {}, + "string": {}, + "true": {}, + "false": {}, + "null": {}, + "void": {}, + "iterable": {}, + } + // rubyPackagePath is the SourceCodeInfo path for the ruby_package option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L453 + rubyPackagePath = []int32{8, 45} +) + +// TODO: is this needed? +// csharpNamespaceValue returns the csharp_namespace for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func csharpNamespaceValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + packageParts := strings.Split(pkg, ".") + for i, part := range packageParts { + packageParts[i] = stringutil.ToPascalCase(part) + } + return strings.Join(packageParts, ".") +} + +// TODO: why was this exported +// GoPackageImportPathForFile returns the go_package import path for the given +// ImageFile. If the package contains a version suffix, and if there are more +// than two components, concatenate the final two components. Otherwise, we +// exclude the ';' separator and adopt the default behavior from the import path. +// +// For example, an ImageFile with `package acme.weather.v1;` will include `;weatherv1` +// in the `go_package` declaration so that the generated package is named as such. +func GoPackageImportPathForFile(imageFile bufimage.ImageFile, importPathPrefix string) string { + goPackageImportPath := path.Join(importPathPrefix, path.Dir(imageFile.Path())) + packageName := imageFile.FileDescriptorProto().GetPackage() + if _, ok := protoversion.NewPackageVersionForPackage(packageName); ok { + parts := strings.Split(packageName, ".") + if len(parts) >= 2 { + goPackageImportPath += ";" + parts[len(parts)-2] + parts[len(parts)-1] + } + } + return goPackageImportPath +} + +func javaOuterClassnameValue(imageFile bufimage.ImageFile) string { + return stringutil.ToPascalCase(normalpath.Base(imageFile.Path())) +} + +// objcClassPrefixValue returns the objc_class_prefix for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func objcClassPrefixValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + _, hasPackageVersion := protoversion.NewPackageVersionForPackage(pkg) + packageParts := strings.Split(pkg, ".") + var prefixParts []rune + for i, part := range packageParts { + // Check if last part is a version before appending. + if i == len(packageParts)-1 && hasPackageVersion { + continue + } + // Probably should never be a non-ASCII character, + // but why not support it just in case? + runeSlice := []rune(part) + prefixParts = append(prefixParts, unicode.ToUpper(runeSlice[0])) + } + for len(prefixParts) < 3 { + prefixParts = append(prefixParts, 'X') + } + prefix := string(prefixParts) + if prefix == "GPB" { + prefix = "GPX" + } + return prefix +} + +// phpMetadataNamespaceValue returns the php_metadata_namespace for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func phpMetadataNamespaceValue(imageFile bufimage.ImageFile) string { + phpNamespace := phpNamespaceValue(imageFile) + if phpNamespace == "" { + return "" + } + return phpNamespace + `\GPBMetadata` +} + +// phpNamespaceValue returns the php_namespace for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func phpNamespaceValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + packageParts := strings.Split(pkg, ".") + for i, part := range packageParts { + packagePart := stringutil.ToPascalCase(part) + if _, ok := phpReservedKeywords[strings.ToLower(part)]; ok { + // Append _ to the package part if it is a reserved keyword. + packagePart += "_" + } + packageParts[i] = packagePart + } + return strings.Join(packageParts, `\`) +} + +// rubyPackageValue returns the ruby_package for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func rubyPackageValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + packageParts := strings.Split(pkg, ".") + for i, part := range packageParts { + packageParts[i] = stringutil.ToPascalCase(part) + } + return strings.Join(packageParts, "::") +} From dd7597fd91a82f7749dad773d7bfc2f14f9ca59a Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Sat, 2 Dec 2023 00:54:10 -0500 Subject: [PATCH 61/66] bufimagemodify partly done --- .../bufimage/bufimagemodify/bufimagemodify.go | 74 +- .../bufimagemodify/bufimagemodify_test.go | 808 ++++++++++++++++-- .../bufimage/bufimagemodify/field_option.go | 9 +- .../{new_modify.go => file_option.go} | 224 +++-- .../{ => internal}/field_options_trie.go | 2 +- .../bufimagemodify/internal/internal.go | 33 + .../{ => internal}/location_path_dfa.go | 2 +- .../{ => internal}/marksweeper.go | 18 +- .../bufimage/bufimagemodify/new_modify2.go | 300 ------- .../bufimagemodify/new_modify_test.go | 798 ----------------- .../bufimage/bufimagemodify/override.go | 421 +++++++++ .../bufpkg/bufimage/bufimagemodify/vars.go | 284 ------ 12 files changed, 1415 insertions(+), 1558 deletions(-) rename private/bufpkg/bufimage/bufimagemodify/{new_modify.go => file_option.go} (54%) rename private/bufpkg/bufimage/bufimagemodify/{ => internal}/field_options_trie.go (99%) create mode 100644 private/bufpkg/bufimage/bufimagemodify/internal/internal.go rename private/bufpkg/bufimage/bufimagemodify/{ => internal}/location_path_dfa.go (99%) rename private/bufpkg/bufimage/bufimagemodify/{ => internal}/marksweeper.go (94%) delete mode 100644 private/bufpkg/bufimage/bufimagemodify/new_modify2.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/new_modify_test.go create mode 100644 private/bufpkg/bufimage/bufimagemodify/override.go delete mode 100644 private/bufpkg/bufimage/bufimagemodify/vars.go diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go index f0160f12d4..f0d973a8dd 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify.go @@ -15,54 +15,50 @@ package bufimagemodify import ( - "fmt" - "strconv" + "context" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify/internal" "github.com/bufbuild/buf/private/gen/data/datawkt" - "google.golang.org/protobuf/types/descriptorpb" ) -// TODO: remove code dealing with the old config (maps) +// TODO: move this package into bufgen/internal -// TODO: rename this one -func isWellKnownType(imageFile bufimage.ImageFile) bool { - return datawkt.Exists(imageFile.Path()) -} - -// int32SliceIsEqual returns true if x and y contain the same elements. -func int32SliceIsEqual(x []int32, y []int32) bool { - if len(x) != len(y) { - return false +// Modify modifies the image according to the managed config. +func Modify( + ctx context.Context, + image bufimage.Image, + config bufconfig.GenerateManagedConfig, +) error { + if !config.Enabled() { + return nil } - for i, elem := range x { - if elem != y[i] { - return false + sweeper := internal.NewMarkSweeper(image) + for _, imageFile := range image.Files() { + if datawkt.Exists(imageFile.Path()) { + continue } - } - return true -} - -func stringOverridesToBoolOverrides(stringOverrides map[string]string) (map[string]bool, error) { - validatedOverrides := make(map[string]bool, len(stringOverrides)) - for fileImportPath, overrideString := range stringOverrides { - overrideBool, err := strconv.ParseBool(overrideString) - if err != nil { - return nil, fmt.Errorf("non-boolean override %s set for file %s", overrideString, fileImportPath) + modifyFuncs := []func(internal.MarkSweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error{ + modifyCcEnableArenas, + modifyCsharpNamespace, + modifyGoPackage, + modifyJavaMultipleFiles, + modifyJavaOuterClass, + modifyJavaPackage, + modifyJavaStringCheckUtf8, + modifyObjcClassPrefix, + modifyOptmizeFor, + modifyPhpMetadataNamespace, + modifyPhpNamespace, + modifyRubyPackage, + modifyJsType, } - validatedOverrides[fileImportPath] = overrideBool - } - return validatedOverrides, nil -} - -func stringOverridesToOptimizeModeOverrides(stringOverrides map[string]string) (map[string]descriptorpb.FileOptions_OptimizeMode, error) { - validatedOverrides := make(map[string]descriptorpb.FileOptions_OptimizeMode, len(stringOverrides)) - for fileImportPath, stringOverride := range stringOverrides { - optimizeMode, ok := descriptorpb.FileOptions_OptimizeMode_value[stringOverride] - if !ok { - return nil, fmt.Errorf("invalid optimize mode %s set for file %s", stringOverride, fileImportPath) + for _, modifyFunc := range modifyFuncs { + if err := modifyFunc(sweeper, imageFile, config); err != nil { + return err + } } - validatedOverrides[fileImportPath] = descriptorpb.FileOptions_OptimizeMode(optimizeMode) } - return validatedOverrides, nil + return nil } diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go index 2d8a10f7a5..af64e5bf4b 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go @@ -16,77 +16,781 @@ package bufimagemodify import ( "context" + "path/filepath" "testing" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufconfig/bufconfigtest" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify/internal" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagetesting" "github.com/bufbuild/buf/private/bufpkg/bufmodule" "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletest" - "github.com/stretchr/testify/assert" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/descriptorpb" ) -const ( - testImportPathPrefix = "github.com/foo/bar/private/gen/proto/go" - testRemote = "buf.test" - testRepositoryOwner = "foo" - testRepositoryName = "bar" -) - -func assertFileOptionSourceCodeInfoEmpty(t *testing.T, image bufimage.Image, fileOptionPath []int32, includeSourceInfo bool) { - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - - if !includeSourceInfo { - assert.Empty(t, descriptor.SourceCodeInfo) - continue - } - - var hasFileOption bool - for _, location := range descriptor.SourceCodeInfo.Location { - if len(location.Path) > 0 && int32SliceIsEqual(location.Path, fileOptionPath) { - hasFileOption = true - break - } +func TestModifyImage(t *testing.T) { + t.Parallel() + testcases := []struct { + description string + dirPathToModuleFullName map[string]string + config bufconfig.GenerateManagedConfig + filePathToExpectedOptions map[string]*descriptorpb.FileOptions + }{ + { + description: "nil_config", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig(false, nil, nil), + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": nil, + "bar_all/with_package.proto": { + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaPackage: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + }, + }, + { + description: "empty_config", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{}, + ), + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": { + // CcEnableArena's default value is true + CsharpNamespace: proto.String("Foo.Empty"), + // GoPackage is not modified by default + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithPackageProto"), + JavaPackage: proto.String("com.foo.empty"), + // JavaStringCheckUtf8 is not modified by default + ObjcClassPrefix: proto.String("FEX"), + // OptimizeFor tries to modifiy this value to SPEED, which is already the default + // Empty is a keyword in php + PhpMetadataNamespace: proto.String(`Foo\Empty_\GPBMetadata`), + PhpNamespace: proto.String(`Foo\Empty_`), + RubyPackage: proto.String("Foo::Empty"), + }, + "foo_empty/without_package.proto": { + // CcEnableArena's default value is true + // GoPackage is not modified by default + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithoutPackageProto"), + // JavaStringCheckUtf8 is not modified by default + // OptimizeFor tries to modifiy this value to SPEED, which is already the default + }, + "bar_all/with_package.proto": { + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("Bar.All"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithPackageProto"), + JavaPackage: proto.String("com.bar.all"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("BAX"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String(`Bar\All\GPBMetadata`), + PhpNamespace: proto.String(`Bar\All`), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("Bar::All"), + SwiftPrefix: proto.String("bar"), + }, + "bar_all/without_package.proto": { + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("WithoutPackageProto"), + JavaPackage: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String(`bar`), + PhpNamespace: proto.String(`bar`), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + }, + }, + } + for _, testcase := range testcases { + testcase := testcase + for _, includeSourceInfo := range []bool{true, false} { + includeSourceInfo := includeSourceInfo + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + image := testGetImageFromDirs(t, testcase.dirPathToModuleFullName, includeSourceInfo) + err := Modify( + context.Background(), + image, + testcase.config, + ) + require.NoError(t, err) + for filePath, expectedOptions := range testcase.filePathToExpectedOptions { + imageFile := image.GetFile(filePath) + require.NotNil(t, imageFile) + require.Empty( + t, + cmp.Diff(expectedOptions, imageFile.FileDescriptorProto().GetOptions(), protocmp.Transform()), + imageFile.FileDescriptorProto().GetOptions(), + ) + } + }) } - assert.False(t, hasFileOption) } } -func assertFileOptionSourceCodeInfoNotEmpty(t *testing.T, image bufimage.Image, fileOptionPath []int32) { - for _, imageFile := range image.Files() { - descriptor := imageFile.FileDescriptorProto() - - var hasFileOption bool - for _, location := range descriptor.SourceCodeInfo.Location { - if len(location.Path) > 0 && int32SliceIsEqual(location.Path, fileOptionPath) { - hasFileOption = true - break - } +func TestModifyImageFile( + t *testing.T, +) { + t.Parallel() + testcases := []struct { + description string + dirPathToModuleFullName map[string]string + config bufconfig.GenerateManagedConfig + modifyFunc func(internal.MarkSweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error + filePathToExpectedOptions map[string]*descriptorpb.FileOptions + filePathToExpectedMarkedLocationPaths map[string][][]int32 + }{ + { + description: "cc_enable_arena", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionCcEnableArenas, false), + }, + ), + modifyFunc: modifyCcEnableArenas, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/without_package.proto": nil, + "bar_empty/without_package.proto": { + CcEnableArenas: proto.Bool(false), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/without_package.proto": {ccEnableArenasPath}, + }, + }, + { + description: "csharp_namespace", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionCsharpNamespacePrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespacePrefix, "BarPrefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespace, "BarValue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespace, "FooValue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespacePrefix, "FooPrefix"), + }, + ), + modifyFunc: modifyCsharpNamespace, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "bar_empty/with_package.proto": { + CsharpNamespace: proto.String("BarPrefix.Bar.Empty"), + }, + "bar_empty/without_package.proto": { + CsharpNamespace: proto.String("BarValue"), + }, + "foo_empty/with_package.proto": { + CsharpNamespace: proto.String("FooValue"), + }, + "foo_empty/without_package.proto": nil, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/with_package.proto": {csharpNamespacePath}, + "bar_empty/without_package.proto": {csharpNamespacePath}, + "foo_empty/with_package.proto": {csharpNamespacePath}, + }, + }, + { + description: "go_package", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionGoPackagePrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionGoPackagePrefix, "barprefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionGoPackage, "barvalue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty/with_package.proto", "buf.build/acme/foo", bufconfig.FileOptionGoPackage, "foovalue"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionGoPackagePrefix, "fooprefix"), + }, + ), + modifyFunc: modifyGoPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "bar_empty/with_package.proto": { + GoPackage: proto.String("barprefix/bar_empty"), + }, + "bar_empty/without_package.proto": { + GoPackage: proto.String("barvalue"), + }, + "foo_empty/with_package.proto": { + GoPackage: proto.String("foovalue"), + }, + "foo_empty/without_package.proto": { + GoPackage: proto.String("fooprefix/foo_empty"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/with_package.proto": {goPackagePath}, + "bar_empty/without_package.proto": {goPackagePath}, + "foo_empty/with_package.proto": {goPackagePath}, + "foo_empty/without_package.proto": {goPackagePath}, + }, + }, + { + description: "java_package_prefix", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackagePrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackagePrefix, "barprefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackageSuffix, "foosuffix"), + }, + ), + modifyFunc: modifyJavaPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": { + // default prefix and override suffix + JavaPackage: proto.String("com.foo.empty.foosuffix"), + }, + // prefix is disabled + "bar_empty/with_package.proto": nil, + // prefix is overridden + "bar_all/with_package.proto": { + JavaPackage: proto.String("barprefix.bar.all"), + // below this point are the values from the file + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + // not modified because it doesn't have a package + "foo_empty/without_package.proto": nil, + "bar_empty/without_package.proto": nil, + "foo_all/without_package.proto": { + // values are from the file + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(true), + CsharpNamespace: proto.String("foo"), + GoPackage: proto.String("foo"), + JavaGenericServices: proto.Bool(true), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("foo"), + JavaPackage: proto.String("foo"), + JavaStringCheckUtf8: proto.Bool(true), + ObjcClassPrefix: proto.String("foo"), + OptimizeFor: descriptorpb.FileOptions_CODE_SIZE.Enum(), + PhpClassPrefix: proto.String("foo"), + PhpGenericServices: proto.Bool(true), + PhpMetadataNamespace: proto.String("foo"), + PhpNamespace: proto.String("foo"), + PyGenericServices: proto.Bool(true), + RubyPackage: proto.String("foo"), + SwiftPrefix: proto.String("foo"), + }, + "bar_all/without_package.proto": { + // values are from the file + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaPackage: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "foo_empty/with_package.proto": {javaPackagePath}, + "bar_all/with_package.proto": {javaPackagePath}, + }, + }, + { + description: "java_package_suffix", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackageSuffix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + modifyFunc: modifyJavaPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "foo_empty/with_package.proto": { + // only suffix matches, but apply both prefix and suffix + JavaPackage: proto.String("com.foo.empty.suffix"), + }, + "bar_empty/with_package.proto": { + // only prefix because suffix is disabled + JavaPackage: proto.String("com.bar.empty"), + }, + "bar_all/with_package.proto": { + JavaPackage: proto.String("com.bar.all.suffix"), + // below this point are the values from the file + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + // not modified + "foo_empty/without_package.proto": nil, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "foo_empty/with_package.proto": {javaPackagePath}, + "bar_empty/with_package.proto": {javaPackagePath}, + "bar_all/with_package.proto": {javaPackagePath}, + }, + }, + { + description: "java_package", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackage, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackage, "bar.value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackage, "foo.value"), + }, + ), + modifyFunc: modifyJavaPackage, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + // bar_empty disabled + "bar_empty/with_package.proto": nil, + "bar_empty/without_package.proto": nil, + "bar_all/with_package.proto": { + JavaPackage: proto.String("bar.value"), + CcEnableArenas: proto.Bool(false), + CcGenericServices: proto.Bool(false), + CsharpNamespace: proto.String("bar"), + GoPackage: proto.String("bar"), + JavaGenericServices: proto.Bool(false), + JavaMultipleFiles: proto.Bool(false), + JavaOuterClassname: proto.String("bar"), + JavaStringCheckUtf8: proto.Bool(false), + ObjcClassPrefix: proto.String("bar"), + OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), + PhpClassPrefix: proto.String("bar"), + PhpGenericServices: proto.Bool(false), + PhpMetadataNamespace: proto.String("bar"), + PhpNamespace: proto.String("bar"), + PyGenericServices: proto.Bool(false), + RubyPackage: proto.String("bar"), + SwiftPrefix: proto.String("bar"), + }, + "foo_empty/with_package.proto": { + JavaPackage: proto.String("foo.value"), + }, + "foo_empty/without_package.proto": { + JavaPackage: proto.String("foo.value"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "foo_empty/with_package.proto": {javaPackagePath}, + "foo_empty/without_package.proto": {javaPackagePath}, + "bar_all/with_package.proto": {javaPackagePath}, + }, + }, + { + description: "objc_class_prefix", + dirPathToModuleFullName: map[string]string{ + filepath.Join("testdata", "foo"): "buf.build/acme/foo", + filepath.Join("testdata", "bar"): "buf.build/acme/bar", + }, + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{ + bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionObjcClassPrefix, bufconfig.FieldOptionUnspecified), + }, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionObjcClassPrefix, "BAR"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOO"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_all", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOOALL"), + }, + ), + modifyFunc: modifyObjcClassPrefix, + filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ + "bar_empty/with_package.proto": { + ObjcClassPrefix: proto.String("BAR"), + }, + "bar_empty/without_package.proto": { + ObjcClassPrefix: proto.String("BAR"), + }, + // disabled + "foo_empty/with_package.proto": nil, + // no package + "foo_empty/without_package.proto": { + ObjcClassPrefix: proto.String("FOO"), + }, + "foo_all/with_package.proto": { + ObjcClassPrefix: proto.String("FOOALL"), + CcEnableArenas: proto.Bool(true), + CcGenericServices: proto.Bool(true), + CsharpNamespace: proto.String("foo"), + GoPackage: proto.String("foo"), + JavaGenericServices: proto.Bool(true), + JavaMultipleFiles: proto.Bool(true), + JavaOuterClassname: proto.String("foo"), + JavaPackage: proto.String("foo"), + JavaStringCheckUtf8: proto.Bool(true), + OptimizeFor: descriptorpb.FileOptions_CODE_SIZE.Enum(), + PhpClassPrefix: proto.String("foo"), + PhpGenericServices: proto.Bool(true), + PhpMetadataNamespace: proto.String("foo"), + PhpNamespace: proto.String("foo"), + PyGenericServices: proto.Bool(true), + RubyPackage: proto.String("foo"), + SwiftPrefix: proto.String("foo"), + }, + }, + filePathToExpectedMarkedLocationPaths: map[string][][]int32{ + "bar_empty/with_package.proto": {objcClassPrefixPath}, + "bar_empty/without_package.proto": {objcClassPrefixPath}, + "foo_empty/without_package.proto": {objcClassPrefixPath}, + "foo_all/without_package.proto": {objcClassPrefixPath}, + "foo_all/with_package.proto": {objcClassPrefixPath}, + }, + }, + } + for _, testcase := range testcases { + testcase := testcase + for _, includeSourceInfo := range []bool{true, false} { + // TODO: we are only testing sweep here, no need to test both include and exclude source info + includeSourceInfo := includeSourceInfo + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + image := testGetImageFromDirs(t, testcase.dirPathToModuleFullName, includeSourceInfo) + sweeper := internal.NewMarkSweeper(image) + // TODO: check include source code info + for filePath, expectedOptions := range testcase.filePathToExpectedOptions { + imageFile := image.GetFile(filePath) + testcase.modifyFunc( + sweeper, + imageFile, + testcase.config, + ) + require.NotNil(t, imageFile) + require.Empty( + t, + cmp.Diff(expectedOptions, imageFile.FileDescriptorProto().GetOptions(), protocmp.Transform()), + "incorrect options result for %s", + filePath, + ) + // TODO: sweep and check paths gone + } + }) } - assert.True(t, hasFileOption) } } -func testGetImage(t *testing.T, dirPath string, includeSourceInfo bool) bufimage.Image { - moduleSet, err := bufmoduletest.NewModuleSet( - bufmoduletest.ModuleData{ - Name: testRemote + "/" + testRepositoryOwner + "/" + testRepositoryName, - DirPath: dirPath, +// TODO: add default values +func TestGetStringOverrideFromConfig(t *testing.T) { + t.Parallel() + testcases := []struct { + description string + config bufconfig.GenerateManagedConfig + imageFile bufimage.ImageFile + defaultOverrideOptions stringOverrideOptions + expectedOverride stringOverrideOptions + expectedDisable bool + }{ + { + description: "only_value", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{value: "value"}, + }, + { + description: "only_prefix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{prefix: "prefix"}, + }, + { + description: "only_suffix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{suffix: "suffix"}, + }, + { + description: "prefix_then_value", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{value: "value"}, + }, + { + description: "value_then_prefix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{prefix: "prefix"}, + }, + { + description: "prefix_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{ + prefix: "prefix", + suffix: "suffix", + }, + }, + { + description: "value_prefix_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{ + prefix: "prefix", + suffix: "suffix", + }, + }, + { + description: "prefix_value_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{suffix: "suffix"}, + }, + { + description: "prefix_then_prefix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix2"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{prefix: "prefix2"}, + }, + { + description: "suffix_then_suffix", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix2"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{suffix: "suffix2"}, + }, + { + description: "value_then_value", + config: bufconfig.NewGenerateManagedConfig( + true, + []bufconfig.ManagedDisableRule{}, + []bufconfig.ManagedOverrideRule{ + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value2"), + }, + ), + imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), + expectedOverride: stringOverrideOptions{value: "value2"}, }, - ) - require.NoError(t, err) - var options []bufimage.BuildImageOption - if !includeSourceInfo { - options = []bufimage.BuildImageOption{bufimage.WithExcludeSourceCodeInfo()} } - image, annotations, err := bufimage.BuildImage( - context.Background(), - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(moduleSet), - options..., - ) + for _, testcase := range testcases { + testcase := testcase + t.Run(testcase.description, func(t *testing.T) { + t.Parallel() + override, err := stringOverrideFromConfig( + testcase.imageFile, + testcase.config, + testcase.defaultOverrideOptions, + bufconfig.FileOptionJavaPackage, + bufconfig.FileOptionJavaPackagePrefix, + bufconfig.FileOptionJavaPackageSuffix, + ) + require.NoError(t, err) + require.Equal(t, testcase.expectedOverride, override) + }) + } +} + +func TestModifyFieldOption(t *testing.T) { + t.Parallel() + // TODO in v2 +} + +func testGetImageFile( + t *testing.T, + path string, + moduleFullName string, +) bufimage.ImageFile { + parsedModuleFullName, err := bufmodule.ParseModuleFullName(moduleFullName) require.NoError(t, err) - require.Empty(t, annotations) - return image + return bufimagetesting.NewImageFile( + t, + &descriptorpb.FileDescriptorProto{ + Name: proto.String(path), + Syntax: proto.String("proto3"), + }, + parsedModuleFullName, + "", + path, + false, + false, + nil, + ) } func testGetImageFromDirs( diff --git a/private/bufpkg/bufimage/bufimagemodify/field_option.go b/private/bufpkg/bufimage/bufimagemodify/field_option.go index bc9547d821..c275662dbd 100644 --- a/private/bufpkg/bufimage/bufimagemodify/field_option.go +++ b/private/bufpkg/bufimage/bufimagemodify/field_option.go @@ -19,6 +19,8 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify/internal" + "github.com/bufbuild/buf/private/gen/data/datawkt" "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/protocompile/walk" "google.golang.org/protobuf/proto" @@ -26,12 +28,13 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) -// JSTypeSubPath is the SourceCodeInfo sub path for the jstype field option. +// jsTypeSubPath is the SourceCodeInfo sub path for the jstype field option. // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L567 +// TODO: where does the 8 come from????? var jsTypeSubPath = []int32{8, 6} func modifyJsType( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -53,7 +56,7 @@ func modifyJsType( return nil } } - if isWellKnownType(imageFile) { + if datawkt.Exists(imageFile.Path()) { return nil } return walk.DescriptorProtosWithPath( diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify.go b/private/bufpkg/bufimage/bufimagemodify/file_option.go similarity index 54% rename from private/bufpkg/bufimage/bufimagemodify/new_modify.go rename to private/bufpkg/bufimage/bufimagemodify/file_option.go index e6ec60bf1a..34276551a2 100644 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify.go +++ b/private/bufpkg/bufimage/bufimagemodify/file_option.go @@ -15,55 +15,55 @@ package bufimagemodify import ( - "context" - "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify/internal" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/descriptorpb" ) -// TODO: this is a temporary file, although it might stay. Need to rename the file at least. (The rest of the package can be deleted, except for ) -// TODO: move this package into bufgen/internal -func Modify( - ctx context.Context, - image bufimage.Image, - config bufconfig.GenerateManagedConfig, -) error { - if !config.Enabled() { - return nil - } - sweeper := newMarkSweeper(image) - for _, imageFile := range image.Files() { - if isWellKnownType(imageFile) { - continue - } - modifyFuncs := []func(*markSweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error{ - modifyCcEnableArenas, - modifyCsharpNamespace, - modifyGoPackage, - modifyJavaMultipleFiles, - modifyJavaOuterClass, - modifyJavaPackage, - modifyJavaStringCheckUtf8, - modifyObjcClassPrefix, - modifyOptmizeFor, - modifyPhpMetadataNamespace, - modifyPhpNamespace, - modifyRubyPackage, - modifyJsType, - } - for _, modifyFunc := range modifyFuncs { - if err := modifyFunc(sweeper, imageFile, config); err != nil { - return err - } - } - } - return nil -} +var ( + // ccEnableArenas is the SourceCodeInfo path for the cc_enable_arenas option. + // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L420 + ccEnableArenasPath = []int32{8, 31} + // csharpNamespacePath is the SourceCodeInfo path for the csharp_namespace option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L428 + csharpNamespacePath = []int32{8, 37} + // goPackagePath is the SourceCodeInfo path for the go_package option. + // https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L392 + goPackagePath = []int32{8, 11} + // javaMultipleFilesPath is the SourceCodeInfo path for the java_multiple_files option. + // https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L364 + javaMultipleFilesPath = []int32{8, 10} + // javaOuterClassnamePath is the SourceCodeInfo path for the java_outer_classname option. + // https://github.com/protocolbuffers/protobuf/blob/87d140f851131fb8a6e8a80449cf08e73e568259/src/google/protobuf/descriptor.proto#L356 + javaOuterClassnamePath = []int32{8, 8} + // javaPackagePath is the SourceCodeInfo path for the java_package option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L348 + javaPackagePath = []int32{8, 1} + // javaStringCheckUtf8Path is the SourceCodeInfo path for the java_string_check_utf8 option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L375 + javaStringCheckUtf8Path = []int32{8, 27} + // objcClassPrefixPath is the SourceCodeInfo path for the objc_class_prefix option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L425 + objcClassPrefixPath = []int32{8, 36} + // optimizeFor is the SourceCodeInfo path for the optimize_for option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L385 + optimizeForPath = []int32{8, 9} + // phpMetadataNamespacePath is the SourceCodeInfo path for the php_metadata_namespace option. + // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L448 + phpMetadataNamespacePath = []int32{8, 44} + // phpNamespacePath is the SourceCodeInfo path for the php_namespace option. + // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L443 + phpNamespacePath = []int32{8, 41} + + // rubyPackagePath is the SourceCodeInfo path for the ruby_package option. + // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L453 + rubyPackagePath = []int32{8, 45} +) func modifyJavaOuterClass( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -91,7 +91,7 @@ func modifyJavaOuterClass( } func modifyJavaPackage( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -117,7 +117,7 @@ func modifyJavaPackage( } func modifyGoPackage( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -131,11 +131,11 @@ func modifyGoPackage( func(bufimage.ImageFile) stringOverrideOptions { return stringOverrideOptions{} }, - func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { - if stringOverride.prefix == "" { + func(imageFile bufimage.ImageFile, stringOverrideOptions stringOverrideOptions) string { + if stringOverrideOptions.prefix == "" { return "" } - return GoPackageImportPathForFile(imageFile, stringOverride.prefix) + return goPackageImportPathForFile(imageFile, stringOverrideOptions.prefix) }, func(options *descriptorpb.FileOptions) string { return options.GetGoPackage() @@ -148,7 +148,7 @@ func modifyGoPackage( } func modifyObjcClassPrefix( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -176,7 +176,7 @@ func modifyObjcClassPrefix( } func modifyCsharpNamespace( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -190,8 +190,8 @@ func modifyCsharpNamespace( func(bufimage.ImageFile) stringOverrideOptions { return stringOverrideOptions{value: csharpNamespaceValue(imageFile)} }, - func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { - return getCsharpNamespaceValue(imageFile, stringOverride.prefix) + func(imageFile bufimage.ImageFile, stringOverrideOptions stringOverrideOptions) string { + return getCsharpNamespaceValue(imageFile, stringOverrideOptions.prefix) }, func(options *descriptorpb.FileOptions) string { return options.GetCsharpNamespace() @@ -204,7 +204,7 @@ func modifyCsharpNamespace( } func modifyPhpNamespace( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -232,7 +232,7 @@ func modifyPhpNamespace( } func modifyPhpMetadataNamespace( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -246,8 +246,8 @@ func modifyPhpMetadataNamespace( func(bufimage.ImageFile) stringOverrideOptions { return stringOverrideOptions{value: phpMetadataNamespaceValue(imageFile)} }, - func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { - return getPhpMetadataNamespaceValue(imageFile, stringOverride.suffix) + func(imageFile bufimage.ImageFile, stringOverrideOptions stringOverrideOptions) string { + return getPhpMetadataNamespaceValue(imageFile, stringOverrideOptions.suffix) }, func(options *descriptorpb.FileOptions) string { return options.GetPhpMetadataNamespace() @@ -260,7 +260,7 @@ func modifyPhpMetadataNamespace( } func modifyRubyPackage( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { @@ -274,8 +274,8 @@ func modifyRubyPackage( func(bufimage.ImageFile) stringOverrideOptions { return stringOverrideOptions{value: rubyPackageValue(imageFile)} }, - func(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { - return getRubyPackageValue(imageFile, stringOverride.suffix) + func(imageFile bufimage.ImageFile, stringOverrideOptions stringOverrideOptions) string { + return getRubyPackageValue(imageFile, stringOverrideOptions.suffix) }, func(options *descriptorpb.FileOptions) string { return options.GetRubyPackage() @@ -288,11 +288,11 @@ func modifyRubyPackage( } func modifyCcEnableArenas( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { - return modifyOption[bool]( + return modifyFileOption[bool]( sweeper, imageFile, config, @@ -309,11 +309,11 @@ func modifyCcEnableArenas( } func modifyJavaMultipleFiles( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { - return modifyOption[bool]( + return modifyFileOption[bool]( sweeper, imageFile, config, @@ -330,11 +330,11 @@ func modifyJavaMultipleFiles( } func modifyJavaStringCheckUtf8( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { - return modifyOption[bool]( + return modifyFileOption[bool]( sweeper, imageFile, config, @@ -351,11 +351,11 @@ func modifyJavaStringCheckUtf8( } func modifyOptmizeFor( - sweeper *markSweeper, + sweeper internal.MarkSweeper, imageFile bufimage.ImageFile, config bufconfig.GenerateManagedConfig, ) error { - return modifyOption[descriptorpb.FileOptions_OptimizeMode]( + return modifyFileOption[descriptorpb.FileOptions_OptimizeMode]( sweeper, imageFile, config, @@ -370,3 +370,97 @@ func modifyOptmizeFor( optimizeForPath, ) } + +func modifyFileOption[T bool | descriptorpb.FileOptions_OptimizeMode]( + sweeper internal.MarkSweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + fileOption bufconfig.FileOption, + // You can set this value to the same as protobuf default, in order to not modify a value by default. + defaultValue T, + getOptionFunc func(*descriptorpb.FileOptions) T, + setOptionFunc func(*descriptorpb.FileOptions, T), + sourceLocationPath []int32, +) error { + value := defaultValue + if isFileOptionDisabledForFile( + imageFile, + fileOption, + config, + ) { + return nil + } + override, err := overrideFromConfig[T]( + imageFile, + config, + fileOption, + ) + if err != nil { + return err + } + if override != nil { + value = *override + } + descriptor := imageFile.FileDescriptorProto() + if getOptionFunc(descriptor.Options) == value { + // The option is already set to the same value, don't modify or mark it. + return nil + } + if descriptor.Options == nil { + descriptor.Options = &descriptorpb.FileOptions{} + } + setOptionFunc(descriptor.Options, value) + sweeper.Mark(imageFile, sourceLocationPath) + return nil +} + +func modifyStringOption( + sweeper internal.MarkSweeper, + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + valueOption bufconfig.FileOption, + prefixOption bufconfig.FileOption, + suffixOption bufconfig.FileOption, + defaultOptionsFunc func(bufimage.ImageFile) stringOverrideOptions, + valueFunc func(bufimage.ImageFile, stringOverrideOptions) string, + getOptionFunc func(*descriptorpb.FileOptions) string, + setOptionFunc func(*descriptorpb.FileOptions, string), + sourceLocationPath []int32, +) error { + overrideOptions, err := stringOverrideFromConfig( + imageFile, + config, + defaultOptionsFunc(imageFile), + valueOption, + prefixOption, + suffixOption, + ) + if err != nil { + return err + } + var emptyOverrideOptions stringOverrideOptions + // This means the options are all disabled. + if overrideOptions == emptyOverrideOptions { + return nil + } + // Now either value is set or prefix and/or suffix is set. + value := overrideOptions.value + if value == "" { + // TODO: pass in prefix and suffix, instead of just override options + value = valueFunc(imageFile, overrideOptions) + } + if value == "" { + return nil + } + descriptor := imageFile.FileDescriptorProto() + if getOptionFunc(descriptor.Options) == value { + // The option is already set to the same value, don't modify or mark it. + return nil + } + if descriptor.Options == nil { + descriptor.Options = &descriptorpb.FileOptions{} + } + setOptionFunc(descriptor.Options, value) + sweeper.Mark(imageFile, sourceLocationPath) + return nil +} diff --git a/private/bufpkg/bufimage/bufimagemodify/field_options_trie.go b/private/bufpkg/bufimage/bufimagemodify/internal/field_options_trie.go similarity index 99% rename from private/bufpkg/bufimage/bufimagemodify/field_options_trie.go rename to private/bufpkg/bufimage/bufimagemodify/internal/field_options_trie.go index db533b7b46..e2de1c8962 100644 --- a/private/bufpkg/bufimage/bufimagemodify/field_options_trie.go +++ b/private/bufpkg/bufimage/bufimagemodify/internal/field_options_trie.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bufimagemodify +package internal import ( "sort" diff --git a/private/bufpkg/bufimage/bufimagemodify/internal/internal.go b/private/bufpkg/bufimage/bufimagemodify/internal/internal.go new file mode 100644 index 0000000000..e2098ca932 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/internal/internal.go @@ -0,0 +1,33 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import "github.com/bufbuild/buf/private/bufpkg/bufimage" + +// fileOptionPath is the path prefix used for FileOptions. +// All file option locations are preceded by a location +// with a path set to the fileOptionPath. +// https://github.com/protocolbuffers/protobuf/blob/053966b4959bdd21e4a24e657bcb97cb9de9e8a4/src/google/protobuf/descriptor.proto#L80 +var fileOptionPath = []int32{8} + +// MarkSweeper is a mark sweeper for an image. +type MarkSweeper interface { + Mark(imageFile bufimage.ImageFile, path []int32) + Sweep() error +} + +func NewMarkSweeper(image bufimage.Image) MarkSweeper { + return newMarkSweeper(image) +} diff --git a/private/bufpkg/bufimage/bufimagemodify/location_path_dfa.go b/private/bufpkg/bufimage/bufimagemodify/internal/location_path_dfa.go similarity index 99% rename from private/bufpkg/bufimage/bufimagemodify/location_path_dfa.go rename to private/bufpkg/bufimage/bufimagemodify/internal/location_path_dfa.go index 8a445b3204..2381fbe6f9 100644 --- a/private/bufpkg/bufimage/bufimagemodify/location_path_dfa.go +++ b/private/bufpkg/bufimage/bufimagemodify/internal/location_path_dfa.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bufimagemodify +package internal const ( pathTypeNotFieldOption pathType = iota + 1 diff --git a/private/bufpkg/bufimage/bufimagemodify/marksweeper.go b/private/bufpkg/bufimage/bufimagemodify/internal/marksweeper.go similarity index 94% rename from private/bufpkg/bufimage/bufimagemodify/marksweeper.go rename to private/bufpkg/bufimage/bufimagemodify/internal/marksweeper.go index 41f587d09f..513bcab1f6 100644 --- a/private/bufpkg/bufimage/bufimagemodify/marksweeper.go +++ b/private/bufpkg/bufimage/bufimagemodify/internal/marksweeper.go @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bufimagemodify +package internal import ( "fmt" "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/slicesext" "google.golang.org/protobuf/types/descriptorpb" ) @@ -97,7 +98,7 @@ func removeLocationsFromSourceCodeInfo( return fmt.Errorf("path %v must have a preceding parent path", location.Path) } if isPathForFileOption(location.Path) { - if !Int32SliceIsEqual(sourceCodeInfo.Location[i-1].Path, fileOptionPath) { + if !slicesext.ElementsEqual(sourceCodeInfo.Location[i-1].Path, fileOptionPath) { return fmt.Errorf("file option path %v must have a preceding parent path equal to %v", location.Path, fileOptionPath) } // Add the target path and its parent. @@ -160,19 +161,6 @@ func isPathForFileOption(path []int32) bool { return len(path) == fileOptionPathLen && path[0] == fileOptionPath[0] } -// Int32SliceIsEqual returns true if x and y contain the same elements. -func Int32SliceIsEqual(x []int32, y []int32) bool { - if len(x) != len(y) { - return false - } - for i, elem := range x { - if elem != y[i] { - return false - } - } - return true -} - // getPathKey returns a unique key for the given path. func getPathKey(path []int32) string { key := make([]byte, len(path)*4) diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go b/private/bufpkg/bufimage/bufimagemodify/new_modify2.go deleted file mode 100644 index 632acfab19..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify2.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "fmt" - - "github.com/bufbuild/buf/private/bufpkg/bufconfig" - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/normalpath" - "google.golang.org/protobuf/types/descriptorpb" -) - -func modifyOption[T bool | descriptorpb.FileOptions_OptimizeMode]( - sweeper *markSweeper, - imageFile bufimage.ImageFile, - config bufconfig.GenerateManagedConfig, - fileOption bufconfig.FileOption, - // You can set this value to the same as protobuf default, in order to not modify a value by default. - defaultValue T, - getOptionFunc func(*descriptorpb.FileOptions) T, - setOptionFunc func(*descriptorpb.FileOptions, T), - sourceLocationPath []int32, -) error { - value := defaultValue - if isFileOptionDisabledForFile( - imageFile, - fileOption, - config, - ) { - return nil - } - override, err := overrideFromConfig[T]( - imageFile, - config, - fileOption, - ) - if err != nil { - return err - } - if override != nil { - value = *override - } - descriptor := imageFile.FileDescriptorProto() - if getOptionFunc(descriptor.Options) == value { - // The option is already set to the same value, don't modify or mark it. - return nil - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - setOptionFunc(descriptor.Options, value) - sweeper.Mark(imageFile, sourceLocationPath) - return nil -} - -// returns the override value and whether managed mode is DISABLED for this file for this file option. -func overrideFromConfig[T bool | descriptorpb.FileOptions_OptimizeMode]( - imageFile bufimage.ImageFile, - config bufconfig.GenerateManagedConfig, - fileOption bufconfig.FileOption, -) (*T, error) { - var override *T - for _, overrideRule := range config.Overrides() { - if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { - continue - } - if overrideRule.FileOption() != fileOption { - continue - } - value, ok := overrideRule.Value().(T) - if !ok { - // This should never happen, since the override rule has been validated. - return nil, fmt.Errorf("invalid value type for %v override: %T", fileOption, overrideRule.Value()) - } - override = &value - } - return override, nil -} - -func modifyStringOption( - sweeper *markSweeper, - imageFile bufimage.ImageFile, - config bufconfig.GenerateManagedConfig, - valueOption bufconfig.FileOption, - prefixOption bufconfig.FileOption, - suffixOption bufconfig.FileOption, - // todo: options? unify - defaultOptionsFunc func(bufimage.ImageFile) stringOverrideOptions, - valueFunc func(bufimage.ImageFile, stringOverrideOptions) string, - getOptionFunc func(*descriptorpb.FileOptions) string, - setOptionFunc func(*descriptorpb.FileOptions, string), - sourceLocationPath []int32, -) error { - overrideOptions, err := stringOverrideFromConfig( - imageFile, - config, - defaultOptionsFunc(imageFile), - valueOption, - prefixOption, - suffixOption, - ) - if err != nil { - return err - } - var emptyOverrideOptions stringOverrideOptions - // This means the options are all disabled. - if overrideOptions == emptyOverrideOptions { - return nil - } - // Now either value is set or prefix and/or suffix is set. - value := overrideOptions.value - if value == "" { - // TODO: pass in prefix and suffix, instead of just override options - value = valueFunc(imageFile, overrideOptions) - } - if value == "" { - return nil - } - descriptor := imageFile.FileDescriptorProto() - if getOptionFunc(descriptor.Options) == value { - // The option is already set to the same value, don't modify or mark it. - return nil - } - if descriptor.Options == nil { - descriptor.Options = &descriptorpb.FileOptions{} - } - setOptionFunc(descriptor.Options, value) - sweeper.Mark(imageFile, sourceLocationPath) - return nil -} - -// TODO: see if this still needs to return a pointer as opposed to a struct -// the first value is nil when no override rule is matched -func stringOverrideFromConfig( - imageFile bufimage.ImageFile, - config bufconfig.GenerateManagedConfig, - defaultOverrideOptions stringOverrideOptions, - valueFileOption bufconfig.FileOption, - prefixFileOption bufconfig.FileOption, - suffixFileOption bufconfig.FileOption, -) (stringOverrideOptions, error) { - if isFileOptionDisabledForFile( - imageFile, - valueFileOption, - config, - ) { - return stringOverrideOptions{}, nil - } - overrideOptions := defaultOverrideOptions - ignorePrefix := prefixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, prefixFileOption, config) - ignoreSuffix := suffixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, suffixFileOption, config) - if ignorePrefix { - overrideOptions.prefix = "" - } - if ignoreSuffix { - overrideOptions.suffix = "" - } - for _, overrideRule := range config.Overrides() { - if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { - continue - } - switch overrideRule.FileOption() { - case valueFileOption: - valueString, ok := overrideRule.Value().(string) - if !ok { - // This should never happen, since the override rule has been validated. - return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", valueFileOption, overrideRule.Value()) - } - // If the latest override matched is a value override (java_package as opposed to java_package_prefix), use the value. - overrideOptions = stringOverrideOptions{value: valueString} - case prefixFileOption: - if ignorePrefix { - continue - } - prefix, ok := overrideRule.Value().(string) - if !ok { - // This should never happen, since the override rule has been validated. - return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", prefixFileOption, overrideRule.Value()) - } - // Keep the suffix if the last two overrides are suffix and prefix. - overrideOptions = stringOverrideOptions{ - prefix: prefix, - suffix: overrideOptions.suffix, - } - case suffixFileOption: - if ignoreSuffix { - continue - } - suffix, ok := overrideRule.Value().(string) - if !ok { - // This should never happen, since the override rule has been validated. - return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", suffixFileOption, overrideRule.Value()) - } - // Keep the prefix if the last two overrides are suffix and prefix. - overrideOptions = stringOverrideOptions{ - prefix: overrideOptions.prefix, - suffix: suffix, - } - } - } - return overrideOptions, nil -} - -// TODO: rename to string override options maybe? -type stringOverrideOptions struct { - value string - prefix string - suffix string -} - -func isFileOptionDisabledForFile( - imageFile bufimage.ImageFile, - fileOption bufconfig.FileOption, - config bufconfig.GenerateManagedConfig, -) bool { - for _, disableRule := range config.Disables() { - if disableRule.FileOption() != bufconfig.FileOptionUnspecified && disableRule.FileOption() != fileOption { - continue - } - if !fileMatchConfig(imageFile, disableRule.Path(), disableRule.ModuleFullName()) { - continue - } - return true - } - return false -} - -func fileMatchConfig( - imageFile bufimage.ImageFile, - requiredPath string, - requiredModuleFullName string, -) bool { - if requiredPath != "" && !normalpath.EqualsOrContainsPath(requiredPath, imageFile.Path(), normalpath.Relative) { - return false - } - if requiredModuleFullName != "" && (imageFile.ModuleFullName() == nil || imageFile.ModuleFullName().String() != requiredModuleFullName) { - return false - } - return true -} - -// TODO: rename/clean up these helpers (and merge with the other original ones as well) -func getJavaPackageValue(imageFile bufimage.ImageFile, stringOverride stringOverrideOptions) string { - if pkg := imageFile.FileDescriptorProto().GetPackage(); pkg != "" { - if stringOverride.prefix != "" { - pkg = stringOverride.prefix + "." + pkg - } - if stringOverride.suffix != "" { - pkg = pkg + "." + stringOverride.suffix - } - return pkg - } - return "" -} - -func getCsharpNamespaceValue(imageFile bufimage.ImageFile, prefix string) string { - namespace := csharpNamespaceValue(imageFile) - if namespace == "" { - return "" - } - if prefix == "" { - return namespace - } - return prefix + "." + namespace -} - -func getPhpMetadataNamespaceValue(imageFile bufimage.ImageFile, suffix string) string { - namespace := phpNamespaceValue(imageFile) - if namespace == "" { - return "" - } - if suffix == "" { - return namespace - } - return namespace + `\` + suffix -} - -func getRubyPackageValue(imageFile bufimage.ImageFile, suffix string) string { - rubyPackage := rubyPackageValue(imageFile) - if rubyPackage == "" { - return "" - } - if suffix == "" { - return rubyPackage - } - return rubyPackage + "::" + suffix -} diff --git a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go b/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go deleted file mode 100644 index b83ca613e6..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/new_modify_test.go +++ /dev/null @@ -1,798 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "context" - "path/filepath" - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufconfig" - "github.com/bufbuild/buf/private/bufpkg/bufconfig/bufconfigtest" - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagetesting" - "github.com/bufbuild/buf/private/bufpkg/bufmodule" - "github.com/bufbuild/buf/private/pkg/slicesext" - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/testing/protocmp" - "google.golang.org/protobuf/types/descriptorpb" -) - -func TestModifyImage(t *testing.T) { - t.Parallel() - testcases := []struct { - description string - dirPathToModuleFullName map[string]string - config bufconfig.GenerateManagedConfig - filePathToExpectedOptions map[string]*descriptorpb.FileOptions - }{ - { - description: "nil_config", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig(false, nil, nil), - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "foo_empty/with_package.proto": nil, - "bar_all/with_package.proto": { - CcEnableArenas: proto.Bool(false), - CcGenericServices: proto.Bool(false), - CsharpNamespace: proto.String("bar"), - GoPackage: proto.String("bar"), - JavaGenericServices: proto.Bool(false), - JavaMultipleFiles: proto.Bool(false), - JavaOuterClassname: proto.String("bar"), - JavaPackage: proto.String("bar"), - JavaStringCheckUtf8: proto.Bool(false), - ObjcClassPrefix: proto.String("bar"), - OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), - PhpClassPrefix: proto.String("bar"), - PhpGenericServices: proto.Bool(false), - PhpMetadataNamespace: proto.String("bar"), - PhpNamespace: proto.String("bar"), - PyGenericServices: proto.Bool(false), - RubyPackage: proto.String("bar"), - SwiftPrefix: proto.String("bar"), - }, - }, - }, - { - description: "empty_config", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{}, - ), - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "foo_empty/with_package.proto": { - // CcEnableArena's default value is true - CsharpNamespace: proto.String("Foo.Empty"), - // GoPackage is not modified by default - JavaMultipleFiles: proto.Bool(true), - JavaOuterClassname: proto.String("WithPackageProto"), - JavaPackage: proto.String("com.foo.empty"), - // JavaStringCheckUtf8 is not modified by default - ObjcClassPrefix: proto.String("FEX"), - // OptimizeFor tries to modifiy this value to SPEED, which is already the default - // Empty is a keyword in php - PhpMetadataNamespace: proto.String(`Foo\Empty_\GPBMetadata`), - PhpNamespace: proto.String(`Foo\Empty_`), - RubyPackage: proto.String("Foo::Empty"), - }, - "foo_empty/without_package.proto": { - // CcEnableArena's default value is true - // GoPackage is not modified by default - JavaMultipleFiles: proto.Bool(true), - JavaOuterClassname: proto.String("WithoutPackageProto"), - // JavaStringCheckUtf8 is not modified by default - // OptimizeFor tries to modifiy this value to SPEED, which is already the default - }, - "bar_all/with_package.proto": { - CcEnableArenas: proto.Bool(true), - CcGenericServices: proto.Bool(false), - CsharpNamespace: proto.String("Bar.All"), - GoPackage: proto.String("bar"), - JavaGenericServices: proto.Bool(false), - JavaMultipleFiles: proto.Bool(true), - JavaOuterClassname: proto.String("WithPackageProto"), - JavaPackage: proto.String("com.bar.all"), - JavaStringCheckUtf8: proto.Bool(false), - ObjcClassPrefix: proto.String("BAX"), - OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), - PhpClassPrefix: proto.String("bar"), - PhpGenericServices: proto.Bool(false), - PhpMetadataNamespace: proto.String(`Bar\All\GPBMetadata`), - PhpNamespace: proto.String(`Bar\All`), - PyGenericServices: proto.Bool(false), - RubyPackage: proto.String("Bar::All"), - SwiftPrefix: proto.String("bar"), - }, - "bar_all/without_package.proto": { - CcEnableArenas: proto.Bool(true), - CcGenericServices: proto.Bool(false), - CsharpNamespace: proto.String("bar"), - GoPackage: proto.String("bar"), - JavaGenericServices: proto.Bool(false), - JavaMultipleFiles: proto.Bool(true), - JavaOuterClassname: proto.String("WithoutPackageProto"), - JavaPackage: proto.String("bar"), - JavaStringCheckUtf8: proto.Bool(false), - ObjcClassPrefix: proto.String("bar"), - OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), - PhpClassPrefix: proto.String("bar"), - PhpGenericServices: proto.Bool(false), - PhpMetadataNamespace: proto.String(`bar`), - PhpNamespace: proto.String(`bar`), - PyGenericServices: proto.Bool(false), - RubyPackage: proto.String("bar"), - SwiftPrefix: proto.String("bar"), - }, - }, - }, - } - for _, testcase := range testcases { - testcase := testcase - for _, includeSourceInfo := range []bool{true, false} { - includeSourceInfo := includeSourceInfo - t.Run(testcase.description, func(t *testing.T) { - t.Parallel() - image := testGetImageFromDirs(t, testcase.dirPathToModuleFullName, includeSourceInfo) - err := Modify( - context.Background(), - image, - testcase.config, - ) - require.NoError(t, err) - for filePath, expectedOptions := range testcase.filePathToExpectedOptions { - imageFile := image.GetFile(filePath) - require.NotNil(t, imageFile) - require.Empty( - t, - cmp.Diff(expectedOptions, imageFile.FileDescriptorProto().GetOptions(), protocmp.Transform()), - imageFile.FileDescriptorProto().GetOptions(), - ) - } - }) - } - } -} - -func TestModifyImageFile( - t *testing.T, -) { - t.Parallel() - testcases := []struct { - description string - dirPathToModuleFullName map[string]string - config bufconfig.GenerateManagedConfig - modifyFunc func(*markSweeper, bufimage.ImageFile, bufconfig.GenerateManagedConfig) error - filePathToExpectedOptions map[string]*descriptorpb.FileOptions - filePathToExpectedMarkedLocationPaths map[string][][]int32 - }{ - { - description: "cc_enable_arena", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionCcEnableArenas, false), - }, - ), - modifyFunc: modifyCcEnableArenas, - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "foo_empty/without_package.proto": nil, - "bar_empty/without_package.proto": { - CcEnableArenas: proto.Bool(false), - }, - }, - filePathToExpectedMarkedLocationPaths: map[string][][]int32{ - "bar_empty/without_package.proto": {ccEnableArenasPath}, - }, - }, - { - description: "csharp_namespace", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionCsharpNamespacePrefix, bufconfig.FieldOptionUnspecified), - }, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespacePrefix, "BarPrefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespace, "BarValue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespace, "FooValue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespacePrefix, "FooPrefix"), - }, - ), - modifyFunc: modifyCsharpNamespace, - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "bar_empty/with_package.proto": { - CsharpNamespace: proto.String("BarPrefix.Bar.Empty"), - }, - "bar_empty/without_package.proto": { - CsharpNamespace: proto.String("BarValue"), - }, - "foo_empty/with_package.proto": { - CsharpNamespace: proto.String("FooValue"), - }, - "foo_empty/without_package.proto": nil, - }, - filePathToExpectedMarkedLocationPaths: map[string][][]int32{ - "bar_empty/with_package.proto": {csharpNamespacePath}, - "bar_empty/without_package.proto": {csharpNamespacePath}, - "foo_empty/with_package.proto": {csharpNamespacePath}, - }, - }, - { - description: "go_package", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionGoPackagePrefix, bufconfig.FieldOptionUnspecified), - }, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionGoPackagePrefix, "barprefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionGoPackage, "barvalue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty/with_package.proto", "buf.build/acme/foo", bufconfig.FileOptionGoPackage, "foovalue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionGoPackagePrefix, "fooprefix"), - }, - ), - modifyFunc: modifyGoPackage, - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "bar_empty/with_package.proto": { - GoPackage: proto.String("barprefix/bar_empty"), - }, - "bar_empty/without_package.proto": { - GoPackage: proto.String("barvalue"), - }, - "foo_empty/with_package.proto": { - GoPackage: proto.String("foovalue"), - }, - "foo_empty/without_package.proto": { - GoPackage: proto.String("fooprefix/foo_empty"), - }, - }, - filePathToExpectedMarkedLocationPaths: map[string][][]int32{ - "bar_empty/with_package.proto": {goPackagePath}, - "bar_empty/without_package.proto": {goPackagePath}, - "foo_empty/with_package.proto": {goPackagePath}, - "foo_empty/without_package.proto": {goPackagePath}, - }, - }, - { - description: "java_package_prefix", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackagePrefix, bufconfig.FieldOptionUnspecified), - }, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackagePrefix, "barprefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackageSuffix, "foosuffix"), - }, - ), - modifyFunc: modifyJavaPackage, - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "foo_empty/with_package.proto": { - // default prefix and override suffix - JavaPackage: proto.String("com.foo.empty.foosuffix"), - }, - // prefix is disabled - "bar_empty/with_package.proto": nil, - // prefix is overridden - "bar_all/with_package.proto": { - JavaPackage: proto.String("barprefix.bar.all"), - // below this point are the values from the file - CcEnableArenas: proto.Bool(false), - CcGenericServices: proto.Bool(false), - CsharpNamespace: proto.String("bar"), - GoPackage: proto.String("bar"), - JavaGenericServices: proto.Bool(false), - JavaMultipleFiles: proto.Bool(false), - JavaOuterClassname: proto.String("bar"), - JavaStringCheckUtf8: proto.Bool(false), - ObjcClassPrefix: proto.String("bar"), - OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), - PhpClassPrefix: proto.String("bar"), - PhpGenericServices: proto.Bool(false), - PhpMetadataNamespace: proto.String("bar"), - PhpNamespace: proto.String("bar"), - PyGenericServices: proto.Bool(false), - RubyPackage: proto.String("bar"), - SwiftPrefix: proto.String("bar"), - }, - // not modified because it doesn't have a package - "foo_empty/without_package.proto": nil, - "bar_empty/without_package.proto": nil, - "foo_all/without_package.proto": { - // values are from the file - CcEnableArenas: proto.Bool(true), - CcGenericServices: proto.Bool(true), - CsharpNamespace: proto.String("foo"), - GoPackage: proto.String("foo"), - JavaGenericServices: proto.Bool(true), - JavaMultipleFiles: proto.Bool(true), - JavaOuterClassname: proto.String("foo"), - JavaPackage: proto.String("foo"), - JavaStringCheckUtf8: proto.Bool(true), - ObjcClassPrefix: proto.String("foo"), - OptimizeFor: descriptorpb.FileOptions_CODE_SIZE.Enum(), - PhpClassPrefix: proto.String("foo"), - PhpGenericServices: proto.Bool(true), - PhpMetadataNamespace: proto.String("foo"), - PhpNamespace: proto.String("foo"), - PyGenericServices: proto.Bool(true), - RubyPackage: proto.String("foo"), - SwiftPrefix: proto.String("foo"), - }, - "bar_all/without_package.proto": { - // values are from the file - CcEnableArenas: proto.Bool(false), - CcGenericServices: proto.Bool(false), - CsharpNamespace: proto.String("bar"), - GoPackage: proto.String("bar"), - JavaGenericServices: proto.Bool(false), - JavaMultipleFiles: proto.Bool(false), - JavaOuterClassname: proto.String("bar"), - JavaPackage: proto.String("bar"), - JavaStringCheckUtf8: proto.Bool(false), - ObjcClassPrefix: proto.String("bar"), - OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), - PhpClassPrefix: proto.String("bar"), - PhpGenericServices: proto.Bool(false), - PhpMetadataNamespace: proto.String("bar"), - PhpNamespace: proto.String("bar"), - PyGenericServices: proto.Bool(false), - RubyPackage: proto.String("bar"), - SwiftPrefix: proto.String("bar"), - }, - }, - filePathToExpectedMarkedLocationPaths: map[string][][]int32{ - "foo_empty/with_package.proto": {javaPackagePath}, - "bar_all/with_package.proto": {javaPackagePath}, - }, - }, - { - description: "java_package_suffix", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackageSuffix, bufconfig.FieldOptionUnspecified), - }, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), - }, - ), - modifyFunc: modifyJavaPackage, - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "foo_empty/with_package.proto": { - // only suffix matches, but apply both prefix and suffix - JavaPackage: proto.String("com.foo.empty.suffix"), - }, - "bar_empty/with_package.proto": { - // only prefix because suffix is disabled - JavaPackage: proto.String("com.bar.empty"), - }, - "bar_all/with_package.proto": { - JavaPackage: proto.String("com.bar.all.suffix"), - // below this point are the values from the file - CcEnableArenas: proto.Bool(false), - CcGenericServices: proto.Bool(false), - CsharpNamespace: proto.String("bar"), - GoPackage: proto.String("bar"), - JavaGenericServices: proto.Bool(false), - JavaMultipleFiles: proto.Bool(false), - JavaOuterClassname: proto.String("bar"), - JavaStringCheckUtf8: proto.Bool(false), - ObjcClassPrefix: proto.String("bar"), - OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), - PhpClassPrefix: proto.String("bar"), - PhpGenericServices: proto.Bool(false), - PhpMetadataNamespace: proto.String("bar"), - PhpNamespace: proto.String("bar"), - PyGenericServices: proto.Bool(false), - RubyPackage: proto.String("bar"), - SwiftPrefix: proto.String("bar"), - }, - // not modified - "foo_empty/without_package.proto": nil, - }, - filePathToExpectedMarkedLocationPaths: map[string][][]int32{ - "foo_empty/with_package.proto": {javaPackagePath}, - "bar_empty/with_package.proto": {javaPackagePath}, - "bar_all/with_package.proto": {javaPackagePath}, - }, - }, - { - description: "java_package", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackage, bufconfig.FieldOptionUnspecified), - }, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackage, "bar.value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackage, "foo.value"), - }, - ), - modifyFunc: modifyJavaPackage, - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - // bar_empty disabled - "bar_empty/with_package.proto": nil, - "bar_empty/without_package.proto": nil, - "bar_all/with_package.proto": { - JavaPackage: proto.String("bar.value"), - CcEnableArenas: proto.Bool(false), - CcGenericServices: proto.Bool(false), - CsharpNamespace: proto.String("bar"), - GoPackage: proto.String("bar"), - JavaGenericServices: proto.Bool(false), - JavaMultipleFiles: proto.Bool(false), - JavaOuterClassname: proto.String("bar"), - JavaStringCheckUtf8: proto.Bool(false), - ObjcClassPrefix: proto.String("bar"), - OptimizeFor: descriptorpb.FileOptions_SPEED.Enum(), - PhpClassPrefix: proto.String("bar"), - PhpGenericServices: proto.Bool(false), - PhpMetadataNamespace: proto.String("bar"), - PhpNamespace: proto.String("bar"), - PyGenericServices: proto.Bool(false), - RubyPackage: proto.String("bar"), - SwiftPrefix: proto.String("bar"), - }, - "foo_empty/with_package.proto": { - JavaPackage: proto.String("foo.value"), - }, - "foo_empty/without_package.proto": { - JavaPackage: proto.String("foo.value"), - }, - }, - filePathToExpectedMarkedLocationPaths: map[string][][]int32{ - "foo_empty/with_package.proto": {javaPackagePath}, - "foo_empty/without_package.proto": {javaPackagePath}, - "bar_all/with_package.proto": {javaPackagePath}, - }, - }, - { - description: "objc_class_prefix", - dirPathToModuleFullName: map[string]string{ - filepath.Join("testdata", "foo"): "buf.build/acme/foo", - filepath.Join("testdata", "bar"): "buf.build/acme/bar", - }, - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionObjcClassPrefix, bufconfig.FieldOptionUnspecified), - }, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionObjcClassPrefix, "BAR"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOO"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_all", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOOALL"), - }, - ), - modifyFunc: modifyObjcClassPrefix, - filePathToExpectedOptions: map[string]*descriptorpb.FileOptions{ - "bar_empty/with_package.proto": { - ObjcClassPrefix: proto.String("BAR"), - }, - "bar_empty/without_package.proto": { - ObjcClassPrefix: proto.String("BAR"), - }, - // disabled - "foo_empty/with_package.proto": nil, - // no package - "foo_empty/without_package.proto": { - ObjcClassPrefix: proto.String("FOO"), - }, - "foo_all/with_package.proto": { - ObjcClassPrefix: proto.String("FOOALL"), - CcEnableArenas: proto.Bool(true), - CcGenericServices: proto.Bool(true), - CsharpNamespace: proto.String("foo"), - GoPackage: proto.String("foo"), - JavaGenericServices: proto.Bool(true), - JavaMultipleFiles: proto.Bool(true), - JavaOuterClassname: proto.String("foo"), - JavaPackage: proto.String("foo"), - JavaStringCheckUtf8: proto.Bool(true), - OptimizeFor: descriptorpb.FileOptions_CODE_SIZE.Enum(), - PhpClassPrefix: proto.String("foo"), - PhpGenericServices: proto.Bool(true), - PhpMetadataNamespace: proto.String("foo"), - PhpNamespace: proto.String("foo"), - PyGenericServices: proto.Bool(true), - RubyPackage: proto.String("foo"), - SwiftPrefix: proto.String("foo"), - }, - }, - filePathToExpectedMarkedLocationPaths: map[string][][]int32{ - "bar_empty/with_package.proto": {objcClassPrefixPath}, - "bar_empty/without_package.proto": {objcClassPrefixPath}, - "foo_empty/without_package.proto": {objcClassPrefixPath}, - "foo_all/without_package.proto": {objcClassPrefixPath}, - "foo_all/with_package.proto": {objcClassPrefixPath}, - }, - }, - } - for _, testcase := range testcases { - testcase := testcase - for _, includeSourceInfo := range []bool{true, false} { - // TODO: we are only testing sweep here, no need to test both include and exclude source info - includeSourceInfo := includeSourceInfo - t.Run(testcase.description, func(t *testing.T) { - t.Parallel() - image := testGetImageFromDirs(t, testcase.dirPathToModuleFullName, includeSourceInfo) - sweeper := newMarkSweeper(image) - // TODO: check include source code info - for filePath, expectedOptions := range testcase.filePathToExpectedOptions { - imageFile := image.GetFile(filePath) - testcase.modifyFunc( - sweeper, - imageFile, - testcase.config, - ) - require.NotNil(t, imageFile) - require.Empty( - t, - cmp.Diff(expectedOptions, imageFile.FileDescriptorProto().GetOptions(), protocmp.Transform()), - "incorrect options result for %s", - filePath, - ) - // TODO: full map compare - require.Equal( - t, - slicesext.Map(testcase.filePathToExpectedMarkedLocationPaths[filePath], getPathKey), - slicesext.MapKeysToSortedSlice(sweeper.sourceCodeInfoPaths[filePath]), - ) - } - }) - } - } -} - -// TODO: add default values -func TestGetStringOverrideFromConfig(t *testing.T) { - t.Parallel() - testcases := []struct { - description string - config bufconfig.GenerateManagedConfig - imageFile bufimage.ImageFile - defaultOverrideOptions stringOverrideOptions - expectedOverride stringOverrideOptions - expectedDisable bool - }{ - { - description: "only_value", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{value: "value"}, - }, - { - description: "only_prefix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{prefix: "prefix"}, - }, - { - description: "only_suffix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{suffix: "suffix"}, - }, - { - description: "prefix_then_value", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{value: "value"}, - }, - { - description: "value_then_prefix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{prefix: "prefix"}, - }, - { - description: "prefix_then_suffix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{ - prefix: "prefix", - suffix: "suffix", - }, - }, - { - description: "value_prefix_then_suffix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{ - prefix: "prefix", - suffix: "suffix", - }, - }, - { - description: "prefix_value_then_suffix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{suffix: "suffix"}, - }, - { - description: "prefix_then_prefix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix2"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{prefix: "prefix2"}, - }, - { - description: "suffix_then_suffix", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix2"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{suffix: "suffix2"}, - }, - { - description: "value_then_value", - config: bufconfig.NewGenerateManagedConfig( - true, - []bufconfig.ManagedDisableRule{}, - []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value2"), - }, - ), - imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), - expectedOverride: stringOverrideOptions{value: "value2"}, - }, - } - for _, testcase := range testcases { - testcase := testcase - t.Run(testcase.description, func(t *testing.T) { - t.Parallel() - override, err := stringOverrideFromConfig( - testcase.imageFile, - testcase.config, - testcase.defaultOverrideOptions, - bufconfig.FileOptionJavaPackage, - bufconfig.FileOptionJavaPackagePrefix, - bufconfig.FileOptionJavaPackageSuffix, - ) - require.NoError(t, err) - require.Equal(t, testcase.expectedOverride, override) - }) - } -} - -func TestModifyFieldOption(t *testing.T) { - t.Parallel() - // TODO in v2 -} - -func testGetImageFile( - t *testing.T, - path string, - moduleFullName string, -) bufimage.ImageFile { - parsedModuleFullName, err := bufmodule.ParseModuleFullName(moduleFullName) - require.NoError(t, err) - return bufimagetesting.NewImageFile( - t, - &descriptorpb.FileDescriptorProto{ - Name: proto.String(path), - Syntax: proto.String("proto3"), - }, - parsedModuleFullName, - "", - path, - false, - false, - nil, - ) -} diff --git a/private/bufpkg/bufimage/bufimagemodify/override.go b/private/bufpkg/bufimage/bufimagemodify/override.go new file mode 100644 index 0000000000..813c05e839 --- /dev/null +++ b/private/bufpkg/bufimage/bufimagemodify/override.go @@ -0,0 +1,421 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufimagemodify + +import ( + "fmt" + "path" + "strings" + "unicode" + + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/pkg/normalpath" + "github.com/bufbuild/buf/private/pkg/protoversion" + "github.com/bufbuild/buf/private/pkg/stringutil" + "google.golang.org/protobuf/types/descriptorpb" +) + +// Keywords and classes that could be produced by our heuristic. +// They must not be used in a php_namespace. +// Ref: https://www.php.net/manual/en/reserved.php +var phpReservedKeywords = map[string]struct{}{ + // Reserved classes as per above. + "directory": {}, + "exception": {}, + "errorexception": {}, + "closure": {}, + "generator": {}, + "arithmeticerror": {}, + "assertionerror": {}, + "divisionbyzeroerror": {}, + "error": {}, + "throwable": {}, + "parseerror": {}, + "typeerror": {}, + // Keywords avoided by protoc. + // Ref: https://github.com/protocolbuffers/protobuf/blob/66d749188ff2a2e30e932110222d58da7c6a8d49/src/google/protobuf/compiler/php/php_generator.cc#L50-L66 + "abstract": {}, + "and": {}, + "array": {}, + "as": {}, + "break": {}, + "callable": {}, + "case": {}, + "catch": {}, + "class": {}, + "clone": {}, + "const": {}, + "continue": {}, + "declare": {}, + "default": {}, + "die": {}, + "do": {}, + "echo": {}, + "else": {}, + "elseif": {}, + "empty": {}, + "enddeclare": {}, + "endfor": {}, + "endforeach": {}, + "endif": {}, + "endswitch": {}, + "endwhile": {}, + "eval": {}, + "exit": {}, + "extends": {}, + "final": {}, + "finally": {}, + "fn": {}, + "for": {}, + "foreach": {}, + "function": {}, + "global": {}, + "goto": {}, + "if": {}, + "implements": {}, + "include": {}, + "include_once": {}, + "instanceof": {}, + "insteadof": {}, + "interface": {}, + "isset": {}, + "list": {}, + "match": {}, + "namespace": {}, + "new": {}, + "or": {}, + "print": {}, + "private": {}, + "protected": {}, + "public": {}, + "require": {}, + "require_once": {}, + "return": {}, + "static": {}, + "switch": {}, + "throw": {}, + "trait": {}, + "try": {}, + "unset": {}, + "use": {}, + "var": {}, + "while": {}, + "xor": {}, + "yield": {}, + "int": {}, + "float": {}, + "bool": {}, + "string": {}, + "true": {}, + "false": {}, + "null": {}, + "void": {}, + "iterable": {}, +} + +type stringOverrideOptions struct { + value string + prefix string + suffix string +} + +func stringOverrideFromConfig( + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + defaultOverrideOptions stringOverrideOptions, + valueFileOption bufconfig.FileOption, + prefixFileOption bufconfig.FileOption, + suffixFileOption bufconfig.FileOption, +) (stringOverrideOptions, error) { + if isFileOptionDisabledForFile( + imageFile, + valueFileOption, + config, + ) { + return stringOverrideOptions{}, nil + } + overrideOptions := defaultOverrideOptions + ignorePrefix := prefixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, prefixFileOption, config) + ignoreSuffix := suffixFileOption == bufconfig.FileOptionUnspecified || isFileOptionDisabledForFile(imageFile, suffixFileOption, config) + if ignorePrefix { + overrideOptions.prefix = "" + } + if ignoreSuffix { + overrideOptions.suffix = "" + } + for _, overrideRule := range config.Overrides() { + if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { + continue + } + switch overrideRule.FileOption() { + case valueFileOption: + valueString, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", valueFileOption, overrideRule.Value()) + } + // If the latest override matched is a value override (java_package as opposed to java_package_prefix), use the value. + overrideOptions = stringOverrideOptions{value: valueString} + case prefixFileOption: + if ignorePrefix { + continue + } + prefix, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", prefixFileOption, overrideRule.Value()) + } + // Keep the suffix if the last two overrides are suffix and prefix. + overrideOptions = stringOverrideOptions{ + prefix: prefix, + suffix: overrideOptions.suffix, + } + case suffixFileOption: + if ignoreSuffix { + continue + } + suffix, ok := overrideRule.Value().(string) + if !ok { + // This should never happen, since the override rule has been validated. + return stringOverrideOptions{}, fmt.Errorf("invalid value type for %v override: %T", suffixFileOption, overrideRule.Value()) + } + // Keep the prefix if the last two overrides are suffix and prefix. + overrideOptions = stringOverrideOptions{ + prefix: overrideOptions.prefix, + suffix: suffix, + } + } + } + return overrideOptions, nil +} + +// returns the override value and whether managed mode is DISABLED for this file for this file option. +func overrideFromConfig[T bool | descriptorpb.FileOptions_OptimizeMode]( + imageFile bufimage.ImageFile, + config bufconfig.GenerateManagedConfig, + fileOption bufconfig.FileOption, +) (*T, error) { + var override *T + for _, overrideRule := range config.Overrides() { + if !fileMatchConfig(imageFile, overrideRule.Path(), overrideRule.ModuleFullName()) { + continue + } + if overrideRule.FileOption() != fileOption { + continue + } + value, ok := overrideRule.Value().(T) + if !ok { + // This should never happen, since the override rule has been validated. + return nil, fmt.Errorf("invalid value type for %v override: %T", fileOption, overrideRule.Value()) + } + override = &value + } + return override, nil +} + +func isFileOptionDisabledForFile( + imageFile bufimage.ImageFile, + fileOption bufconfig.FileOption, + config bufconfig.GenerateManagedConfig, +) bool { + for _, disableRule := range config.Disables() { + if disableRule.FileOption() != bufconfig.FileOptionUnspecified && disableRule.FileOption() != fileOption { + continue + } + if !fileMatchConfig(imageFile, disableRule.Path(), disableRule.ModuleFullName()) { + continue + } + return true + } + return false +} + +func fileMatchConfig( + imageFile bufimage.ImageFile, + requiredPath string, + requiredModuleFullName string, +) bool { + if requiredPath != "" && !normalpath.EqualsOrContainsPath(requiredPath, imageFile.Path(), normalpath.Relative) { + return false + } + if requiredModuleFullName != "" && (imageFile.ModuleFullName() == nil || imageFile.ModuleFullName().String() != requiredModuleFullName) { + return false + } + return true +} + +// TODO: unify naming of these helpers +func getJavaPackageValue(imageFile bufimage.ImageFile, stringOverrideOptions stringOverrideOptions) string { + if pkg := imageFile.FileDescriptorProto().GetPackage(); pkg != "" { + if stringOverrideOptions.prefix != "" { + pkg = stringOverrideOptions.prefix + "." + pkg + } + if stringOverrideOptions.suffix != "" { + pkg = pkg + "." + stringOverrideOptions.suffix + } + return pkg + } + return "" +} + +func getCsharpNamespaceValue(imageFile bufimage.ImageFile, prefix string) string { + namespace := csharpNamespaceValue(imageFile) + if namespace == "" { + return "" + } + if prefix == "" { + return namespace + } + return prefix + "." + namespace +} + +func getPhpMetadataNamespaceValue(imageFile bufimage.ImageFile, suffix string) string { + namespace := phpNamespaceValue(imageFile) + if namespace == "" { + return "" + } + if suffix == "" { + return namespace + } + return namespace + `\` + suffix +} + +func getRubyPackageValue(imageFile bufimage.ImageFile, suffix string) string { + rubyPackage := rubyPackageValue(imageFile) + if rubyPackage == "" { + return "" + } + if suffix == "" { + return rubyPackage + } + return rubyPackage + "::" + suffix +} + +// TODO: is this needed? +// csharpNamespaceValue returns the csharp_namespace for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func csharpNamespaceValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + packageParts := strings.Split(pkg, ".") + for i, part := range packageParts { + packageParts[i] = stringutil.ToPascalCase(part) + } + return strings.Join(packageParts, ".") +} + +// goPackageImportPathForFile returns the go_package import path for the given +// ImageFile. If the package contains a version suffix, and if there are more +// than two components, concatenate the final two components. Otherwise, we +// exclude the ';' separator and adopt the default behavior from the import path. +// +// For example, an ImageFile with `package acme.weather.v1;` will include `;weatherv1` +// in the `go_package` declaration so that the generated package is named as such. +func goPackageImportPathForFile(imageFile bufimage.ImageFile, importPathPrefix string) string { + goPackageImportPath := path.Join(importPathPrefix, path.Dir(imageFile.Path())) + packageName := imageFile.FileDescriptorProto().GetPackage() + if _, ok := protoversion.NewPackageVersionForPackage(packageName); ok { + parts := strings.Split(packageName, ".") + if len(parts) >= 2 { + goPackageImportPath += ";" + parts[len(parts)-2] + parts[len(parts)-1] + } + } + return goPackageImportPath +} + +func javaOuterClassnameValue(imageFile bufimage.ImageFile) string { + return stringutil.ToPascalCase(normalpath.Base(imageFile.Path())) +} + +// objcClassPrefixValue returns the objc_class_prefix for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func objcClassPrefixValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + _, hasPackageVersion := protoversion.NewPackageVersionForPackage(pkg) + packageParts := strings.Split(pkg, ".") + var prefixParts []rune + for i, part := range packageParts { + // Check if last part is a version before appending. + if i == len(packageParts)-1 && hasPackageVersion { + continue + } + // Probably should never be a non-ASCII character, + // but why not support it just in case? + runeSlice := []rune(part) + prefixParts = append(prefixParts, unicode.ToUpper(runeSlice[0])) + } + for len(prefixParts) < 3 { + prefixParts = append(prefixParts, 'X') + } + prefix := string(prefixParts) + if prefix == "GPB" { + prefix = "GPX" + } + return prefix +} + +// phpMetadataNamespaceValue returns the php_metadata_namespace for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func phpMetadataNamespaceValue(imageFile bufimage.ImageFile) string { + phpNamespace := phpNamespaceValue(imageFile) + if phpNamespace == "" { + return "" + } + return phpNamespace + `\GPBMetadata` +} + +// phpNamespaceValue returns the php_namespace for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func phpNamespaceValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + packageParts := strings.Split(pkg, ".") + for i, part := range packageParts { + packagePart := stringutil.ToPascalCase(part) + if _, ok := phpReservedKeywords[strings.ToLower(part)]; ok { + // Append _ to the package part if it is a reserved keyword. + packagePart += "_" + } + packageParts[i] = packagePart + } + return strings.Join(packageParts, `\`) +} + +// rubyPackageValue returns the ruby_package for the given ImageFile based on its +// package declaration. If the image file doesn't have a package declaration, an +// empty string is returned. +func rubyPackageValue(imageFile bufimage.ImageFile) string { + pkg := imageFile.FileDescriptorProto().GetPackage() + if pkg == "" { + return "" + } + packageParts := strings.Split(pkg, ".") + for i, part := range packageParts { + packageParts[i] = stringutil.ToPascalCase(part) + } + return strings.Join(packageParts, "::") +} diff --git a/private/bufpkg/bufimage/bufimagemodify/vars.go b/private/bufpkg/bufimage/bufimagemodify/vars.go deleted file mode 100644 index 23d70b907e..0000000000 --- a/private/bufpkg/bufimage/bufimagemodify/vars.go +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufimagemodify - -import ( - "path" - "strings" - "unicode" - - "github.com/bufbuild/buf/private/bufpkg/bufimage" - "github.com/bufbuild/buf/private/pkg/normalpath" - "github.com/bufbuild/buf/private/pkg/protoversion" - "github.com/bufbuild/buf/private/pkg/stringutil" -) - -var ( - // fileOptionPath is the path prefix used for FileOptions. - // All file option locations are preceded by a location - // with a path set to the fileOptionPath. - // https://github.com/protocolbuffers/protobuf/blob/053966b4959bdd21e4a24e657bcb97cb9de9e8a4/src/google/protobuf/descriptor.proto#L80 - fileOptionPath = []int32{8} - // ccEnableArenas is the SourceCodeInfo path for the cc_enable_arenas option. - // https://github.com/protocolbuffers/protobuf/blob/29152fbc064921ca982d64a3a9eae1daa8f979bb/src/google/protobuf/descriptor.proto#L420 - ccEnableArenasPath = []int32{8, 31} - // csharpNamespacePath is the SourceCodeInfo path for the csharp_namespace option. - // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L428 - csharpNamespacePath = []int32{8, 37} - // goPackagePath is the SourceCodeInfo path for the go_package option. - // https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L392 - goPackagePath = []int32{8, 11} - // javaMultipleFilesPath is the SourceCodeInfo path for the java_multiple_files option. - // https://github.com/protocolbuffers/protobuf/blob/ee04809540c098718121e092107fbc0abc231725/src/google/protobuf/descriptor.proto#L364 - javaMultipleFilesPath = []int32{8, 10} - // javaOuterClassnamePath is the SourceCodeInfo path for the java_outer_classname option. - // https://github.com/protocolbuffers/protobuf/blob/87d140f851131fb8a6e8a80449cf08e73e568259/src/google/protobuf/descriptor.proto#L356 - javaOuterClassnamePath = []int32{8, 8} - // javaPackagePath is the SourceCodeInfo path for the java_package option. - // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L348 - javaPackagePath = []int32{8, 1} - // javaStringCheckUtf8Path is the SourceCodeInfo path for the java_string_check_utf8 option. - // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L375 - javaStringCheckUtf8Path = []int32{8, 27} - // objcClassPrefixPath is the SourceCodeInfo path for the objc_class_prefix option. - // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L425 - objcClassPrefixPath = []int32{8, 36} - // optimizeFor is the SourceCodeInfo path for the optimize_for option. - // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L385 - optimizeForPath = []int32{8, 9} - // phpMetadataNamespacePath is the SourceCodeInfo path for the php_metadata_namespace option. - // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L448 - phpMetadataNamespacePath = []int32{8, 44} - // phpNamespacePath is the SourceCodeInfo path for the php_namespace option. - // Ref: https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L443 - phpNamespacePath = []int32{8, 41} - // Keywords and classes that could be produced by our heuristic. - // They must not be used in a php_namespace. - // Ref: https://www.php.net/manual/en/reserved.php - phpReservedKeywords = map[string]struct{}{ - // Reserved classes as per above. - "directory": {}, - "exception": {}, - "errorexception": {}, - "closure": {}, - "generator": {}, - "arithmeticerror": {}, - "assertionerror": {}, - "divisionbyzeroerror": {}, - "error": {}, - "throwable": {}, - "parseerror": {}, - "typeerror": {}, - // Keywords avoided by protoc. - // Ref: https://github.com/protocolbuffers/protobuf/blob/66d749188ff2a2e30e932110222d58da7c6a8d49/src/google/protobuf/compiler/php/php_generator.cc#L50-L66 - "abstract": {}, - "and": {}, - "array": {}, - "as": {}, - "break": {}, - "callable": {}, - "case": {}, - "catch": {}, - "class": {}, - "clone": {}, - "const": {}, - "continue": {}, - "declare": {}, - "default": {}, - "die": {}, - "do": {}, - "echo": {}, - "else": {}, - "elseif": {}, - "empty": {}, - "enddeclare": {}, - "endfor": {}, - "endforeach": {}, - "endif": {}, - "endswitch": {}, - "endwhile": {}, - "eval": {}, - "exit": {}, - "extends": {}, - "final": {}, - "finally": {}, - "fn": {}, - "for": {}, - "foreach": {}, - "function": {}, - "global": {}, - "goto": {}, - "if": {}, - "implements": {}, - "include": {}, - "include_once": {}, - "instanceof": {}, - "insteadof": {}, - "interface": {}, - "isset": {}, - "list": {}, - "match": {}, - "namespace": {}, - "new": {}, - "or": {}, - "print": {}, - "private": {}, - "protected": {}, - "public": {}, - "require": {}, - "require_once": {}, - "return": {}, - "static": {}, - "switch": {}, - "throw": {}, - "trait": {}, - "try": {}, - "unset": {}, - "use": {}, - "var": {}, - "while": {}, - "xor": {}, - "yield": {}, - "int": {}, - "float": {}, - "bool": {}, - "string": {}, - "true": {}, - "false": {}, - "null": {}, - "void": {}, - "iterable": {}, - } - // rubyPackagePath is the SourceCodeInfo path for the ruby_package option. - // https://github.com/protocolbuffers/protobuf/blob/61689226c0e3ec88287eaed66164614d9c4f2bf7/src/google/protobuf/descriptor.proto#L453 - rubyPackagePath = []int32{8, 45} -) - -// TODO: is this needed? -// csharpNamespaceValue returns the csharp_namespace for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func csharpNamespaceValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - packageParts := strings.Split(pkg, ".") - for i, part := range packageParts { - packageParts[i] = stringutil.ToPascalCase(part) - } - return strings.Join(packageParts, ".") -} - -// TODO: why was this exported -// GoPackageImportPathForFile returns the go_package import path for the given -// ImageFile. If the package contains a version suffix, and if there are more -// than two components, concatenate the final two components. Otherwise, we -// exclude the ';' separator and adopt the default behavior from the import path. -// -// For example, an ImageFile with `package acme.weather.v1;` will include `;weatherv1` -// in the `go_package` declaration so that the generated package is named as such. -func GoPackageImportPathForFile(imageFile bufimage.ImageFile, importPathPrefix string) string { - goPackageImportPath := path.Join(importPathPrefix, path.Dir(imageFile.Path())) - packageName := imageFile.FileDescriptorProto().GetPackage() - if _, ok := protoversion.NewPackageVersionForPackage(packageName); ok { - parts := strings.Split(packageName, ".") - if len(parts) >= 2 { - goPackageImportPath += ";" + parts[len(parts)-2] + parts[len(parts)-1] - } - } - return goPackageImportPath -} - -func javaOuterClassnameValue(imageFile bufimage.ImageFile) string { - return stringutil.ToPascalCase(normalpath.Base(imageFile.Path())) -} - -// objcClassPrefixValue returns the objc_class_prefix for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func objcClassPrefixValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - _, hasPackageVersion := protoversion.NewPackageVersionForPackage(pkg) - packageParts := strings.Split(pkg, ".") - var prefixParts []rune - for i, part := range packageParts { - // Check if last part is a version before appending. - if i == len(packageParts)-1 && hasPackageVersion { - continue - } - // Probably should never be a non-ASCII character, - // but why not support it just in case? - runeSlice := []rune(part) - prefixParts = append(prefixParts, unicode.ToUpper(runeSlice[0])) - } - for len(prefixParts) < 3 { - prefixParts = append(prefixParts, 'X') - } - prefix := string(prefixParts) - if prefix == "GPB" { - prefix = "GPX" - } - return prefix -} - -// phpMetadataNamespaceValue returns the php_metadata_namespace for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func phpMetadataNamespaceValue(imageFile bufimage.ImageFile) string { - phpNamespace := phpNamespaceValue(imageFile) - if phpNamespace == "" { - return "" - } - return phpNamespace + `\GPBMetadata` -} - -// phpNamespaceValue returns the php_namespace for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func phpNamespaceValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - packageParts := strings.Split(pkg, ".") - for i, part := range packageParts { - packagePart := stringutil.ToPascalCase(part) - if _, ok := phpReservedKeywords[strings.ToLower(part)]; ok { - // Append _ to the package part if it is a reserved keyword. - packagePart += "_" - } - packageParts[i] = packagePart - } - return strings.Join(packageParts, `\`) -} - -// rubyPackageValue returns the ruby_package for the given ImageFile based on its -// package declaration. If the image file doesn't have a package declaration, an -// empty string is returned. -func rubyPackageValue(imageFile bufimage.ImageFile) string { - pkg := imageFile.FileDescriptorProto().GetPackage() - if pkg == "" { - return "" - } - packageParts := strings.Split(pkg, ".") - for i, part := range packageParts { - packageParts[i] = stringutil.ToPascalCase(part) - } - return strings.Join(packageParts, "::") -} From f61c862d68210762c2639ea2be911d9a57046c18 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 4 Dec 2023 10:54:55 -0500 Subject: [PATCH 62/66] build --- private/buf/bufctl/controller.go | 48 ++++++++------------------------ 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 18e2efe9c6..74f3815b8d 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -333,42 +333,6 @@ func (c *controller) GetImageForInputConfig( functionOptions := newFunctionOptions() for _, option := range options { option(functionOptions) - switch t := ref.(type) { - case buffetch.ProtoFileRef: - workspace, err := c.getWorkspaceForProtoFileRef(ctx, t, functionOptions) - if err != nil { - return nil, err - } - return c.buildImage( - ctx, - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), - functionOptions, - ) - case buffetch.SourceRef: - workspace, err := c.getWorkspaceForSourceRef(ctx, t, functionOptions) - if err != nil { - return nil, err - } - return c.buildImage( - ctx, - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), - functionOptions, - ) - case buffetch.ModuleRef: - workspace, err := c.getWorkspaceForModuleRef(ctx, t, functionOptions) - if err != nil { - return nil, err - } - return c.buildImage( - ctx, - bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), - functionOptions, - ) - case buffetch.MessageRef: - return c.getImageForMessageRef(ctx, t, functionOptions) - default: - // This is a system error. - return nil, syserror.Newf("invalid Ref: %T", ref) } ref, err := c.buffetchRefParser.GetRefForInputConfig(ctx, inputConfig) if err != nil { @@ -630,7 +594,15 @@ func (c *controller) getImage( ) (bufimage.Image, error) { switch t := ref.(type) { case buffetch.ProtoFileRef: - return nil, errors.New("TODO") + workspace, err := c.getWorkspaceForProtoFileRef(ctx, t, functionOptions) + if err != nil { + return nil, err + } + return c.buildImage( + ctx, + bufmodule.ModuleSetToModuleReadBucketWithOnlyProtoFiles(workspace), + functionOptions, + ) case buffetch.SourceRef: workspace, err := c.getWorkspaceForSourceRef(ctx, t, functionOptions) if err != nil { @@ -657,6 +629,8 @@ func (c *controller) getImage( // This is a system error. return nil, syserror.Newf("invalid Ref: %T", ref) } +} + func (c *controller) getWorkspaceForProtoFileRef( ctx context.Context, protoFileRef buffetch.ProtoFileRef, From 235ebed5165c2c6802bc99c6a870ac1c2a066d8b Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 4 Dec 2023 11:22:42 -0500 Subject: [PATCH 63/66] remove empty line --- private/buf/buffetch/internal/internal.go | 1 - 1 file changed, 1 deletion(-) diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index 96c40cdec4..058b00ebd1 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -906,7 +906,6 @@ func GetInputConfigForRef(ref Ref, value string) (bufconfig.InputConfig, error) t.IncludePackageFiles(), ) case GitRef: - return bufconfig.NewGitRepoInputConfig( t.Path(), t.SubDirPath(), From 42dacab1f2c9adefa726e5bdaec3a2131f7afd97 Mon Sep 17 00:00:00 2001 From: Oliver Sun Date: Mon, 4 Dec 2023 12:00:49 -0500 Subject: [PATCH 64/66] update todo --- TODO.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/TODO.txt b/TODO.txt index 638339598b..385095625a 100644 --- a/TODO.txt +++ b/TODO.txt @@ -28,11 +28,10 @@ - go through all todos - Un-skip relevant tests in generate_test.go and generate_unix_test.go, when - the behavior of --path and --exclude-path is updated to match what's on main - - proto file ref is implemented +- Debug the skipped test with proto file ref in generate_test.go - Use syserror when possible (especially in buf gen related code). -- Add Enabled() bool to GenerateManagedConfig: this is useful for printing the warning for the - case where managed mode external config is not empty but the user forgot to set enable: true. - Otherwise we would need to pass a logger to many functions, breaking the uniformity. +- Find the right place (right way to pass a logger) to print a warning when a buf.gen.yaml with + non-empty managed mode but without `enabled: true`. The key is to decide where to pass the logger. - Fix the issue where reading a buf.lock with empty digest errors. Run `buf mod update` to reproduce. NOTE: We are not allowing cross-workspace finding for include_package_files=true From 4834f7b21d3c19b9f0cd5e09365702ed1e2770e9 Mon Sep 17 00:00:00 2001 From: bufdev Date: Mon, 11 Dec 2023 12:34:09 -0500 Subject: [PATCH 65/66] updates --- private/buf/bufctl/controller.go | 38 +- private/buf/buffetch/internal/internal.go | 2 +- private/buf/buffetch/internal/ref_parser.go | 121 ++-- .../buf/buffetch/internal/ref_parser_test.go | 2 +- private/buf/bufgen/bufgen.go | 12 +- private/buf/bufgen/generator.go | 12 +- private/buf/bufmigrate/bufmigrate.go | 42 -- private/buf/bufmigrate/usage.gen.go | 19 - private/buf/bufmigrate/v1beta1_migrator.go | 553 ------------------ private/buf/cmd/buf/buf.go | 1 - private/buf/cmd/buf/buf_test.go | 253 -------- .../beta/migratev1beta1/migratev1beta1.go | 87 --- .../command/beta/migratev1beta1/usage.gen.go | 19 - .../buf/cmd/buf/command/generate/generate.go | 2 + .../cmd/buf/command/generate/generate_test.go | 2 +- .../bufconfig/generate_plugin_config.go | 2 +- .../bufimagemodify/bufimagemodify_test.go | 7 +- 17 files changed, 118 insertions(+), 1056 deletions(-) delete mode 100644 private/buf/bufmigrate/bufmigrate.go delete mode 100644 private/buf/bufmigrate/usage.gen.go delete mode 100644 private/buf/bufmigrate/v1beta1_migrator.go delete mode 100644 private/buf/cmd/buf/command/beta/migratev1beta1/migratev1beta1.go delete mode 100644 private/buf/cmd/buf/command/beta/migratev1beta1/usage.gen.go diff --git a/private/buf/bufctl/controller.go b/private/buf/bufctl/controller.go index 5f6c46cb60..b18300d4f2 100644 --- a/private/buf/bufctl/controller.go +++ b/private/buf/bufctl/controller.go @@ -81,12 +81,16 @@ type Controller interface { dirPath string, options ...FunctionOption, ) (bufworkspace.UpdateableWorkspace, error) - // TODO: rename to GetImageForInputString, but in a separate PR to minimize merge conflicts GetImage( ctx context.Context, input string, options ...FunctionOption, ) (bufimage.Image, error) + GetImageForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + options ...FunctionOption, + ) (bufimage.Image, error) GetImageForWorkspace( ctx context.Context, workspace bufworkspace.Workspace, @@ -276,6 +280,18 @@ func (c *controller) GetImage( return c.getImage(ctx, input, functionOptions) } +func (c *controller) GetImageForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + options ...FunctionOption, +) (bufimage.Image, error) { + functionOptions := newFunctionOptions() + for _, option := range options { + option(functionOptions) + } + return c.getImageForInputConfig(ctx, inputConfig, functionOptions) +} + func (c *controller) GetImageForWorkspace( ctx context.Context, workspace bufworkspace.Workspace, @@ -631,6 +647,26 @@ func (c *controller) getImage( if err != nil { return nil, err } + return c.getImageForRef(ctx, ref, functionOptions) +} + +func (c *controller) getImageForInputConfig( + ctx context.Context, + inputConfig bufconfig.InputConfig, + functionOptions *functionOptions, +) (bufimage.Image, error) { + ref, err := c.buffetchRefParser.GetRefForInputConfig(ctx, inputConfig) + if err != nil { + return nil, err + } + return c.getImageForRef(ctx, ref, functionOptions) +} + +func (c *controller) getImageForRef( + ctx context.Context, + ref buffetch.Ref, + functionOptions *functionOptions, +) (bufimage.Image, error) { switch t := ref.(type) { case buffetch.ProtoFileRef: workspace, err := c.getWorkspaceForProtoFileRef(ctx, t, functionOptions) diff --git a/private/buf/buffetch/internal/internal.go b/private/buf/buffetch/internal/internal.go index d35c95759d..262f2672da 100644 --- a/private/buf/buffetch/internal/internal.go +++ b/private/buf/buffetch/internal/internal.go @@ -837,7 +837,7 @@ type GetModuleOption func(*getModuleOptions) // passed because if the ref is a git ref, it would only have a git.Name, instead // of a git branch, a git ref and a git tag. Therefore the original string is passed. func GetInputConfigForRef(ref Ref, value string) (bufconfig.InputConfig, error) { - _, options, err := getRawPathAndOptionsForInputString(value) + _, options, err := getRawPathAndOptions(value) if err != nil { return nil, err } diff --git a/private/buf/buffetch/internal/ref_parser.go b/private/buf/buffetch/internal/ref_parser.go index f01742a122..b5d10efa0d 100644 --- a/private/buf/buffetch/internal/ref_parser.go +++ b/private/buf/buffetch/internal/ref_parser.go @@ -63,7 +63,7 @@ func (a *refParser) GetParsedRef( for _, option := range options { option(getParsedRefOptions) } - return a.getParsedRefForInputString(ctx, value, getParsedRefOptions.allowedFormats) + return a.getParsedRef(ctx, value, getParsedRefOptions.allowedFormats) } func (a *refParser) GetParsedRefForInputConfig( @@ -78,17 +78,17 @@ func (a *refParser) GetParsedRefForInputConfig( return a.getParsedRefForInputConfig(ctx, inputConfig, getParsedRefOptions.allowedFormats) } -func (a *refParser) getParsedRefForInputString( +func (a *refParser) getParsedRef( ctx context.Context, value string, allowedFormats map[string]struct{}, ) (ParsedRef, error) { // path is never empty after returning from this function - path, options, err := getRawPathAndOptionsForInputString(value) + path, options, err := getRawPathAndOptions(value) if err != nil { return nil, err } - rawRef, err := a.getRawRefFromInputString(path, value, options) + rawRef, err := a.getRawRef(path, value, options) if err != nil { return nil, err } @@ -100,66 +100,16 @@ func (a *refParser) getParsedRefForInputConfig( inputConfig bufconfig.InputConfig, allowedFormats map[string]struct{}, ) (ParsedRef, error) { - rawRef, err := a.getRawRefFromInputConfig(inputConfig) + rawRef, err := a.getRawRefForInputConfig(inputConfig) if err != nil { return nil, err } return a.parseRawRef(rawRef, allowedFormats) } -func (a *refParser) parseRawRef( - rawRef *RawRef, - allowedFormats map[string]struct{}, -) (ParsedRef, error) { - singleFormatInfo, singleOK := a.singleFormatToInfo[rawRef.Format] - archiveFormatInfo, archiveOK := a.archiveFormatToInfo[rawRef.Format] - _, dirOK := a.dirFormatToInfo[rawRef.Format] - _, gitOK := a.gitFormatToInfo[rawRef.Format] - _, moduleOK := a.moduleFormatToInfo[rawRef.Format] - _, protoFileOK := a.protoFileFormatToInfo[rawRef.Format] - if !(singleOK || archiveOK || dirOK || gitOK || moduleOK || protoFileOK) { - return nil, NewFormatUnknownError(rawRef.Format) - } - if len(allowedFormats) > 0 { - if _, ok := allowedFormats[rawRef.Format]; !ok { - return nil, NewFormatNotAllowedError(rawRef.Format, allowedFormats) - } - } - if !singleOK && len(rawRef.UnrecognizedOptions) > 0 { - // Only single refs allow custom options. In every other case, this is an error. - // - // We verify unrecognized options match what is expected in getSingleRef. - keys := make([]string, 0, len(rawRef.UnrecognizedOptions)) - for key := range rawRef.UnrecognizedOptions { - keys = append(keys, key) - } - sort.Strings(keys) - return nil, NewOptionsInvalidKeysError(keys...) - } - if singleOK { - return getSingleRef(rawRef, singleFormatInfo.defaultCompressionType, singleFormatInfo.customOptionKeys) - } - if archiveOK { - return getArchiveRef(rawRef, archiveFormatInfo.archiveType, archiveFormatInfo.defaultCompressionType) - } - if protoFileOK { - return getProtoFileRef(rawRef) - } - if dirOK { - return getDirRef(rawRef) - } - if gitOK { - return getGitRef(rawRef) - } - if moduleOK { - return getModuleRef(rawRef) - } - return nil, NewFormatUnknownError(rawRef.Format) -} - -func (a *refParser) getRawRefFromInputString( +func (a *refParser) getRawRef( path string, - // TODO: need a better name + // Used to reference the input config in error messages. displayName string, options map[string]string, ) (*RawRef, error) { @@ -225,7 +175,6 @@ func (a *refParser) getRawRefFromInputString( switch value { case "true": rawRef.IncludePackageFiles = true - // TODO: perhaps empty values should not be accepted case "false", "": rawRef.IncludePackageFiles = false default: @@ -250,7 +199,7 @@ func (a *refParser) getRawRefFromInputString( return rawRef, nil } -func (a *refParser) getRawRefFromInputConfig( +func (a *refParser) getRawRefForInputConfig( inputConfig bufconfig.InputConfig, ) (*RawRef, error) { rawRef := &RawRef{ @@ -320,6 +269,56 @@ func (a *refParser) getRawRefFromInputConfig( return rawRef, nil } +func (a *refParser) parseRawRef( + rawRef *RawRef, + allowedFormats map[string]struct{}, +) (ParsedRef, error) { + singleFormatInfo, singleOK := a.singleFormatToInfo[rawRef.Format] + archiveFormatInfo, archiveOK := a.archiveFormatToInfo[rawRef.Format] + _, dirOK := a.dirFormatToInfo[rawRef.Format] + _, gitOK := a.gitFormatToInfo[rawRef.Format] + _, moduleOK := a.moduleFormatToInfo[rawRef.Format] + _, protoFileOK := a.protoFileFormatToInfo[rawRef.Format] + if !(singleOK || archiveOK || dirOK || gitOK || moduleOK || protoFileOK) { + return nil, NewFormatUnknownError(rawRef.Format) + } + if len(allowedFormats) > 0 { + if _, ok := allowedFormats[rawRef.Format]; !ok { + return nil, NewFormatNotAllowedError(rawRef.Format, allowedFormats) + } + } + if !singleOK && len(rawRef.UnrecognizedOptions) > 0 { + // Only single refs allow custom options. In every other case, this is an error. + // + // We verify unrecognized options match what is expected in getSingleRef. + keys := make([]string, 0, len(rawRef.UnrecognizedOptions)) + for key := range rawRef.UnrecognizedOptions { + keys = append(keys, key) + } + sort.Strings(keys) + return nil, NewOptionsInvalidKeysError(keys...) + } + if singleOK { + return getSingleRef(rawRef, singleFormatInfo.defaultCompressionType, singleFormatInfo.customOptionKeys) + } + if archiveOK { + return getArchiveRef(rawRef, archiveFormatInfo.archiveType, archiveFormatInfo.defaultCompressionType) + } + if protoFileOK { + return getProtoFileRef(rawRef) + } + if dirOK { + return getDirRef(rawRef) + } + if gitOK { + return getGitRef(rawRef) + } + if moduleOK { + return getModuleRef(rawRef) + } + return nil, NewFormatUnknownError(rawRef.Format) +} + func (a *refParser) validateRawRef( displayName string, rawRef *RawRef, @@ -403,9 +402,9 @@ func parseSubDirPath(value string) (string, error) { return subDirPath, nil } -// getRawPathAndOptionsForInputString returns the raw path and options from the value provided, +// getRawPathAndOptions returns the raw path and options from the value provided, // the rawPath will be non-empty when returning without error here. -func getRawPathAndOptionsForInputString(value string) (string, map[string]string, error) { +func getRawPathAndOptions(value string) (string, map[string]string, error) { value = strings.TrimSpace(value) if value == "" { return "", nil, newValueEmptyError() diff --git a/private/buf/buffetch/internal/ref_parser_test.go b/private/buf/buffetch/internal/ref_parser_test.go index 669ceda078..83fbd6a721 100644 --- a/private/buf/buffetch/internal/ref_parser_test.go +++ b/private/buf/buffetch/internal/ref_parser_test.go @@ -81,7 +81,7 @@ func testGetRawPathAndOptionsError( ) { t.Run(value, func(t *testing.T) { t.Parallel() - _, _, err := getRawPathAndOptionsForInputString(value) + _, _, err := getRawPathAndOptions(value) assert.EqualError(t, err, expectedErr.Error()) }) } diff --git a/private/buf/bufgen/bufgen.go b/private/buf/bufgen/bufgen.go index 9741d40ab3..39f68329e8 100644 --- a/private/buf/bufgen/bufgen.go +++ b/private/buf/bufgen/bufgen.go @@ -29,18 +29,10 @@ import ( "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/connectclient" "github.com/bufbuild/buf/private/pkg/storage/storageos" + "github.com/bufbuild/buf/private/pkg/tracing" "go.uber.org/zap" ) -const ( - // ExternalConfigFilePath is the default external configuration file path. - ExternalConfigFilePath = "buf.gen.yaml" - // V1Version is the string used to identify the v1 version of the generate template. - V1Version = "v1" - // V1Beta1Version is the string used to identify the v1beta1 version of the generate template. - V1Beta1Version = "v1beta1" -) - const ( // StrategyDirectory is the strategy that says to generate per directory. // @@ -97,6 +89,7 @@ type Generator interface { // NewGenerator returns a new Generator. func NewGenerator( logger *zap.Logger, + tracer tracing.Tracer, storageosProvider storageos.Provider, runner command.Runner, wasmPluginExecutor bufwasm.PluginExecutor, @@ -107,6 +100,7 @@ func NewGenerator( ) Generator { return newGenerator( logger, + tracer, storageosProvider, runner, wasmPluginExecutor, diff --git a/private/buf/bufgen/generator.go b/private/buf/bufgen/generator.go index f7ca75f723..3bbd759706 100644 --- a/private/buf/bufgen/generator.go +++ b/private/buf/bufgen/generator.go @@ -21,22 +21,23 @@ import ( "path/filepath" connect "connectrpc.com/connect" + "github.com/bufbuild/buf/private/buf/bufpluginexec" "github.com/bufbuild/buf/private/bufpkg/bufconfig" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify" "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" - "github.com/bufbuild/buf/private/buf/bufpluginexec" "github.com/bufbuild/buf/private/bufpkg/bufwasm" "github.com/bufbuild/buf/private/gen/proto/connect/buf/alpha/registry/v1alpha1/registryv1alpha1connect" registryv1alpha1 "github.com/bufbuild/buf/private/gen/proto/go/buf/alpha/registry/v1alpha1" "github.com/bufbuild/buf/private/pkg/app" - "github.com/bufbuild/buf/private/pkg/protoplugin" - "github.com/bufbuild/buf/private/pkg/protoplugin/protopluginos" "github.com/bufbuild/buf/private/pkg/command" "github.com/bufbuild/buf/private/pkg/connectclient" + "github.com/bufbuild/buf/private/pkg/protoplugin" + "github.com/bufbuild/buf/private/pkg/protoplugin/protopluginos" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/thread" + "github.com/bufbuild/buf/private/pkg/tracing" "go.uber.org/multierr" "go.uber.org/zap" "google.golang.org/protobuf/types/pluginpb" @@ -44,6 +45,7 @@ import ( type generator struct { logger *zap.Logger + tracer tracing.Tracer storageosProvider storageos.Provider pluginexecGenerator bufpluginexec.Generator clientConfig *connectclient.Config @@ -51,6 +53,7 @@ type generator struct { func newGenerator( logger *zap.Logger, + tracer tracing.Tracer, storageosProvider storageos.Provider, runner command.Runner, wasmPluginExecutor bufwasm.PluginExecutor, @@ -58,8 +61,9 @@ func newGenerator( ) *generator { return &generator{ logger: logger, + tracer: tracer, storageosProvider: storageosProvider, - pluginexecGenerator: bufpluginexec.NewGenerator(logger, storageosProvider, runner, wasmPluginExecutor), + pluginexecGenerator: bufpluginexec.NewGenerator(logger, tracer, storageosProvider, runner, wasmPluginExecutor), clientConfig: clientConfig, } } diff --git a/private/buf/bufmigrate/bufmigrate.go b/private/buf/bufmigrate/bufmigrate.go deleted file mode 100644 index f0f27c46be..0000000000 --- a/private/buf/bufmigrate/bufmigrate.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package bufmigrate contains logic for migrating between different -// configuration file versions. -package bufmigrate - -// Migrator describes the interface used to migrate -// a set of files in a directory from one version to another. -type Migrator interface { - Migrate(dirPath string) error -} - -// V1Beta1MigrateOption defines the type used -// to configure the v1beta1 migrator. -type V1Beta1MigrateOption func(*v1beta1Migrator) - -// NewV1Beta1Migrator creates a new migrator that migrates files -// between version v1beta1 and v1. -func NewV1Beta1Migrator(commandName string, options ...V1Beta1MigrateOption) Migrator { - return newV1Beta1Migrator(commandName, options...) -} - -// V1Beta1MigratorWithNotifier instruments the migrator with -// a callback to call whenever an event that should notify the -// user occurs during the migration. -func V1Beta1MigratorWithNotifier(notifier func(message string) error) V1Beta1MigrateOption { - return func(migrateOptions *v1beta1Migrator) { - migrateOptions.notifier = notifier - } -} diff --git a/private/buf/bufmigrate/usage.gen.go b/private/buf/bufmigrate/usage.gen.go deleted file mode 100644 index a27acc9b2a..0000000000 --- a/private/buf/bufmigrate/usage.gen.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Generated. DO NOT EDIT. - -package bufmigrate - -import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/bufmigrate/v1beta1_migrator.go b/private/buf/bufmigrate/v1beta1_migrator.go deleted file mode 100644 index 8b62262419..0000000000 --- a/private/buf/bufmigrate/v1beta1_migrator.go +++ /dev/null @@ -1,553 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufmigrate - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/bufbuild/buf/private/buf/bufgen" - "github.com/bufbuild/buf/private/buf/bufwork" - "github.com/bufbuild/buf/private/bufpkg/bufcheck/bufbreaking/bufbreakingconfig" - "github.com/bufbuild/buf/private/bufpkg/bufcheck/buflint/buflintconfig" - "github.com/bufbuild/buf/private/bufpkg/bufconfig" - "github.com/bufbuild/buf/private/bufpkg/buflock" - "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleconfig" - "github.com/bufbuild/buf/private/pkg/encoding" - "github.com/bufbuild/buf/private/pkg/stringutil" -) - -const ( - bufModHeaderWithName = `# Generated by %q. Edit as necessary, and -# remove this comment when you're finished. -# -# This module represents the %q root found in -# the previous configuration file for the -# %q module. -` - bufModHeaderWithoutName = `# Generated by %q. Edit as necessary, and -# remove this comment when you're finished. -# -# This module represents the %q root found in -# the previous configuration. -` - bufGenHeader = `# Generated by %q. Edit as necessary, and -# remove this comment when you're finished. -` - bufWorkHeader = `# Generated by %q. Edit as necessary, and -# remove this comment when you're finished. -# -# This workspace file points to the roots found in your -# previous %q configuration. -` -) - -type v1beta1Migrator struct { - notifier func(string) error - commandName string -} - -func newV1Beta1Migrator(commandName string, options ...V1Beta1MigrateOption) *v1beta1Migrator { - migrator := v1beta1Migrator{ - commandName: commandName, - notifier: func(string) error { return nil }, - } - for _, option := range options { - option(&migrator) - } - return &migrator -} - -func (m *v1beta1Migrator) Migrate(dirPath string) error { - migratedConfig, err := m.maybeMigrateConfig(dirPath) - if err != nil { - return fmt.Errorf("failed to migrate config: %w", err) - } - migratedGenTemplate, err := m.maybeMigrateGenTemplate(dirPath) - if err != nil { - return fmt.Errorf("failed to migrate generation template: %w", err) - } - migratedLockFile, err := m.maybeMigrateLockFile(dirPath) - if err != nil { - return fmt.Errorf("failed to migrate lock file: %w", err) - } - if !migratedConfig && !migratedGenTemplate && !migratedLockFile { - return nil - } - var migratedFiles []string - if migratedConfig { - migratedFiles = append(migratedFiles, bufconfig.ExternalConfigV1Beta1FilePath) - } - if migratedGenTemplate { - migratedFiles = append(migratedFiles, bufgen.ExternalConfigFilePath) - } - if migratedLockFile { - migratedFiles = append(migratedFiles, buflock.ExternalConfigFilePath) - } - if err := m.notifier( - fmt.Sprintf("Successfully migrated your %s to v1.\n", stringutil.SliceToHumanString(migratedFiles)), - ); err != nil { - return fmt.Errorf("failed to write success message: %w", err) - } - return nil -} - -func (m *v1beta1Migrator) maybeMigrateConfig(dirPath string) (bool, error) { - oldConfigPath := filepath.Join(dirPath, bufconfig.ExternalConfigV1Beta1FilePath) - oldConfigBytes, err := os.ReadFile(oldConfigPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - // OK, no old config file - return false, nil - } - return false, fmt.Errorf("failed to read file: %w", err) - } - var versionedConfig bufconfig.ExternalConfigVersion - if err := encoding.UnmarshalYAMLNonStrict(oldConfigBytes, &versionedConfig); err != nil { - return false, fmt.Errorf( - "failed to read %s version: %w", - oldConfigPath, - err, - ) - } - switch versionedConfig.Version { - case bufconfig.V1Version: - // OK, file was already v1 - return false, nil - case bufconfig.V1Beta1Version, "": - // Continue to migrate - default: - return false, fmt.Errorf("unknown config file version: %s", versionedConfig.Version) - } - var v1beta1Config bufconfig.ExternalConfigV1Beta1 - if err := encoding.UnmarshalYAMLStrict(oldConfigBytes, &v1beta1Config); err != nil { - return false, fmt.Errorf( - "failed to unmarshal %s as %s version v1beta1: %w", - oldConfigPath, - bufconfig.ExternalConfigV1Beta1FilePath, - err, - ) - } - buildConfig, err := bufmoduleconfig.NewConfigV1Beta1(v1beta1Config.Build, v1beta1Config.Deps...) - if err != nil { - return false, err - } - if excludes, ok := buildConfig.RootToExcludes["."]; len(buildConfig.RootToExcludes) == 1 && ok { - // Only "." root present, just recreate file - v1Config := bufconfig.ExternalConfigV1{ - Version: bufconfig.V1Version, - Name: v1beta1Config.Name, - Deps: v1beta1Config.Deps, - Build: bufmoduleconfig.ExternalConfigV1{ - Excludes: excludes, - }, - Breaking: bufbreakingconfig.ExternalConfigV1(v1beta1Config.Breaking), - Lint: buflintconfig.ExternalConfigV1(v1beta1Config.Lint), - } - newConfigPath := filepath.Join(dirPath, bufconfig.ExternalConfigV1FilePath) - if err := m.writeV1Config(newConfigPath, v1Config, ".", v1beta1Config.Name); err != nil { - return false, err - } - // Delete the old file once we've created the new one, - // unless it's the same file as before. - if newConfigPath != oldConfigPath { - if err := os.Remove(oldConfigPath); err != nil { - return false, fmt.Errorf("failed to delete old config file: %w", err) - } - } - return true, nil - } - // Check if we have a co-resident lock file, of any version - oldLockFilePath := filepath.Join(dirPath, buflock.ExternalConfigFilePath) - externalLockFileV1, hasLockFile, err := maybeReadLockFile(oldLockFilePath) - if err != nil { - return false, err - } - pathToProcessed := make(map[string]bool) - for root, excludes := range buildConfig.RootToExcludes { - // Convert universal settings - var name string - if v1beta1Config.Name != "" { - name = v1beta1Config.Name + "-" + strings.ReplaceAll(root, "/", "-") // Note: roots are normalized, "/" is universal - } - v1Config := bufconfig.ExternalConfigV1{ - Version: bufconfig.V1Version, - Name: name, - Deps: v1beta1Config.Deps, - Build: bufmoduleconfig.ExternalConfigV1{ - Excludes: excludes, - }, - Breaking: bufbreakingconfig.ExternalConfigV1{ - Use: v1beta1Config.Breaking.Use, - Except: v1beta1Config.Breaking.Except, - IgnoreUnstablePackages: v1beta1Config.Breaking.IgnoreUnstablePackages, - }, - Lint: buflintconfig.ExternalConfigV1{ - Use: v1beta1Config.Lint.Use, - Except: v1beta1Config.Lint.Except, - ServiceSuffix: v1beta1Config.Lint.ServiceSuffix, - EnumZeroValueSuffix: v1beta1Config.Lint.EnumZeroValueSuffix, - RPCAllowSameRequestResponse: v1beta1Config.Lint.RPCAllowSameRequestResponse, - RPCAllowGoogleProtobufEmptyRequests: v1beta1Config.Lint.RPCAllowGoogleProtobufEmptyRequests, - RPCAllowGoogleProtobufEmptyResponses: v1beta1Config.Lint.RPCAllowGoogleProtobufEmptyResponses, - AllowCommentIgnores: v1beta1Config.Lint.AllowCommentIgnores, - }, - } - - // Process Ignore's for those related to the root - v1Config.Breaking.Ignore, err = convertIgnoreSlice(v1beta1Config.Breaking.Ignore, dirPath, root, pathToProcessed) - if err != nil { - return false, err - } - v1Config.Breaking.IgnoreOnly, err = convertIgnoreMap(v1beta1Config.Breaking.IgnoreOnly, dirPath, root, pathToProcessed) - if err != nil { - return false, err - } - v1Config.Lint.Ignore, err = convertIgnoreSlice(v1beta1Config.Lint.Ignore, dirPath, root, pathToProcessed) - if err != nil { - return false, err - } - v1Config.Lint.IgnoreOnly, err = convertIgnoreMap(v1beta1Config.Lint.IgnoreOnly, dirPath, root, pathToProcessed) - if err != nil { - return false, err - } - if err := m.writeV1Config( - filepath.Join(dirPath, root, bufconfig.ExternalConfigV1FilePath), - v1Config, - root, - v1beta1Config.Name, - ); err != nil { - return false, err - } - if hasLockFile { - if err := m.writeV1LockFile( - filepath.Join(dirPath, root, buflock.ExternalConfigFilePath), - externalLockFileV1, - ); err != nil { - return false, err - } - } - } - for path, processed := range pathToProcessed { - if !processed { - if err := m.notifier( - fmt.Sprintf( - "The ignored file %q was not found in any roots and has been removed.\n", - path, - ), - ); err != nil { - return false, fmt.Errorf("failed to warn about ignored file: %w", err) - } - } - } - workConfig := bufwork.ExternalConfigV1{ - Version: bufwork.V1Version, - Directories: v1beta1Config.Build.Roots, - } - // Sort directories before marshalling for deterministic output - sort.Strings(workConfig.Directories) - workConfigBytes, err := encoding.MarshalYAML(&workConfig) - if err != nil { - return false, fmt.Errorf("failed to marshal workspace file: %w", err) - } - header := fmt.Sprintf(bufWorkHeader, m.commandName, bufconfig.ExternalConfigV1Beta1FilePath) - if err := os.WriteFile( - filepath.Join(dirPath, bufwork.ExternalConfigV1FilePath), - append([]byte(header), workConfigBytes...), - 0600, - ); err != nil { - return false, fmt.Errorf("failed to write workspace file: %w", err) - } - // Finally, delete the old `buf.yaml` and any `buf.lock`. This is safe to do unconditionally - // as we know that there can't be a new `buf.yaml` here, since the only case - // where that would be true is if the only root is ".", which is handled separately. - if err := os.Remove(oldConfigPath); err != nil { - return false, fmt.Errorf("failed to clean up old config file: %w", err) - } - if hasLockFile { - if err := os.Remove(oldLockFilePath); err != nil { - return false, fmt.Errorf("failed to clean up old lock file: %w", err) - } - } - return true, nil -} - -// writeV1Config atomically replaces the old configuration file by first writing -// the new config to a temporary file and then moving it to the old config file path. -// If we fail to marshal or write, the old config file is not touched. -func (m *v1beta1Migrator) writeV1Config( - configPath string, - config bufconfig.ExternalConfigV1, - originalRootName string, - originalModuleName string, -) (retErr error) { - v1ConfigData, err := encoding.MarshalYAML(&config) - if err != nil { - return fmt.Errorf("failed to marshal new config: %w", err) - } - header := fmt.Sprintf(bufModHeaderWithName, m.commandName, originalRootName, originalModuleName) - if originalModuleName == "" { - header = fmt.Sprintf(bufModHeaderWithoutName, m.commandName, originalRootName) - } - v1ConfigData = append([]byte(header), v1ConfigData...) - if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil { - // This happens if the user has a root specified that doesn't have a corresponding - // directory on the filesystem. - return fmt.Errorf("failed to create new directories for writing config: %w", err) - } - return os.WriteFile(configPath, v1ConfigData, 0600) -} - -func (m *v1beta1Migrator) maybeMigrateGenTemplate(dirPath string) (bool, error) { - oldConfigPath := filepath.Join(dirPath, bufgen.ExternalConfigFilePath) - oldConfigBytes, err := os.ReadFile(oldConfigPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - // OK, no old config file - return false, nil - } - return false, fmt.Errorf("failed to read file: %w", err) - } - var versionedConfig bufgen.ExternalConfigVersion - if err := encoding.UnmarshalYAMLNonStrict(oldConfigBytes, &versionedConfig); err != nil { - return false, fmt.Errorf( - "failed to read %s version: %w", - oldConfigPath, - err, - ) - } - switch versionedConfig.Version { - case bufgen.V1Version: - // OK, file was already v1 - return false, nil - case bufgen.V1Beta1Version, "": - // Continue to migrate - default: - return false, fmt.Errorf("unknown config file version: %s", versionedConfig.Version) - } - var v1beta1GenTemplate bufgen.ExternalConfigV1Beta1 - if err := encoding.UnmarshalYAMLStrict(oldConfigBytes, &v1beta1GenTemplate); err != nil { - return false, fmt.Errorf( - "failed to unmarshal %s as %s version v1beta1: %w", - oldConfigPath, - bufgen.ExternalConfigFilePath, - err, - ) - } - v1GenTemplate := bufgen.ExternalConfigV1{ - Version: bufgen.V1Version, - Managed: bufgen.ExternalManagedConfigV1{ - Enabled: v1beta1GenTemplate.Managed, - CcEnableArenas: v1beta1GenTemplate.Options.CcEnableArenas, - JavaMultipleFiles: v1beta1GenTemplate.Options.JavaMultipleFiles, - OptimizeFor: bufgen.ExternalOptimizeForConfigV1{Default: v1beta1GenTemplate.Options.OptimizeFor}, - }, - } - for _, plugin := range v1beta1GenTemplate.Plugins { - pluginConfig := bufgen.ExternalPluginConfigV1{ - Name: plugin.Name, - Out: plugin.Out, - Opt: plugin.Opt, - Strategy: plugin.Strategy, - } - if plugin.Path != "" { - pluginConfig.Path = plugin.Path - } - v1GenTemplate.Plugins = append(v1GenTemplate.Plugins, pluginConfig) - } - newConfigPath := filepath.Join(dirPath, bufgen.ExternalConfigFilePath) - if err := m.writeV1GenTemplate(newConfigPath, v1GenTemplate); err != nil { - return false, err - } - return true, nil -} - -// writeV1GenTemplate atomically replaces the old configuration file by first writing -// the new config to a temporary file and then moving it to the old config file path. -// If we fail to marshal or write, the old config file is not touched. -func (m *v1beta1Migrator) writeV1GenTemplate( - configPath string, - config bufgen.ExternalConfigV1, -) (retErr error) { - v1ConfigData, err := encoding.MarshalYAML(&config) - if err != nil { - return fmt.Errorf("failed to marshal new config: %w", err) - } - header := fmt.Sprintf(bufGenHeader, m.commandName) - v1ConfigData = append([]byte(header), v1ConfigData...) - return os.WriteFile(configPath, v1ConfigData, 0600) -} - -func (m *v1beta1Migrator) maybeMigrateLockFile(dirPath string) (bool, error) { - oldConfigPath := filepath.Join(dirPath, buflock.ExternalConfigFilePath) - oldConfigBytes, err := os.ReadFile(oldConfigPath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - // OK, no old config file - return false, nil - } - return false, fmt.Errorf("failed to read file: %w", err) - } - var versionedConfig buflock.ExternalConfigVersion - if err := encoding.UnmarshalYAMLNonStrict(oldConfigBytes, &versionedConfig); err != nil { - return false, fmt.Errorf( - "failed to read %s version: %w", - oldConfigPath, - err, - ) - } - switch versionedConfig.Version { - case buflock.V1Version: - // OK, file was already v1 - return false, nil - case buflock.V1Beta1Version, "": - // Continue to migrate - default: - return false, fmt.Errorf("unknown lock file version: %s", versionedConfig.Version) - } - var v1beta1LockFile buflock.ExternalConfigV1Beta1 - if err := encoding.UnmarshalYAMLStrict(oldConfigBytes, &v1beta1LockFile); err != nil { - return false, fmt.Errorf( - "failed to unmarshal %s as %s version v1beta1: %w", - oldConfigPath, - buflock.ExternalConfigFilePath, - err, - ) - } - v1LockFile := buflock.ExternalConfigV1{ - Version: buflock.V1Version, - } - for _, dependency := range v1beta1LockFile.Deps { - v1LockFile.Deps = append(v1LockFile.Deps, buflock.ExternalConfigDependencyV1(dependency)) - } - newConfigPath := filepath.Join(dirPath, buflock.ExternalConfigFilePath) - if err := m.writeV1LockFile(newConfigPath, v1LockFile); err != nil { - return false, err - } - return true, nil -} - -// writeV1LockFile atomically replaces the old lock file by first writing -// the new lock file to a temporary file and then moving it to the old lock file path. -// If we fail to marshal or write, the old lock file is not touched. -func (m *v1beta1Migrator) writeV1LockFile( - configPath string, - config buflock.ExternalConfigV1, -) (retErr error) { - v1ConfigData, err := encoding.MarshalYAML(&config) - if err != nil { - return fmt.Errorf("failed to marshal new lock file: %w", err) - } - v1ConfigData = append([]byte(buflock.Header), v1ConfigData...) - return os.WriteFile(configPath, v1ConfigData, 0600) -} - -func maybeReadLockFile(oldLockFilePath string) (buflock.ExternalConfigV1, bool, error) { - lockFileBytes, err := os.ReadFile(oldLockFilePath) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - // OK, no old lock file - return buflock.ExternalConfigV1{}, false, nil - } - - return buflock.ExternalConfigV1{}, false, fmt.Errorf("failed to read lock file path: %w", err) - } - var versionedConfig buflock.ExternalConfigVersion - if err := encoding.UnmarshalYAMLNonStrict(lockFileBytes, &versionedConfig); err != nil { - return buflock.ExternalConfigV1{}, false, fmt.Errorf( - "failed to read %s version: %w", - oldLockFilePath, - err, - ) - } - switch versionedConfig.Version { - case "", buflock.V1Beta1Version: - var externalConfig buflock.ExternalConfigV1Beta1 - if err := encoding.UnmarshalYAMLStrict(lockFileBytes, &externalConfig); err != nil { - return buflock.ExternalConfigV1{}, false, fmt.Errorf( - "failed to unmarshal lock file at %s: %w", - buflock.V1Beta1Version, - err, - ) - } - externalLockFileV1 := buflock.ExternalConfigV1{ - Version: buflock.V1Version, - } - for _, dependency := range externalConfig.Deps { - externalLockFileV1.Deps = append(externalLockFileV1.Deps, buflock.ExternalConfigDependencyV1(dependency)) - } - return externalLockFileV1, true, nil - case buflock.V1Version: - externalLockFileV1 := buflock.ExternalConfigV1{} - if err := encoding.UnmarshalYAMLStrict(lockFileBytes, &externalLockFileV1); err != nil { - return buflock.ExternalConfigV1{}, false, fmt.Errorf("failed to unmarshal lock file at %s: %w", buflock.V1Version, err) - } - return externalLockFileV1, true, nil - default: - return buflock.ExternalConfigV1{}, false, fmt.Errorf("unknown lock file version: %s", versionedConfig.Version) - } -} - -func convertIgnoreSlice(paths []string, dirPath string, root string, pathToProcessed map[string]bool) ([]string, error) { - var ignoresForRoot []string - for _, ignoredFile := range paths { - if _, ok := pathToProcessed[ignoredFile]; !ok { - pathToProcessed[ignoredFile] = false - } - filePath := filepath.Join(dirPath, root, ignoredFile) - if _, err := os.Stat(filePath); err != nil { - if errors.Is(err, os.ErrNotExist) { - continue - } - return nil, fmt.Errorf("failed to check for presence of file %s: %w", filePath, err) - } - pathToProcessed[ignoredFile] = true - ignoresForRoot = append(ignoresForRoot, ignoredFile) - } - sort.Strings(ignoresForRoot) - return ignoresForRoot, nil -} - -func convertIgnoreMap(ruleToIgnores map[string][]string, dirPath string, root string, pathToProcessed map[string]bool) (map[string][]string, error) { - var ruleToIgnoresForRoot map[string][]string - for rule, ignores := range ruleToIgnores { - for _, ignoredFile := range ignores { - if _, ok := pathToProcessed[ignoredFile]; !ok { - pathToProcessed[ignoredFile] = false - } - filePath := filepath.Join(dirPath, root, ignoredFile) - if _, err := os.Stat(filePath); err != nil { - if errors.Is(err, os.ErrNotExist) { - continue - } - return nil, fmt.Errorf("failed to check for presence of file %s: %w", filePath, err) - } - if ruleToIgnoresForRoot == nil { - ruleToIgnoresForRoot = make(map[string][]string) - } - pathToProcessed[ignoredFile] = true - ruleToIgnoresForRoot[rule] = append( - ruleToIgnoresForRoot[rule], - ignoredFile, - ) - } - sort.Strings(ruleToIgnoresForRoot[rule]) - } - return ruleToIgnoresForRoot, nil -} diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index 0978bab0c6..780a1ec915 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -142,7 +142,6 @@ func NewRootCommand(name string) *appcmd.Command { graph.NewCommand("graph", builder), price.NewCommand("price", builder), stats.NewCommand("stats", builder), - //migratev1beta1.NewCommand("migrate-v1beta1", builder), studioagent.NewCommand("studio-agent", builder), { Use: "registry", diff --git a/private/buf/cmd/buf/buf_test.go b/private/buf/cmd/buf/buf_test.go index 32312d11d4..434fdba310 100644 --- a/private/buf/cmd/buf/buf_test.go +++ b/private/buf/cmd/buf/buf_test.go @@ -16,7 +16,6 @@ package buf import ( "bytes" - "context" "fmt" "io" "os" @@ -29,8 +28,6 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/internal/internaltesting" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appcmd/appcmdtesting" - "github.com/bufbuild/buf/private/pkg/command" - "github.com/bufbuild/buf/private/pkg/storage" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/storage/storagetesting" "github.com/stretchr/testify/assert" @@ -1566,207 +1563,6 @@ func TestVersion(t *testing.T) { testRunStdout(t, nil, 0, bufcli.Version, "--version") } -func TestMigrateV1Beta1(t *testing.T) { - t.Parallel() - // TODO - t.Skip("TODO") - storageosProvider := storageos.NewProvider() - runner := command.NewRunner() - - // These test cases are ordered alphabetically to align with the folders in testadata. - t.Run("buf-gen-yaml-without-version", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "buf-gen-yaml-without-version", - "Successfully migrated your buf.gen.yaml to v1.", - ) - }) - t.Run("buf-yaml-without-version", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "buf-yaml-without-version", - "Successfully migrated your buf.yaml to v1.", - ) - }) - t.Run("complex", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "complex", - `The ignored file "file3.proto" was not found in any roots and has been removed. -Successfully migrated your buf.yaml and buf.gen.yaml to v1.`, - ) - }) - t.Run("deps-without-name", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "deps-without-name", - "Successfully migrated your buf.yaml to v1.", - ) - }) - t.Run("flat-deps-without-name", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "flat-deps-without-name", - "Successfully migrated your buf.yaml and buf.lock to v1.", - ) - }) - t.Run("lock-file-without-deps", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "lock-file-without-deps", - `Successfully migrated your buf.yaml and buf.lock to v1.`, - ) - }) - t.Run("nested-folder", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "nested-folder", - "Successfully migrated your buf.yaml to v1.", - ) - }) - t.Run("nested-root", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "nested-root", - "Successfully migrated your buf.yaml and buf.gen.yaml to v1.", - ) - }) - t.Run("no-deps", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "no-deps", - "Successfully migrated your buf.yaml and buf.gen.yaml to v1.", - ) - }) - t.Run("noop", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "noop", - "", - ) - }) - t.Run("only-buf-gen-yaml", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "only-buf-gen-yaml", - "Successfully migrated your buf.gen.yaml to v1.", - ) - }) - t.Run("only-buf-lock", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "only-buf-lock", - "Successfully migrated your buf.lock to v1.", - ) - }) - t.Run("only-buf-yaml", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "only-buf-yaml", - "Successfully migrated your buf.yaml to v1.", - ) - }) - t.Run("only-old-buf-gen-yaml", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "only-old-buf-gen-yaml", - "Successfully migrated your buf.gen.yaml to v1.", - ) - }) - t.Run("only-old-buf-lock", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "only-old-buf-lock", - `Successfully migrated your buf.lock to v1.`, - ) - }) - t.Run("only-old-buf-yaml", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "only-old-buf-yaml", - "Successfully migrated your buf.yaml to v1.", - ) - }) - t.Run("simple", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "simple", - "Successfully migrated your buf.yaml, buf.gen.yaml, and buf.lock to v1.", - ) - }) - t.Run("v1beta1-lock-file", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Diff( - t, - storageosProvider, - runner, - "v1beta1-lock-file", - `Successfully migrated your buf.yaml and buf.lock to v1.`, - ) - }) - - t.Run("fails-on-invalid-version", func(t *testing.T) { - t.Parallel() - testMigrateV1Beta1Failure( - t, - storageosProvider, - "invalid-version", - `failed to migrate config: unknown config file version: spaghetti`, - ) - }) -} - func TestConvertWithImage(t *testing.T) { t.Parallel() tempDir := t.TempDir() @@ -2508,55 +2304,6 @@ func TestConvertRoundTrip(t *testing.T) { }) } -func testMigrateV1Beta1Diff( - t *testing.T, - storageosProvider storageos.Provider, - runner command.Runner, - scenario string, - expectedStderr string, -) { - // Copy test setup to temporary directory to avoid writing to filesystem - inputBucket, err := storageosProvider.NewReadWriteBucket(filepath.Join("testdata", "migrate-v1beta1", "success", scenario, "input")) - require.NoError(t, err) - tempDir, readWriteBucket := internaltesting.CopyReadBucketToTempDir(context.Background(), t, storageosProvider, inputBucket) - - testRunStdoutStderrNoWarn( - t, - nil, - 0, - "", - expectedStderr, - "beta", - "migrate-v1beta1", - tempDir, - ) - - expectedOutputBucket, err := storageosProvider.NewReadWriteBucket(filepath.Join("testdata", "migrate-v1beta1", "success", scenario, "output")) - require.NoError(t, err) - - diff, err := storage.DiffBytes(context.Background(), runner, expectedOutputBucket, readWriteBucket) - require.NoError(t, err) - require.Empty(t, string(diff)) -} - -func testMigrateV1Beta1Failure(t *testing.T, storageosProvider storageos.Provider, scenario string, expectedStderr string) { - // Copy test setup to temporary directory to avoid writing to filesystem - inputBucket, err := storageosProvider.NewReadWriteBucket(filepath.Join("testdata", "migrate-v1beta1", "failure", scenario)) - require.NoError(t, err) - tempDir, _ := internaltesting.CopyReadBucketToTempDir(context.Background(), t, storageosProvider, inputBucket) - - testRunStdoutStderrNoWarn( - t, - nil, - 1, - "", - expectedStderr, - "beta", - "migrate-v1beta1", - tempDir, - ) -} - func testModInit(t *testing.T, expectedData string, document bool, name string, deps ...string) { tempDir := t.TempDir() baseArgs := []string{"mod", "init"} diff --git a/private/buf/cmd/buf/command/beta/migratev1beta1/migratev1beta1.go b/private/buf/cmd/buf/command/beta/migratev1beta1/migratev1beta1.go deleted file mode 100644 index ee00073789..0000000000 --- a/private/buf/cmd/buf/command/beta/migratev1beta1/migratev1beta1.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package migratev1beta1 - -import ( - "context" - - "github.com/bufbuild/buf/private/buf/bufmigrate" - "github.com/bufbuild/buf/private/pkg/app" - "github.com/bufbuild/buf/private/pkg/app/appcmd" - "github.com/bufbuild/buf/private/pkg/app/appext" - "github.com/spf13/pflag" -) - -// NewCommand returns a new Command. -func NewCommand( - name string, - builder appext.SubCommandBuilder, -) *appcmd.Command { - flags := newFlags() - return &appcmd.Command{ - Use: name + " ", - Short: `Migrate v1beta1 configuration to the latest version`, - Long: `Migrate any v1beta1 configuration files in the directory to the latest version. -Defaults to the current directory if not specified.`, - Args: appcmd.MaximumNArgs(1), - Run: builder.NewRunFunc( - func(ctx context.Context, container appext.Container) error { - return run(ctx, container, flags) - }, - ), - BindFlags: flags.Bind, - } -} - -type flags struct{} - -func newFlags() *flags { - return &flags{} -} - -func (f *flags) Bind(flagSet *pflag.FlagSet) {} - -func run( - ctx context.Context, - container appext.Container, - flags *flags, -) error { - dirPath, err := getDirPath(container) - if err != nil { - return err - } - return bufmigrate.NewV1Beta1Migrator( - "buf config migrate-v1beta1", - bufmigrate.V1Beta1MigratorWithNotifier(newWriteMessageFunc(container)), - ).Migrate(dirPath) -} - -func getDirPath(container app.Container) (string, error) { - switch numArgs := container.NumArgs(); numArgs { - case 0: - return ".", nil - case 1: - return container.Arg(0), nil - default: - return "", appcmd.NewInvalidArgumentErrorf("only 1 argument allowed but %d arguments specified", numArgs) - } -} - -func newWriteMessageFunc(container app.StderrContainer) func(string) error { - return func(message string) error { - _, err := container.Stderr().Write([]byte(message)) - return err - } -} diff --git a/private/buf/cmd/buf/command/beta/migratev1beta1/usage.gen.go b/private/buf/cmd/buf/command/beta/migratev1beta1/usage.gen.go deleted file mode 100644 index a3de0ecf16..0000000000 --- a/private/buf/cmd/buf/command/beta/migratev1beta1/usage.gen.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Generated. DO NOT EDIT. - -package migratev1beta1 - -import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/buf/cmd/buf/command/generate/generate.go b/private/buf/cmd/buf/command/generate/generate.go index dadd887841..1d12856415 100644 --- a/private/buf/cmd/buf/command/generate/generate.go +++ b/private/buf/cmd/buf/command/generate/generate.go @@ -35,6 +35,7 @@ import ( "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/storage/storageos" "github.com/bufbuild/buf/private/pkg/stringutil" + "github.com/bufbuild/buf/private/pkg/tracing" "github.com/spf13/pflag" "go.uber.org/zap" ) @@ -439,6 +440,7 @@ func run( } return bufgen.NewGenerator( logger, + tracing.NewTracer(container.Tracer()), storageosProvider, command.NewRunner(), wasmPluginExecutor, diff --git a/private/buf/cmd/buf/command/generate/generate_test.go b/private/buf/cmd/buf/command/generate/generate_test.go index 568d8fb231..7771e7c87f 100644 --- a/private/buf/cmd/buf/command/generate/generate_test.go +++ b/private/buf/cmd/buf/command/generate/generate_test.go @@ -522,7 +522,7 @@ func testRunStdoutStderr(t *testing.T, stdin io.Reader, expectedExitCode int, ex name, appext.NewBuilder( name, - appflag.BuilderWithInterceptor( + appext.BuilderWithInterceptor( // TODO: use the real interceptor. Currently in buf.go, NewBuilder receives appflag.BuilderWithInterceptor(newErrorInterceptor()). // However we cannot depend on newErrorInterceptor because it would create an import cycle, not to mention it needs to be exported first. // This can depend on newErroInterceptor when it's moved to a separate package and made public. diff --git a/private/bufpkg/bufconfig/generate_plugin_config.go b/private/bufpkg/bufconfig/generate_plugin_config.go index a3b84d1e63..f8b67aaa4e 100644 --- a/private/bufpkg/bufconfig/generate_plugin_config.go +++ b/private/bufpkg/bufconfig/generate_plugin_config.go @@ -21,8 +21,8 @@ import ( "os/exec" "strings" + "github.com/bufbuild/buf/private/buf/bufpluginexec" "github.com/bufbuild/buf/private/bufpkg/bufplugin/bufpluginref" - "github.com/bufbuild/buf/private/bufpkg/bufpluginexec" "github.com/bufbuild/buf/private/bufpkg/bufremoteplugin" "github.com/bufbuild/buf/private/pkg/encoding" "github.com/bufbuild/buf/private/pkg/syserror" diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go index 5c6a810b4a..a6c3d5c89d 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go @@ -25,6 +25,7 @@ import ( "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify/internal" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagetesting" "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduletesting" "github.com/bufbuild/buf/private/pkg/tracing" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" @@ -798,17 +799,17 @@ func testGetImageFromDirs( dirPathToModuleFullName map[string]string, includeSourceInfo bool, ) bufimage.Image { - moduleDatas := make([]bufmoduletest.ModuleData, 0, len(dirPathToModuleFullName)) + moduleDatas := make([]bufmoduletesting.ModuleData, 0, len(dirPathToModuleFullName)) for dirPath, moduleFullName := range dirPathToModuleFullName { moduleDatas = append( moduleDatas, - bufmoduletest.ModuleData{ + bufmoduletesting.ModuleData{ Name: moduleFullName, DirPath: dirPath, }, ) } - moduleSet, err := bufmoduletest.NewModuleSet(moduleDatas...) + moduleSet, err := bufmoduletesting.NewModuleSet(moduleDatas...) require.NoError(t, err) var options []bufimage.BuildImageOption if !includeSourceInfo { From c0d88c9a42d2bb00fa7381063c6dc57ba99e7e75 Mon Sep 17 00:00:00 2001 From: bufdev Date: Mon, 11 Dec 2023 12:38:33 -0500 Subject: [PATCH 66/66] move bufconfigtest functions --- .../bufconfig/bufconfigtest/bufconfigtest.go | 77 ---------- .../bufconfig/bufconfigtest/usage.gen.go | 19 --- .../bufimagemodify/bufimagemodify_test.go | 144 ++++++++++++------ 3 files changed, 99 insertions(+), 141 deletions(-) delete mode 100644 private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go delete mode 100644 private/bufpkg/bufconfig/bufconfigtest/usage.gen.go diff --git a/private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go b/private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go deleted file mode 100644 index d4ad333115..0000000000 --- a/private/bufpkg/bufconfig/bufconfigtest/bufconfigtest.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package bufconfigtest - -import ( - "testing" - - "github.com/bufbuild/buf/private/bufpkg/bufconfig" - "github.com/stretchr/testify/require" -) - -func NewTestManagedDisableRule( - t *testing.T, - path string, - moduleFullName string, - fieldName string, - fileOption bufconfig.FileOption, - fieldOption bufconfig.FieldOption, -) bufconfig.ManagedDisableRule { - disable, err := bufconfig.NewDisableRule( - path, - moduleFullName, - fieldName, - fileOption, - fieldOption, - ) - require.NoError(t, err) - return disable -} - -func NewTestFileOptionOverrideRule( - t *testing.T, - path string, - moduleFullName string, - fileOption bufconfig.FileOption, - value interface{}, -) bufconfig.ManagedOverrideRule { - fileOptionOverride, err := bufconfig.NewFileOptionOverrideRule( - path, - moduleFullName, - fileOption, - value, - ) - require.NoError(t, err) - return fileOptionOverride -} - -func NewTestFieldOptionOverrideRule( - t *testing.T, - path string, - moduleFullName string, - fieldName string, - fieldOption bufconfig.FieldOption, - value interface{}, -) bufconfig.ManagedOverrideRule { - fieldOptionOverrid, err := bufconfig.NewFieldOptionOverrideRule( - path, - moduleFullName, - bufconfig.FileOptionPhpMetadataNamespace.String(), - fieldOption, - value, - ) - require.NoError(t, err) - return fieldOptionOverrid -} diff --git a/private/bufpkg/bufconfig/bufconfigtest/usage.gen.go b/private/bufpkg/bufconfig/bufconfigtest/usage.gen.go deleted file mode 100644 index 7703e24a1c..0000000000 --- a/private/bufpkg/bufconfig/bufconfigtest/usage.gen.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2020-2023 Buf Technologies, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Generated. DO NOT EDIT. - -package bufconfigtest - -import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go index a6c3d5c89d..0b8dbc7d99 100644 --- a/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go +++ b/private/bufpkg/bufimage/bufimagemodify/bufimagemodify_test.go @@ -20,7 +20,6 @@ import ( "testing" "github.com/bufbuild/buf/private/bufpkg/bufconfig" - "github.com/bufbuild/buf/private/bufpkg/bufconfig/bufconfigtest" "github.com/bufbuild/buf/private/bufpkg/bufimage" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagemodify/internal" "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagetesting" @@ -200,7 +199,7 @@ func TestModifyImageFile( true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionCcEnableArenas, false), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionCcEnableArenas, false), }, ), modifyFunc: modifyCcEnableArenas, @@ -223,13 +222,13 @@ func TestModifyImageFile( config: bufconfig.NewGenerateManagedConfig( true, []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionCsharpNamespacePrefix, bufconfig.FieldOptionUnspecified), + newTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionCsharpNamespacePrefix, bufconfig.FieldOptionUnspecified), }, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespacePrefix, "BarPrefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespace, "BarValue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespace, "FooValue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespacePrefix, "FooPrefix"), + newTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespacePrefix, "BarPrefix"), + newTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionCsharpNamespace, "BarValue"), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespace, "FooValue"), + newTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionCsharpNamespacePrefix, "FooPrefix"), }, ), modifyFunc: modifyCsharpNamespace, @@ -260,13 +259,13 @@ func TestModifyImageFile( config: bufconfig.NewGenerateManagedConfig( true, []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionGoPackagePrefix, bufconfig.FieldOptionUnspecified), + newTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionGoPackagePrefix, bufconfig.FieldOptionUnspecified), }, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionGoPackagePrefix, "barprefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionGoPackage, "barvalue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty/with_package.proto", "buf.build/acme/foo", bufconfig.FileOptionGoPackage, "foovalue"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionGoPackagePrefix, "fooprefix"), + newTestFileOptionOverrideRule(t, "bar_empty", "buf.build/acme/bar", bufconfig.FileOptionGoPackagePrefix, "barprefix"), + newTestFileOptionOverrideRule(t, "bar_empty/without_package.proto", "buf.build/acme/bar", bufconfig.FileOptionGoPackage, "barvalue"), + newTestFileOptionOverrideRule(t, "foo_empty/with_package.proto", "buf.build/acme/foo", bufconfig.FileOptionGoPackage, "foovalue"), + newTestFileOptionOverrideRule(t, "foo_empty", "buf.build/acme/foo", bufconfig.FileOptionGoPackagePrefix, "fooprefix"), }, ), modifyFunc: modifyGoPackage, @@ -300,11 +299,11 @@ func TestModifyImageFile( config: bufconfig.NewGenerateManagedConfig( true, []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackagePrefix, bufconfig.FieldOptionUnspecified), + newTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackagePrefix, bufconfig.FieldOptionUnspecified), }, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackagePrefix, "barprefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackageSuffix, "foosuffix"), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackagePrefix, "barprefix"), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackageSuffix, "foosuffix"), }, ), modifyFunc: modifyJavaPackage, @@ -397,10 +396,10 @@ func TestModifyImageFile( config: bufconfig.NewGenerateManagedConfig( true, []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackageSuffix, bufconfig.FieldOptionUnspecified), + newTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackageSuffix, bufconfig.FieldOptionUnspecified), }, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), }, ), modifyFunc: modifyJavaPackage, @@ -452,11 +451,11 @@ func TestModifyImageFile( config: bufconfig.NewGenerateManagedConfig( true, []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackage, bufconfig.FieldOptionUnspecified), + newTestManagedDisableRule(t, "bar_empty", "", "", bufconfig.FileOptionJavaPackage, bufconfig.FieldOptionUnspecified), }, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackage, "bar.value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackage, "foo.value"), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionJavaPackage, "bar.value"), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionJavaPackage, "foo.value"), }, ), modifyFunc: modifyJavaPackage, @@ -506,12 +505,12 @@ func TestModifyImageFile( config: bufconfig.NewGenerateManagedConfig( true, []bufconfig.ManagedDisableRule{ - bufconfigtest.NewTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionObjcClassPrefix, bufconfig.FieldOptionUnspecified), + newTestManagedDisableRule(t, "foo_empty/with_package.proto", "", "", bufconfig.FileOptionObjcClassPrefix, bufconfig.FieldOptionUnspecified), }, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionObjcClassPrefix, "BAR"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOO"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "foo_all", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOOALL"), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/bar", bufconfig.FileOptionObjcClassPrefix, "BAR"), + newTestFileOptionOverrideRule(t, "", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOO"), + newTestFileOptionOverrideRule(t, "foo_all", "buf.build/acme/foo", bufconfig.FileOptionObjcClassPrefix, "FOOALL"), }, ), modifyFunc: modifyObjcClassPrefix, @@ -606,7 +605,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -618,7 +617,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -630,7 +629,7 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -642,8 +641,8 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -655,8 +654,8 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -668,8 +667,8 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -684,9 +683,9 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -701,9 +700,9 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -715,8 +714,8 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix2"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackagePrefix, "prefix2"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -728,8 +727,8 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix2"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackageSuffix, "suffix2"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -741,8 +740,8 @@ func TestGetStringOverrideFromConfig(t *testing.T) { true, []bufconfig.ManagedDisableRule{}, []bufconfig.ManagedOverrideRule{ - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), - bufconfigtest.NewTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value2"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value"), + newTestFileOptionOverrideRule(t, "", "", bufconfig.FileOptionJavaPackage, "value2"), }, ), imageFile: testGetImageFile(t, "a.proto", "buf.build/foo/bar"), @@ -825,3 +824,58 @@ func testGetImageFromDirs( require.Empty(t, annotations) return image } + +func newTestManagedDisableRule( + t *testing.T, + path string, + moduleFullName string, + fieldName string, + fileOption bufconfig.FileOption, + fieldOption bufconfig.FieldOption, +) bufconfig.ManagedDisableRule { + disable, err := bufconfig.NewDisableRule( + path, + moduleFullName, + fieldName, + fileOption, + fieldOption, + ) + require.NoError(t, err) + return disable +} + +func newTestFileOptionOverrideRule( + t *testing.T, + path string, + moduleFullName string, + fileOption bufconfig.FileOption, + value interface{}, +) bufconfig.ManagedOverrideRule { + fileOptionOverride, err := bufconfig.NewFileOptionOverrideRule( + path, + moduleFullName, + fileOption, + value, + ) + require.NoError(t, err) + return fileOptionOverride +} + +func newTestFieldOptionOverrideRule( + t *testing.T, + path string, + moduleFullName string, + fieldName string, + fieldOption bufconfig.FieldOption, + value interface{}, +) bufconfig.ManagedOverrideRule { + fieldOptionOverrid, err := bufconfig.NewFieldOptionOverrideRule( + path, + moduleFullName, + bufconfig.FileOptionPhpMetadataNamespace.String(), + fieldOption, + value, + ) + require.NoError(t, err) + return fieldOptionOverrid +}