From ffd018eba67e250756a5c3267773d8ee72e1558e Mon Sep 17 00:00:00 2001 From: axl Date: Sun, 16 Jun 2024 06:56:04 +0500 Subject: [PATCH 1/5] Added command for building json --- README.md | 40 +++++++- cmd/cmd_build_test.go | 39 ++++++++ cmd/cmd_buld.go | 127 ++++++++++++++++++++++++++ cmd/cmd_root.go | 1 + internal/proto/testdata/example.proto | 19 +++- 5 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 cmd/cmd_build_test.go create mode 100644 cmd/cmd_buld.go diff --git a/README.md b/README.md index 56b87c2..b9b650e 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ $ protokaf produce -h ### Examples This proto file will be used in the examples below. -`api/example.ptoto` +`api/example.proto` ```protobuf syntax = "proto3"; @@ -127,6 +127,44 @@ $ protokaf produce HelloRequest -t test -d '{"name": "Alice", "age": 11}' $ protokaf produce --template-functions-print ``` +## Build json template by proto file +This can be useful for creating body for produce command +```sh +$ protokaf build HelloRequest --proto internal/proto/testdata/example.proto +``` +For proto file +```protobuf +syntax = "proto3"; + +package example; + +message HelloRequest { + enum Status { + PENDING = 0; + COMPLETED = 1; + } + + string name = 1; + int32 age = 2; + optional float amount = 4; + Status status = 5; + repeated string keys = 6; +} +``` +command will print +```json +{ + "name": "", + "age": 0, + "amount": 0, + "status": "PENDING", + "keys": [ + "" + ] +} + +``` + ## Consume ### Help ```sh diff --git a/cmd/cmd_build_test.go b/cmd/cmd_build_test.go new file mode 100644 index 0000000..2b90e13 --- /dev/null +++ b/cmd/cmd_build_test.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_NewBuildCmd(t *testing.T) { + cmd := NewBuildCmd() + NewFlags(cmd).Init() + cmd.SetArgs([]string{"HelloRequest", "--proto", "../internal/proto/testdata/example.proto"}) + + expected := `{ + "name": "", + "age": 0, + "amount": 0, + "status": "PENDING", + "numbers": [ + { + "num": 0 + } + ], + "data": { + "timestamp": "1970-01-01T00:00:00Z", + "completed": false, + "properties": { + "0": "" + }, + "blob": "" + } +}` + + stdout, stderr, err := getCommandOut(t, cmd) + + require.Nil(t, err) + require.Equal(t, "", stderr) + require.Contains(t, stdout, expected) +} diff --git a/cmd/cmd_buld.go b/cmd/cmd_buld.go new file mode 100644 index 0000000..eaa6f35 --- /dev/null +++ b/cmd/cmd_buld.go @@ -0,0 +1,127 @@ +package cmd + +import ( + "fmt" + + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/protoc-gen-go/descriptor" + "github.com/jhump/protoreflect/desc" + "github.com/jhump/protoreflect/dynamic" + "github.com/spf13/cobra" + "google.golang.org/protobuf/types/descriptorpb" +) + +func NewBuildCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "build ", + Short: "Build json by proto message", + Args: messageNameRequired, + RunE: func(cmd *cobra.Command, args []string) (err error) { + // parse protofiles & create proto object + p, err := parseProtofiles() + if err != nil { + return + } + + messageDescriptor, err := findMessage(p, args[0]) + if err != nil { + return + } + + m := createMessage( + *dynamic.NewMessageFactoryWithDefaults(), + messageDescriptor, + ) + + b, err := m.MarshalJSONPB(&jsonpb.Marshaler{ + EmitDefaults: true, + Indent: " ", + }) + if err != nil { + return + } + + cmd.Println(string(b)) + + return + }, + } + + return cmd +} + +func createMessage(f dynamic.MessageFactory, md *desc.MessageDescriptor) *dynamic.Message { + m := f.NewDynamicMessage(md) + + for _, v := range m.GetKnownFields() { + if v.IsProto3Optional() { + if v.IsRepeated() { + if mt := v.GetMessageType(); mt != nil { + m.AddRepeatedField(v, createMessage(f, mt)) + continue + } + } + + if v.GetType() == descriptor.FieldDescriptorProto_TYPE_MESSAGE { + m.SetField(v, createMessage(f, v.GetMessageType())) + continue + } + + m.SetField(v, v.GetDefaultValue()) + } + + if v.IsRepeated() { + if mt := v.GetMessageType(); mt != nil { + m.AddRepeatedField(v, createMessage(f, mt)) + } else { + switch v.GetType() { + case descriptorpb.FieldDescriptorProto_TYPE_FIXED32, + descriptorpb.FieldDescriptorProto_TYPE_UINT32: + m.AddRepeatedField(v, uint32(0)) + case descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, + descriptorpb.FieldDescriptorProto_TYPE_INT32, + descriptorpb.FieldDescriptorProto_TYPE_SINT32: + m.AddRepeatedField(v, int32(0)) + case descriptorpb.FieldDescriptorProto_TYPE_FIXED64, + descriptorpb.FieldDescriptorProto_TYPE_UINT64: + m.AddRepeatedField(v, uint64(0)) + case descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, + descriptorpb.FieldDescriptorProto_TYPE_INT64, + descriptorpb.FieldDescriptorProto_TYPE_SINT64: + m.AddRepeatedField(v, int64(0)) + case descriptorpb.FieldDescriptorProto_TYPE_FLOAT: + m.AddRepeatedField(v, float32(0)) + case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE: + m.AddRepeatedField(v, float64(0)) + case descriptorpb.FieldDescriptorProto_TYPE_BOOL: + m.AddRepeatedField(v, false) + case descriptorpb.FieldDescriptorProto_TYPE_BYTES: + m.AddRepeatedField(v, []byte(nil)) + case descriptorpb.FieldDescriptorProto_TYPE_STRING: + m.AddRepeatedField(v, "") + case descriptorpb.FieldDescriptorProto_TYPE_ENUM: + if v.IsProto3Optional() { + m.AddRepeatedField(v, 0) + } else { + enumVals := v.GetEnumType().GetValues() + if len(enumVals) > 0 { + m.AddRepeatedField(v, enumVals[0].GetNumber()) + } else { + m.AddRepeatedField(v, 0) + } + } + default: + panic(fmt.Sprintf("Unknown field type: %v", v.GetType())) + } + } + continue + } + + if v.GetType() == descriptor.FieldDescriptorProto_TYPE_MESSAGE { + m.SetField(v, createMessage(f, v.GetMessageType())) + continue + } + } + + return m +} diff --git a/cmd/cmd_root.go b/cmd/cmd_root.go index 7bb4b7e..4b1502b 100644 --- a/cmd/cmd_root.go +++ b/cmd/cmd_root.go @@ -85,6 +85,7 @@ func NewRootCmd() *cobra.Command { NewProduceCmd(), NewConsumeCmd(), NewListCmd(), + NewBuildCmd(), ) return cmd diff --git a/internal/proto/testdata/example.proto b/internal/proto/testdata/example.proto index df3257b..0cc3b68 100644 --- a/internal/proto/testdata/example.proto +++ b/internal/proto/testdata/example.proto @@ -11,15 +11,32 @@ service Example { } message HelloRequest { + enum Status { + PENDING = 0; + COMPLETED = 1; + } + + message Data { + google.protobuf.Timestamp timestamp = 1; + bool completed = 2; + map properties = 3; + bytes blob = 4; + } + string name = 1; int32 age = 2; + float amount = 4; + reserved 3; + Status status = 5; + repeated Numbers numbers = 6; + optional Data data = 7; } message HelloResponse { string answer = 1; } -message Num { +message Numbers { int32 num = 1; } From 152007359a948c36f577733f7cb49c204380c541 Mon Sep 17 00:00:00 2001 From: axl Date: Thu, 4 Jul 2024 04:23:04 +0500 Subject: [PATCH 2/5] Added command for building json --- cmd/cmd_build_test.go | 70 +++++++++++---- cmd/cmd_buld.go | 135 ++++++++++++---------------- internal/proto/testdata/types.proto | 80 +++++++++++++++++ 3 files changed, 193 insertions(+), 92 deletions(-) create mode 100644 internal/proto/testdata/types.proto diff --git a/cmd/cmd_build_test.go b/cmd/cmd_build_test.go index 2b90e13..27bce1a 100644 --- a/cmd/cmd_build_test.go +++ b/cmd/cmd_build_test.go @@ -9,26 +9,64 @@ import ( func Test_NewBuildCmd(t *testing.T) { cmd := NewBuildCmd() NewFlags(cmd).Init() - cmd.SetArgs([]string{"HelloRequest", "--proto", "../internal/proto/testdata/example.proto"}) + cmd.SetArgs([]string{"ExampleMessage", "--proto", "../internal/proto/testdata/types.proto"}) expected := `{ - "name": "", - "age": 0, - "amount": 0, - "status": "PENDING", - "numbers": [ - { - "num": 0 + "int32Field": 0, + "int64Field": "0", + "uint32Field": 0, + "uint64Field": "0", + "sint32Field": 0, + "sint64Field": "0", + "fixed32Field": 0, + "fixed64Field": "0", + "sfixed32Field": 0, + "sfixed64Field": "0", + "floatField": 0, + "doubleField": 0, + "boolField": true, + "stringField": "", + "bytesField": "", + "enumField": "UNKNOWN", + "messageField": { + "nestedInt32": 0, + "nestedString": "" + }, + "repeatedInt32Field": [ + 0 + ], + "repeatedStringField": [ + "" + ], + "mapStringInt32Field": { + "": 0 + }, + "mapInt32MessageField": { + "0": { + "nestedInt32": 0, + "nestedString": "" } + }, + "option1": 0, + "anyField": null, + "timestampField": "1970-01-01T00:00:00Z", + "durationField": "0s", + "structField": { + "": null + }, + "valueField": null, + "listValueField": [ + null ], - "data": { - "timestamp": "1970-01-01T00:00:00Z", - "completed": false, - "properties": { - "0": "" - }, - "blob": "" - } + "boolValueField": true, + "bytesValueField": null, + "doubleValueField": 0, + "floatValueField": 0, + "int32ValueField": 0, + "int64ValueField": "0", + "stringValueField": "", + "uint32ValueField": 0, + "uint64ValueField": "0" }` stdout, stderr, err := getCommandOut(t, cmd) diff --git a/cmd/cmd_buld.go b/cmd/cmd_buld.go index eaa6f35..b45898c 100644 --- a/cmd/cmd_buld.go +++ b/cmd/cmd_buld.go @@ -1,14 +1,12 @@ package cmd import ( - "fmt" - "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/protoc-gen-go/descriptor" "github.com/jhump/protoreflect/desc" "github.com/jhump/protoreflect/dynamic" "github.com/spf13/cobra" "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/known/anypb" ) func NewBuildCmd() *cobra.Command { @@ -28,12 +26,9 @@ func NewBuildCmd() *cobra.Command { return } - m := createMessage( - *dynamic.NewMessageFactoryWithDefaults(), - messageDescriptor, - ) + msg := buildMessage(dynamic.NewMessage(messageDescriptor)) - b, err := m.MarshalJSONPB(&jsonpb.Marshaler{ + b, err := msg.MarshalJSONPB(&jsonpb.Marshaler{ EmitDefaults: true, Indent: " ", }) @@ -50,78 +45,66 @@ func NewBuildCmd() *cobra.Command { return cmd } -func createMessage(f dynamic.MessageFactory, md *desc.MessageDescriptor) *dynamic.Message { - m := f.NewDynamicMessage(md) - - for _, v := range m.GetKnownFields() { - if v.IsProto3Optional() { - if v.IsRepeated() { - if mt := v.GetMessageType(); mt != nil { - m.AddRepeatedField(v, createMessage(f, mt)) - continue - } - } - - if v.GetType() == descriptor.FieldDescriptorProto_TYPE_MESSAGE { - m.SetField(v, createMessage(f, v.GetMessageType())) - continue - } - - m.SetField(v, v.GetDefaultValue()) +func buildMessage(message *dynamic.Message) *dynamic.Message { + for _, field := range message.GetMessageDescriptor().GetFields() { + if field.IsRepeated() { + message.SetField(field, []interface{}{buildDefaultValue(field)}) + } else if field.IsMap() { + message.SetField( + field, + map[interface{}]interface{}{ + buildDefaultValue(field.GetMapKeyType()): buildDefaultValue(field.GetMapValueType()), + }, + ) + } else if field.GetOneOf() != nil { + oneOfField := field.GetOneOf().GetChoices()[0] + //fmt.Println(message.GetField(oneOfFields[0])) + message.SetField(oneOfField, buildDefaultValue(oneOfField)) + } else { + message.SetField(field, buildDefaultValue(field)) } + } - if v.IsRepeated() { - if mt := v.GetMessageType(); mt != nil { - m.AddRepeatedField(v, createMessage(f, mt)) - } else { - switch v.GetType() { - case descriptorpb.FieldDescriptorProto_TYPE_FIXED32, - descriptorpb.FieldDescriptorProto_TYPE_UINT32: - m.AddRepeatedField(v, uint32(0)) - case descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, - descriptorpb.FieldDescriptorProto_TYPE_INT32, - descriptorpb.FieldDescriptorProto_TYPE_SINT32: - m.AddRepeatedField(v, int32(0)) - case descriptorpb.FieldDescriptorProto_TYPE_FIXED64, - descriptorpb.FieldDescriptorProto_TYPE_UINT64: - m.AddRepeatedField(v, uint64(0)) - case descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, - descriptorpb.FieldDescriptorProto_TYPE_INT64, - descriptorpb.FieldDescriptorProto_TYPE_SINT64: - m.AddRepeatedField(v, int64(0)) - case descriptorpb.FieldDescriptorProto_TYPE_FLOAT: - m.AddRepeatedField(v, float32(0)) - case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE: - m.AddRepeatedField(v, float64(0)) - case descriptorpb.FieldDescriptorProto_TYPE_BOOL: - m.AddRepeatedField(v, false) - case descriptorpb.FieldDescriptorProto_TYPE_BYTES: - m.AddRepeatedField(v, []byte(nil)) - case descriptorpb.FieldDescriptorProto_TYPE_STRING: - m.AddRepeatedField(v, "") - case descriptorpb.FieldDescriptorProto_TYPE_ENUM: - if v.IsProto3Optional() { - m.AddRepeatedField(v, 0) - } else { - enumVals := v.GetEnumType().GetValues() - if len(enumVals) > 0 { - m.AddRepeatedField(v, enumVals[0].GetNumber()) - } else { - m.AddRepeatedField(v, 0) - } - } - default: - panic(fmt.Sprintf("Unknown field type: %v", v.GetType())) - } - } - continue - } + return message +} - if v.GetType() == descriptor.FieldDescriptorProto_TYPE_MESSAGE { - m.SetField(v, createMessage(f, v.GetMessageType())) - continue +func buildDefaultValue(field *desc.FieldDescriptor) interface{} { + switch field.GetType() { + case descriptorpb.FieldDescriptorProto_TYPE_FIXED32, + descriptorpb.FieldDescriptorProto_TYPE_UINT32: + return uint32(0) + case descriptorpb.FieldDescriptorProto_TYPE_SFIXED32, + descriptorpb.FieldDescriptorProto_TYPE_INT32, + descriptorpb.FieldDescriptorProto_TYPE_SINT32: + return int32(0) + case descriptorpb.FieldDescriptorProto_TYPE_FIXED64, + descriptorpb.FieldDescriptorProto_TYPE_UINT64: + return uint64(0) + case descriptorpb.FieldDescriptorProto_TYPE_SFIXED64, + descriptorpb.FieldDescriptorProto_TYPE_INT64, + descriptorpb.FieldDescriptorProto_TYPE_SINT64: + return int64(0) + case descriptorpb.FieldDescriptorProto_TYPE_FLOAT: + return float32(0) + case descriptorpb.FieldDescriptorProto_TYPE_DOUBLE: + return float64(0) + case descriptorpb.FieldDescriptorProto_TYPE_BOOL: + return true + case descriptorpb.FieldDescriptorProto_TYPE_BYTES: + return []byte(nil) + case descriptorpb.FieldDescriptorProto_TYPE_STRING: + return "" + case descriptorpb.FieldDescriptorProto_TYPE_ENUM: + return field.GetEnumType().GetValues()[0].GetNumber() + case descriptorpb.FieldDescriptorProto_TYPE_MESSAGE: + if field.GetMessageType().GetFullyQualifiedName() == "google.protobuf.Any" { + val, _ := anypb.New(nil) + + return val } + + return buildMessage(dynamic.NewMessage(field.GetMessageType())) } - return m + return nil } diff --git a/internal/proto/testdata/types.proto b/internal/proto/testdata/types.proto new file mode 100644 index 0000000..1fbe3f3 --- /dev/null +++ b/internal/proto/testdata/types.proto @@ -0,0 +1,80 @@ +syntax = "proto3"; + +package example; + +import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +// Enum definition +enum ExampleEnum { + UNKNOWN = 0; + OPTION_ONE = 1; + OPTION_TWO = 2; +} + +// Message definition +message ExampleMessage { + // Scalar types + int32 int32_field = 1; + int64 int64_field = 2; + uint32 uint32_field = 3; + uint64 uint64_field = 4; + sint32 sint32_field = 5; + sint64 sint64_field = 6; + fixed32 fixed32_field = 7; + fixed64 fixed64_field = 8; + sfixed32 sfixed32_field = 9; + sfixed64 sfixed64_field = 10; + float float_field = 11; + double double_field = 12; + bool bool_field = 13; + string string_field = 14; + bytes bytes_field = 15; + + // Enum type + ExampleEnum enum_field = 16; + + // Nested message + message NestedMessage { + int32 nested_int32 = 1; + string nested_string = 2; + } + + // Message type + NestedMessage message_field = 17; + + // Repeated fields + repeated int32 repeated_int32_field = 18; + repeated string repeated_string_field = 19; + + // Map fields + map map_string_int32_field = 20; + map map_int32_message_field = 21; + + // Oneof field + oneof my_oneof { + int32 option1 = 22; + string option2 = 23; + NestedMessage option3 = 24; + } + + // Well-known types + google.protobuf.Any any_field = 25; + google.protobuf.Timestamp timestamp_field = 26; + google.protobuf.Duration duration_field = 27; + google.protobuf.Struct struct_field = 28; + google.protobuf.Value value_field = 29; + google.protobuf.ListValue list_value_field = 30; + google.protobuf.BoolValue bool_value_field = 31; + google.protobuf.BytesValue bytes_value_field = 32; + google.protobuf.DoubleValue double_value_field = 33; + google.protobuf.FloatValue float_value_field = 34; + google.protobuf.Int32Value int32_value_field = 35; + google.protobuf.Int64Value int64_value_field = 36; + google.protobuf.StringValue string_value_field = 37; + google.protobuf.UInt32Value uint32_value_field = 38; + google.protobuf.UInt64Value uint64_value_field = 39; +} From 80f5e8f38f553258aec7b4a22571fd0816825757 Mon Sep 17 00:00:00 2001 From: axl Date: Thu, 4 Jul 2024 04:26:50 +0500 Subject: [PATCH 3/5] Updates --- cmd/cmd_buld.go | 1 - internal/proto/testdata/example.proto | 19 +------------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/cmd/cmd_buld.go b/cmd/cmd_buld.go index b45898c..6ca5258 100644 --- a/cmd/cmd_buld.go +++ b/cmd/cmd_buld.go @@ -58,7 +58,6 @@ func buildMessage(message *dynamic.Message) *dynamic.Message { ) } else if field.GetOneOf() != nil { oneOfField := field.GetOneOf().GetChoices()[0] - //fmt.Println(message.GetField(oneOfFields[0])) message.SetField(oneOfField, buildDefaultValue(oneOfField)) } else { message.SetField(field, buildDefaultValue(field)) diff --git a/internal/proto/testdata/example.proto b/internal/proto/testdata/example.proto index 0cc3b68..df3257b 100644 --- a/internal/proto/testdata/example.proto +++ b/internal/proto/testdata/example.proto @@ -11,32 +11,15 @@ service Example { } message HelloRequest { - enum Status { - PENDING = 0; - COMPLETED = 1; - } - - message Data { - google.protobuf.Timestamp timestamp = 1; - bool completed = 2; - map properties = 3; - bytes blob = 4; - } - string name = 1; int32 age = 2; - float amount = 4; - reserved 3; - Status status = 5; - repeated Numbers numbers = 6; - optional Data data = 7; } message HelloResponse { string answer = 1; } -message Numbers { +message Num { int32 num = 1; } From 1adc9dfa1ae986655018e561b81c196d569bd304 Mon Sep 17 00:00:00 2001 From: axl Date: Thu, 4 Jul 2024 20:06:21 +0500 Subject: [PATCH 4/5] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2f5b905..ff46216 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/uber/jaeger-client-go v2.29.1+incompatible github.com/xdg/scram v1.0.3 go.uber.org/zap v1.18.1 + google.golang.org/protobuf v1.33.0 ) require ( @@ -55,7 +56,6 @@ require ( golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect - google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect From 4a05b9ccb2174fec16fcd6ad9013bb3a688e9e38 Mon Sep 17 00:00:00 2001 From: axl1232 Date: Thu, 4 Jul 2024 18:47:51 +0300 Subject: [PATCH 5/5] Update cmd/cmd_buld.go Co-authored-by: Lev Zakharov --- cmd/cmd_buld.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/cmd_buld.go b/cmd/cmd_buld.go index 6ca5258..b894484 100644 --- a/cmd/cmd_buld.go +++ b/cmd/cmd_buld.go @@ -47,19 +47,20 @@ func NewBuildCmd() *cobra.Command { func buildMessage(message *dynamic.Message) *dynamic.Message { for _, field := range message.GetMessageDescriptor().GetFields() { - if field.IsRepeated() { + switch { + case field.IsRepeated(): message.SetField(field, []interface{}{buildDefaultValue(field)}) - } else if field.IsMap() { + case field.IsMap(): message.SetField( field, map[interface{}]interface{}{ buildDefaultValue(field.GetMapKeyType()): buildDefaultValue(field.GetMapValueType()), }, ) - } else if field.GetOneOf() != nil { + case field.GetOneOf() != nil: oneOfField := field.GetOneOf().GetChoices()[0] message.SetField(oneOfField, buildDefaultValue(oneOfField)) - } else { + default: message.SetField(field, buildDefaultValue(field)) } }