From 7e972a296c0651a7ddc0bf0ff2fcfed66a12b419 Mon Sep 17 00:00:00 2001 From: Vicent Marti Date: Mon, 16 Sep 2024 10:05:41 +0200 Subject: [PATCH 1/2] grpc: upgrade to 1.66.2 and use Codec v2 Signed-off-by: Vicent Marti --- go.mod | 14 ++++---- go.sum | 14 ++++++++ go/vt/servenv/grpc_codec.go | 70 ++++++++++++++++++++++--------------- 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index ec3eef4fe32..c022a9f2eb4 100644 --- a/go.mod +++ b/go.mod @@ -68,18 +68,18 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.15 go.etcd.io/etcd/client/v3 v3.5.15 go.uber.org/mock v0.2.0 - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.27.0 // indirect golang.org/x/mod v0.20.0 // indirect - golang.org/x/net v0.28.0 + golang.org/x/net v0.29.0 golang.org/x/oauth2 v0.22.0 - golang.org/x/sys v0.24.0 - golang.org/x/term v0.23.0 - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.25.0 + golang.org/x/term v0.24.0 + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.6.0 golang.org/x/tools v0.24.0 google.golang.org/api v0.195.0 google.golang.org/genproto v0.0.0-20240827150818-7e3bb234dfed // indirect - google.golang.org/grpc v1.66.0 + google.golang.org/grpc v1.66.2 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/grpc/examples v0.0.0-20210430044426-28078834f35b google.golang.org/protobuf v1.34.2 @@ -186,7 +186,7 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index be825d9ab8a..b33c9943c3f 100644 --- a/go.sum +++ b/go.sum @@ -570,6 +570,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= @@ -606,6 +608,8 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= @@ -658,15 +662,21 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -706,6 +716,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1: google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed h1:J6izYgfBXAI3xTKLgxzTmUltdYaLsuBxFCgDHWJ/eXg= google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -715,6 +727,8 @@ google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/grpc/examples v0.0.0-20210430044426-28078834f35b h1:D/GTYPo6I1oEo08Bfpuj3xl5XE+UGHj7//5fVyKxhsQ= diff --git a/go/vt/servenv/grpc_codec.go b/go/vt/servenv/grpc_codec.go index 7d2b6364d3b..0fae44ebfe8 100644 --- a/go/vt/servenv/grpc_codec.go +++ b/go/vt/servenv/grpc_codec.go @@ -17,52 +17,64 @@ limitations under the License. package servenv import ( - "fmt" - - // use the original golang/protobuf package we can continue serializing - // messages from our dependencies, particularly from etcd - "github.com/golang/protobuf/proto" //nolint - "google.golang.org/grpc/encoding" + "google.golang.org/grpc/mem" + + // Guarantee that the built-in proto is called registered before this one + // so that it can be replaced. _ "google.golang.org/grpc/encoding/proto" // nolint:revive ) // Name is the name registered for the proto compressor. const Name = "proto" -type vtprotoCodec struct{} - type vtprotoMessage interface { - MarshalVT() ([]byte, error) + MarshalToSizedBufferVT(data []byte) (int, error) UnmarshalVT([]byte) error + SizeVT() int } -func (vtprotoCodec) Marshal(v any) ([]byte, error) { - switch v := v.(type) { - case vtprotoMessage: - return v.MarshalVT() - case proto.Message: - return proto.Marshal(v) - default: - return nil, fmt.Errorf("failed to marshal, message is %T, must satisfy the vtprotoMessage interface or want proto.Message", v) - } +type Codec struct { + fallback encoding.CodecV2 } -func (vtprotoCodec) Unmarshal(data []byte, v any) error { - switch v := v.(type) { - case vtprotoMessage: - return v.UnmarshalVT(data) - case proto.Message: - return proto.Unmarshal(data, v) - default: - return fmt.Errorf("failed to unmarshal, message is %T, must satisfy the vtprotoMessage interface or want proto.Message", v) +func (Codec) Name() string { return Name } + +var defaultBufferPool = mem.DefaultBufferPool() + +func (c *Codec) Marshal(v any) (mem.BufferSlice, error) { + if m, ok := v.(vtprotoMessage); ok { + size := m.SizeVT() + if mem.IsBelowBufferPoolingThreshold(size) { + buf := make([]byte, 0, size) + if _, err := m.MarshalToSizedBufferVT(buf[:size]); err != nil { + return nil, err + } + return mem.BufferSlice{mem.SliceBuffer(buf)}, nil + } + buf := defaultBufferPool.Get(size) + if _, err := m.MarshalToSizedBufferVT((*buf)[:size]); err != nil { + defaultBufferPool.Put(buf) + return nil, err + } + return mem.BufferSlice{mem.NewBuffer(buf, defaultBufferPool)}, nil } + + return c.fallback.Marshal(v) } -func (vtprotoCodec) Name() string { - return Name +func (c *Codec) Unmarshal(data mem.BufferSlice, v any) error { + if m, ok := v.(vtprotoMessage); ok { + buf := data.MaterializeToBuffer(defaultBufferPool) + defer buf.Free() + return m.UnmarshalVT(buf.ReadOnlyData()) + } + + return c.fallback.Unmarshal(data, v) } func init() { - encoding.RegisterCodec(vtprotoCodec{}) + encoding.RegisterCodecV2(&Codec{ + fallback: encoding.GetCodecV2("proto"), + }) } From fedf86784d6510a1837aa4d0f96817edab0b1fb8 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 16 Oct 2024 13:49:16 -0400 Subject: [PATCH 2/2] Set len of byte slice rather than cap Signed-off-by: Matt Lord --- go/vt/servenv/grpc_codec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/servenv/grpc_codec.go b/go/vt/servenv/grpc_codec.go index 0fae44ebfe8..35441feb261 100644 --- a/go/vt/servenv/grpc_codec.go +++ b/go/vt/servenv/grpc_codec.go @@ -46,7 +46,7 @@ func (c *Codec) Marshal(v any) (mem.BufferSlice, error) { if m, ok := v.(vtprotoMessage); ok { size := m.SizeVT() if mem.IsBelowBufferPoolingThreshold(size) { - buf := make([]byte, 0, size) + buf := make([]byte, size) if _, err := m.MarshalToSizedBufferVT(buf[:size]); err != nil { return nil, err }