From 128abdb5feda2411f2d2d706cc569222cccb8579 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Mon, 30 Sep 2024 23:59:36 -0700 Subject: [PATCH] Add native json.Number support (#364) Allow encoding and decoding of `json.Number` values, either as struct members or as interface members. Numbers will be encoded as integer, if possible, otherwise float64/float32 is used. The zero value json.Number will be encoded as 0. It is possible to encode as string with `//msgp:replace json.Number with:string`. Fixes #292 --- _generated/def.go | 8 +++ _generated/def_test.go | 105 +++++++++++++++++++++++++++++++++++++ _generated/replace.go | 10 ++++ _generated/replace_test.go | 73 ++++++++++++++++++++++++++ gen/elem.go | 20 +++++-- gen/marshal.go | 2 +- msgp/read.go | 34 ++++++++++++ msgp/read_bytes.go | 23 ++++++++ msgp/size.go | 9 ++-- msgp/write.go | 20 +++++++ msgp/write_bytes.go | 21 ++++++++ 11 files changed, 315 insertions(+), 10 deletions(-) diff --git a/_generated/def.go b/_generated/def.go index a3d97b9e..dc3ec3f4 100644 --- a/_generated/def.go +++ b/_generated/def.go @@ -1,6 +1,7 @@ package _generated import ( + "encoding/json" "os" "time" @@ -299,3 +300,10 @@ type StructByteSlice struct { AComplex128 []complex128 `msg:",allownil"` AStruct []Fixed `msg:",allownil"` } + +type NumberJSONSample struct { + Single json.Number + Array []json.Number + Map map[string]json.Number + OE json.Number `msg:",omitempty"` +} diff --git a/_generated/def_test.go b/_generated/def_test.go index 5d2e80ff..ad2b5fc9 100644 --- a/_generated/def_test.go +++ b/_generated/def_test.go @@ -2,6 +2,7 @@ package _generated import ( "bytes" + "encoding/json" "reflect" "testing" @@ -74,3 +75,107 @@ func TestRuneMarshalUnmarshal(t *testing.T) { t.Errorf("rune slice mismatch") } } + +func TestJSONNumber(t *testing.T) { + test := NumberJSONSample{ + Single: "-42", + Array: []json.Number{"0", "-0", "1", "-1", "0.1", "-0.1", "1234", "-1234", "12.34", "-12.34", "12E0", "12E1", "12e34", "12E-0", "12e+1", "12e-34", "-12E0", "-12E1", "-12e34", "-12E-0", "-12e+1", "-12e-34", "1.2E0", "1.2E1", "1.2e34", "1.2E-0", "1.2e+1", "1.2e-34", "-1.2E0", "-1.2E1", "-1.2e34", "-1.2E-0", "-1.2e+1", "-1.2e-34", "0E0", "0E1", "0e34", "0E-0", "0e+1", "0e-34", "-0E0", "-0E1", "-0e34", "-0E-0", "-0e+1", "-0e-34"}, + Map: map[string]json.Number{ + "a": json.Number("50.2"), + }, + } + + // This is not guaranteed to be symmetric + encoded, err := test.MarshalMsg(nil) + if err != nil { + t.Errorf("%v", err) + } + var v NumberJSONSample + _, err = v.UnmarshalMsg(encoded) + if err != nil { + t.Errorf("%v", err) + } + // Test two values + if v.Single != "-42" { + t.Errorf("want %v, got %v", "-42", v.Single) + } + if v.Map["a"] != "50.2" { + t.Errorf("want %v, got %v", "50.2", v.Map["a"]) + } + + var jsBuf bytes.Buffer + remain, err := msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + wantjs := `{"Single":-42,"Array":[0,0,1,-1,0.1,-0.1,1234,-1234,12.34,-12.34,12,120,120000000000000000000000000000000000,12,120,0.0000000000000000000000000000000012,-12,-120,-120000000000000000000000000000000000,-12,-120,-0.0000000000000000000000000000000012,1.2,12,12000000000000000000000000000000000,1.2,12,0.00000000000000000000000000000000012,-1.2,-12,-12000000000000000000000000000000000,-1.2,-12,-0.00000000000000000000000000000000012,0,0,0,0,0,0,-0,-0,-0,-0,-0,-0],"Map":{"a":50.2}}` + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } + // Test encoding + var buf bytes.Buffer + en := msgp.NewWriter(&buf) + err = test.EncodeMsg(en) + if err != nil { + t.Errorf("%v", err) + } + en.Flush() + encoded = buf.Bytes() + + dc := msgp.NewReader(&buf) + err = v.DecodeMsg(dc) + if err != nil { + t.Errorf("%v", err) + } + // Test two values + if v.Single != "-42" { + t.Errorf("want %s, got %s", "-42", v.Single) + } + if v.Map["a"] != "50.2" { + t.Errorf("want %s, got %s", "50.2", v.Map["a"]) + } + + jsBuf.Reset() + remain, err = msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } + + // Try interface encoder + jd := json.NewDecoder(&jsBuf) + jd.UseNumber() + var jsIntf map[string]any + err = jd.Decode(&jsIntf) + if err != nil { + t.Errorf("%v", err) + } + // Ensure we encode correctly + _ = (jsIntf["Single"]).(json.Number) + + fromInt, err := msgp.AppendIntf(nil, jsIntf) + if err != nil { + t.Errorf("%v", err) + } + + // Take the value from the JSON interface encoder and unmarshal back into our struct. + v = NumberJSONSample{} + _, err = v.UnmarshalMsg(fromInt) + if err != nil { + t.Errorf("%v", err) + } + if v.Single != "-42" { + t.Errorf("want %s, got %s", "-42", v.Single) + } + if v.Map["a"] != "50.2" { + t.Errorf("want %s, got %s", "50.2", v.Map["a"]) + } +} diff --git a/_generated/replace.go b/_generated/replace.go index c41b8610..121cf2cc 100644 --- a/_generated/replace.go +++ b/_generated/replace.go @@ -1,5 +1,7 @@ package _generated +import "encoding/json" + //go:generate msgp //msgp:replace Any with:any //msgp:replace MapString with:CompatibleMapString @@ -74,3 +76,11 @@ type ( String String } ) + +//msgp:replace json.Number with:string + +type NumberJSONSampleReplace struct { + Single json.Number + Array []json.Number + Map map[string]json.Number +} diff --git a/_generated/replace_test.go b/_generated/replace_test.go index 6764b388..31d26d71 100644 --- a/_generated/replace_test.go +++ b/_generated/replace_test.go @@ -1,8 +1,13 @@ package _generated import ( + "bytes" + "encoding/json" + "reflect" "testing" "time" + + "github.com/tinylib/msgp/msgp" ) func compareStructD(t *testing.T, a, b *CompatibleStructD) { @@ -288,3 +293,71 @@ func TestReplace_Dummy(t *testing.T) { t.Fatal("not same string") } } + +func TestJSONNumberReplace(t *testing.T) { + test := NumberJSONSampleReplace{ + Single: "-42", + Array: []json.Number{"0", "-0", "1", "-1", "0.1", "-0.1", "1234", "-1234", "12.34", "-12.34", "12E0", "12E1", "12e34", "12E-0", "12e+1", "12e-34", "-12E0", "-12E1", "-12e34", "-12E-0", "-12e+1", "-12e-34", "1.2E0", "1.2E1", "1.2e34", "1.2E-0", "1.2e+1", "1.2e-34", "-1.2E0", "-1.2E1", "-1.2e34", "-1.2E-0", "-1.2e+1", "-1.2e-34", "0E0", "0E1", "0e34", "0E-0", "0e+1", "0e-34", "-0E0", "-0E1", "-0e34", "-0E-0", "-0e+1", "-0e-34"}, + Map: map[string]json.Number{ + "a": json.Number("50"), + }, + } + + encoded, err := test.MarshalMsg(nil) + if err != nil { + t.Errorf("%v", err) + } + var v NumberJSONSampleReplace + _, err = v.UnmarshalMsg(encoded) + if err != nil { + t.Errorf("%v", err) + } + // Symmetric since we store strings. + if !reflect.DeepEqual(v, test) { + t.Fatalf("want %v, got %v", test, v) + } + + var jsBuf bytes.Buffer + remain, err := msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + // Retains number formatting. Map order is random, though. + wantjs := `{"Single":"-42","Array":["0","-0","1","-1","0.1","-0.1","1234","-1234","12.34","-12.34","12E0","12E1","12e34","12E-0","12e+1","12e-34","-12E0","-12E1","-12e34","-12E-0","-12e+1","-12e-34","1.2E0","1.2E1","1.2e34","1.2E-0","1.2e+1","1.2e-34","-1.2E0","-1.2E1","-1.2e34","-1.2E-0","-1.2e+1","-1.2e-34","0E0","0E1","0e34","0E-0","0e+1","0e-34","-0E0","-0E1","-0e34","-0E-0","-0e+1","-0e-34"],"Map":{"a":"50"}}` + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } + // Test encoding + var buf bytes.Buffer + en := msgp.NewWriter(&buf) + err = test.EncodeMsg(en) + if err != nil { + t.Errorf("%v", err) + } + en.Flush() + encoded = buf.Bytes() + + dc := msgp.NewReader(&buf) + err = v.DecodeMsg(dc) + if err != nil { + t.Errorf("%v", err) + } + if !reflect.DeepEqual(v, test) { + t.Fatalf("want %v, got %v", test, v) + } + + jsBuf.Reset() + remain, err = msgp.UnmarshalAsJSON(&jsBuf, encoded) + if err != nil { + t.Errorf("%v", err) + } + if len(remain) != 0 { + t.Errorf("remain should be empty") + } + if jsBuf.String() != wantjs { + t.Errorf("jsBuf.String() = \n%s, want \n%s", jsBuf.String(), wantjs) + } +} diff --git a/gen/elem.go b/gen/elem.go index 3ace7764..1fd26ff7 100644 --- a/gen/elem.go +++ b/gen/elem.go @@ -88,10 +88,11 @@ const ( Int32 Int64 Bool - Intf // interface{} - Time // time.Time - Duration // time.Duration - Ext // extension + Intf // interface{} + Time // time.Time + Duration // time.Duration + Ext // extension + JsonNumber // json.Number IDENT // IDENT means an unrecognized identifier ) @@ -123,6 +124,7 @@ var primitives = map[string]Primitive{ "time.Time": Time, "time.Duration": Duration, "msgp.Extension": Ext, + "json.Number": JsonNumber, } // types built into the library @@ -634,6 +636,9 @@ func (s *BaseElem) BaseName() string { if s.Value == Duration { return "Duration" } + if s.Value == JsonNumber { + return "JSONNumber" + } return s.Value.String() } @@ -652,6 +657,8 @@ func (s *BaseElem) BaseType() string { return "time.Time" case Duration: return "time.Duration" + case JsonNumber: + return "json.Number" case Ext: return "msgp.Extension" @@ -719,9 +726,10 @@ func (s *BaseElem) ZeroExpr() string { return "0" case Bool: return "false" - case Time: return "(time.Time{})" + case JsonNumber: + return `""` } @@ -783,6 +791,8 @@ func (k Primitive) String() string { return "time.Duration" case Ext: return "Extension" + case JsonNumber: + return "json.Number" case IDENT: return "Ident" default: diff --git a/gen/marshal.go b/gen/marshal.go index b58fd633..fdc8a48a 100644 --- a/gen/marshal.go +++ b/gen/marshal.go @@ -306,7 +306,7 @@ func (m *marshalGen) gBase(b *BaseElem) { case IDENT: echeck = true m.p.printf("\no, err = %s.MarshalMsg(o)", vname) - case Intf, Ext: + case Intf, Ext, JsonNumber: echeck = true m.p.printf("\no, err = msgp.Append%s(o, %s)", b.BaseName(), vname) default: diff --git a/msgp/read.go b/msgp/read.go index 31628782..5eb0b107 100644 --- a/msgp/read.go +++ b/msgp/read.go @@ -1,8 +1,10 @@ package msgp import ( + "encoding/json" "io" "math" + "strconv" "sync" "time" @@ -45,6 +47,7 @@ const ( Complex64Type Complex128Type TimeType + NumberType _maxtype ) @@ -74,6 +77,8 @@ func (t Type) String() string { return "ext" case NilType: return "nil" + case NumberType: + return "number" default: return "" } @@ -1276,6 +1281,35 @@ func (m *Reader) ReadTime() (t time.Time, err error) { return } +// ReadJSONNumber reads an integer or a float value and return as json.Number +func (m *Reader) ReadJSONNumber() (n json.Number, err error) { + t, err := m.NextType() + if err != nil { + return + } + switch t { + case IntType: + v, err := m.ReadInt64() + if err == nil { + return json.Number(strconv.FormatInt(v, 10)), nil + } + return "", err + case UintType: + v, err := m.ReadUint64() + if err == nil { + return json.Number(strconv.FormatUint(v, 10)), nil + } + return "", err + case Float32Type, Float64Type: + v, err := m.ReadFloat64() + if err == nil { + return json.Number(strconv.FormatFloat(v, 'f', -1, 64)), nil + } + return "", err + } + return "", TypeError{Method: NumberType, Encoded: t} +} + // ReadIntf reads out the next object as a raw interface{}/any. // Arrays are decoded as []interface{}, and maps are decoded // as map[string]interface{}. Integers are decoded as int64 diff --git a/msgp/read_bytes.go b/msgp/read_bytes.go index 05ca3bc7..cd20e97f 100644 --- a/msgp/read_bytes.go +++ b/msgp/read_bytes.go @@ -3,7 +3,9 @@ package msgp import ( "bytes" "encoding/binary" + "encoding/json" "math" + "strconv" "time" ) @@ -1327,3 +1329,24 @@ func getSize(b []byte) (uintptr, uintptr, error) { return 0, 0, fatal } } + +// ReadJSONNumberBytes tries to read a number +// from 'b' and return the value and the remaining bytes. +// +// Possible errors: +// +// - [ErrShortBytes] (too few bytes) +// - TypeError (not a number (int/float)) +func ReadJSONNumberBytes(b []byte) (number json.Number, o []byte, err error) { + if len(b) < 1 { + return "", nil, ErrShortBytes + } + if i, o, err := ReadInt64Bytes(b); err == nil { + return json.Number(strconv.FormatInt(i, 10)), o, nil + } + f, o, err := ReadFloat64Bytes(b) + if err == nil { + return json.Number(strconv.FormatFloat(f, 'f', -1, 64)), o, nil + } + return "", nil, TypeError{Method: NumberType, Encoded: getType(b[0])} +} diff --git a/msgp/size.go b/msgp/size.go index e3a613b2..585a67fd 100644 --- a/msgp/size.go +++ b/msgp/size.go @@ -25,10 +25,11 @@ const ( Complex64Size = 10 Complex128Size = 18 - DurationSize = Int64Size - TimeSize = 15 - BoolSize = 1 - NilSize = 1 + DurationSize = Int64Size + TimeSize = 15 + BoolSize = 1 + NilSize = 1 + JSONNumberSize = Int64Size // Same as Float64Size MapHeaderSize = 5 ArrayHeaderSize = 5 diff --git a/msgp/write.go b/msgp/write.go index e1b46a18..dfe0d3e8 100644 --- a/msgp/write.go +++ b/msgp/write.go @@ -1,6 +1,7 @@ package msgp import ( + "encoding/json" "errors" "io" "math" @@ -634,6 +635,23 @@ func (mw *Writer) WriteTime(t time.Time) error { return nil } +// WriteJSONNumber writes the json.Number to the stream as either integer or float. +func (mw *Writer) WriteJSONNumber(n json.Number) error { + if n == "" { + // The zero value outputs the 0 integer. + return mw.push(0) + } + ii, err := n.Int64() + if err == nil { + return mw.WriteInt64(ii) + } + ff, err := n.Float64() + if err == nil { + return mw.WriteFloat(ff) + } + return err +} + // WriteIntf writes the concrete type of 'v'. // WriteIntf will error if 'v' is not one of the following: // - A bool, float, string, []byte, int, uint, or complex @@ -699,6 +717,8 @@ func (mw *Writer) WriteIntf(v interface{}) error { return mw.WriteTime(v) case time.Duration: return mw.WriteDuration(v) + case json.Number: + return mw.WriteJSONNumber(v) } val := reflect.ValueOf(v) diff --git a/msgp/write_bytes.go b/msgp/write_bytes.go index fb7f7db5..a95b1d0b 100644 --- a/msgp/write_bytes.go +++ b/msgp/write_bytes.go @@ -1,6 +1,7 @@ package msgp import ( + "encoding/json" "errors" "math" "reflect" @@ -412,6 +413,8 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) { return AppendMapStrIntf(b, i) case map[string]string: return AppendMapStrStr(b, i), nil + case json.Number: + return AppendJSONNumber(b, i) case []interface{}: b = AppendArrayHeader(b, uint32(len(i))) var err error @@ -462,3 +465,21 @@ func AppendIntf(b []byte, i interface{}) ([]byte, error) { return b, &ErrUnsupportedType{T: v.Type()} } } + +// AppendJSONNumber appends a json.Number to the slice. +// An error will be returned if the json.Number returns error as both integer and float. +func AppendJSONNumber(b []byte, n json.Number) ([]byte, error) { + if n == "" { + // The zero value outputs the 0 integer. + return append(b, 0), nil + } + ii, err := n.Int64() + if err == nil { + return AppendInt64(b, ii), nil + } + ff, err := n.Float64() + if err == nil { + return AppendFloat(b, ff), nil + } + return b, err +}