From 8609f7be466cbc373ebbb3078fe2003df76ba87f Mon Sep 17 00:00:00 2001 From: Noah Treuhaft Date: Tue, 9 May 2023 10:54:15 -0400 Subject: [PATCH] Improve JSON output for Zed maps In JSON output, a Zed map becomes an array of {"key":KEY,"value":VALUE} objects. Change that to a single object object whose keys are ZSON representations of the map keys, as in {"KEY":VALUE}. * String keys are used directly. * Non-string primitive keys are converted to undecorated ZSON. * Union keys are untagged and then converted to ZSON (with decoration for uniqueness). * Enum keys are converted to their corresponding symbol. * All other keys are converted to ZSON. Closes #4567. --- service/ztests/python.yaml | 2 +- zio/jsonio/marshal.go | 34 +++++++++++----- zio/jsonio/ztests/map-output.yaml | 67 +++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 12 deletions(-) create mode 100644 zio/jsonio/ztests/map-output.yaml diff --git a/service/ztests/python.yaml b/service/ztests/python.yaml index c128877c73..4327fa7b2b 100644 --- a/service/ztests/python.yaml +++ b/service/ztests/python.yaml @@ -155,7 +155,7 @@ outputs: {'u8': 0, 'u16': 0, 'u32': 0, 'u64': 0, 'i8': 0, 'i16': 0, 'i32': 0, 'i64': 0, 'dur': datetime.timedelta(0), 'tim': datetime.datetime(1970, 1, 1, 0, 0, tzinfo=tzutc()), 'f64': 0.0, 'boo': False, 'byt': b'\x00', 'str': '', 'ip': IPv4Address('0.0.0.0'), 'net': IPv4Network('0.0.0.0/0'), 'err': '', 'nul': None} {'u8': 0, 'u16': 0, 'u32': 0, 'u64': 0, 'i8': 0, 'i16': 0, 'i32': 0, 'i64': 0, 'dur': datetime.timedelta(0), 'tim': datetime.datetime(1970, 1, 1, 0, 0, tzinfo=tzutc()), 'f64': 0.0, 'boo': False, 'byt': b'\x00', 'str': '', 'ip': IPv4Address('0.0.0.0'), 'net': IPv4Network('0.0.0.0/0'), 'err': '', 'nul': None} === JSON - [{"map":[{"key":"a","value":{"a":1,"b":2}},{"key":"b","value":{"a":2,"b":3}},{"key":"c","value":{"a":3,"b":4}}]},{"set":[1,2,3,4]},{"union":["a","b"]},{"union":[1,2]},{"union":"hello"},{"array":[{"a":1},{"a":2}]},{"union":123},{"enum":"bar"},{"u8":0,"u16":0,"u32":0,"u64":0,"i8":0,"i16":0,"i32":0,"i64":0,"dur":"0s","tim":"1970-01-01T00:00:00Z","f64":0,"boo":false,"byt":"0x00","str":"","ip":"0.0.0.0","net":"0.0.0.0/0","err":{"error":""},"nul":null},{"u8":0,"u16":0,"u32":0,"u64":0,"i8":0,"i16":0,"i32":0,"i64":0,"dur":"0s","tim":"1970-01-01T00:00:00Z","f64":0,"boo":false,"byt":"0x00","str":"","ip":"0.0.0.0","net":"0.0.0.0/0","err":{"error":""},"nul":null}] + [{"map":{"a":{"a":1,"b":2},"b":{"a":2,"b":3},"c":{"a":3,"b":4}}},{"set":[1,2,3,4]},{"union":["a","b"]},{"union":[1,2]},{"union":"hello"},{"array":[{"a":1},{"a":2}]},{"union":123},{"enum":"bar"},{"u8":0,"u16":0,"u32":0,"u64":0,"i8":0,"i16":0,"i32":0,"i64":0,"dur":"0s","tim":"1970-01-01T00:00:00Z","f64":0,"boo":false,"byt":"0x00","str":"","ip":"0.0.0.0","net":"0.0.0.0/0","err":{"error":""},"nul":null},{"u8":0,"u16":0,"u32":0,"u64":0,"i8":0,"i16":0,"i32":0,"i64":0,"dur":"0s","tim":"1970-01-01T00:00:00Z","f64":0,"boo":false,"byt":"0x00","str":"","ip":"0.0.0.0","net":"0.0.0.0/0","err":{"error":""},"nul":null}] === RequestError('test: pool already exists') diff --git a/zio/jsonio/marshal.go b/zio/jsonio/marshal.go index b180756031..9bcc7c3a0b 100644 --- a/zio/jsonio/marshal.go +++ b/zio/jsonio/marshal.go @@ -120,20 +120,32 @@ func marshalSet(typ *zed.TypeSet, bytes zcode.Bytes) interface{} { return s } -type Entry struct { - Key interface{} `json:"key"` - Value interface{} `json:"value"` -} - func marshalMap(typ *zed.TypeMap, bytes zcode.Bytes) interface{} { - var entries []Entry - it := bytes.Iter() - for !it.Done() { - key := marshalAny(typ.KeyType, it.Next()) + keyType := zed.TypeUnder(typ.KeyType) + rec := record{} + for it := bytes.Iter(); !it.Done(); { + var key string + switch kind := keyType.Kind(); { + case keyType == zed.TypeString: + // Don't quote strings. + key = zed.DecodeString(it.Next()) + case kind == zed.PrimitiveKind: + // Undecorated ZSON. + key = zson.FormatPrimitive(keyType, it.Next()) + case kind == zed.UnionKind: + // Untagged, decorated ZSON so + // |{0:1,0(uint64):2,0(=t):3,"0":4}| gets unique keys. + typ, bytes := keyType.(*zed.TypeUnion).Untag(it.Next()) + key = zson.MustFormatValue(zed.NewValue(typ, bytes)) + case kind == zed.EnumKind: + key = marshalEnum(keyType.(*zed.TypeEnum), it.Next()).(string) + default: + key = zson.MustFormatValue(zed.NewValue(keyType, it.Next())) + } val := marshalAny(typ.ValType, it.Next()) - entries = append(entries, Entry{key, val}) + rec = append(rec, field{key, val}) } - return entries + return rec } func marshalEnum(typ *zed.TypeEnum, bytes zcode.Bytes) interface{} { diff --git a/zio/jsonio/ztests/map-output.yaml b/zio/jsonio/ztests/map-output.yaml new file mode 100644 index 0000000000..8ccd3374ce --- /dev/null +++ b/zio/jsonio/ztests/map-output.yaml @@ -0,0 +1,67 @@ +zed: '*' + +input: | + |{}| + // Primitive keys + |{8(uint8):1}| + |{16(uint16):1}| + |{32(uint32):1}| + |{64:1}| + |{-8(int8):1}| + |{-16(int16):1}| + |{-32(int32):1}| + |{-64:1}| + |{1s:1}| + |{1970-01-01T00:00:00Z:1}| + |{16.(float16):1}| + |{32.(float32):1}| + |{64.:1}| + |{false:1}| + |{0x01020304:1}| + |{"string":1}| + |{1.2.3.4:1}| + |{5.6.7.0/24:1}| + |{:1}| + |{null:1}| + // Complex keys + |{{record:0}:1}| + |{["array"]:1}| + |{|["set"]|:1}| + |{|{"map":0}|:1}| + |{0:1,0(uint64):2,0(=t):3,"0":4}| + |{%e0(enum(e0)):1}| + |{error(0):1}| + |{"named"(=t):1}| + +output-flags: -f json + +output: | + {} + {"8":1} + {"16":1} + {"32":1} + {"64":1} + {"-8":1} + {"-16":1} + {"-32":1} + {"-64":1} + {"1s":1} + {"1970-01-01T00:00:00Z":1} + {"16.":1} + {"32.":1} + {"64.":1} + {"false":1} + {"0x01020304":1} + {"string":1} + {"1.2.3.4":1} + {"5.6.7.0/24":1} + {"":1} + {"null":1} + {"{record:0}":1} + {"[\"array\"]":1} + {"|[\"set\"]|":1} + {"|{\"map\":0}|":1} + {"0(uint64)":2,"0":1,"0(=t)":3,"\"0\"":4} + {"e0":1} + {"error(0)":1} + {"named":1}