feat(events): introduce compact JSON form of EventEntry

api/docgen/docgen.go

@@ -43,6 +43,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/types/ethtypes"
+	"github.com/filecoin-project/lotus/lib/must"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo/imports"
 	sealing "github.com/filecoin-project/lotus/storage/pipeline"
@@ -433,6 +434,28 @@ func init() {
 		FromHeight: epochPtr(1010),
 		ToHeight:   epochPtr(1020),
 	})
+
+	ae := types.ActorEvent{
+		Emitter: must.One(address.NewIDAddress(1234)),
+		Entries: []types.EventEntry{
+			{
+				Codec: 81,
+				Flags: 3,
+				Key:   "$type",
+				Value: []byte("jallocation"),
+			},
+			{
+				Codec: 81,
+				Flags: 3,
+				Key:   "client",
+				Value: []byte("\x19\x03\xf3"),
+			},
+		},
+		Height:    abi.ChainEpoch(101010),
+		MsgCid:    cid.MustParse("bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"),
+		TipSetKey: types.NewTipSetKey(cid.MustParse("bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"), cid.MustParse("bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve")),
+	}.AsCompactEncoded()
+	addExample(&ae)
 }
 
 func GetAPIType(name, pkg string) (i interface{}, t reflect.Type, permStruct []reflect.Type) {

chain/types/actor_event.go

@@ -1,7 +1,21 @@
 package types
 
 import (
+	"encoding/base64"
+	"errors"
+	"fmt"
+
 	"github.com/ipfs/go-cid"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/codec"
+	"github.com/ipld/go-ipld-prime/codec/dagcbor"
+	"github.com/ipld/go-ipld-prime/codec/dagjson"
+	"github.com/ipld/go-ipld-prime/codec/raw"
+	"github.com/ipld/go-ipld-prime/datamodel"
+	"github.com/ipld/go-ipld-prime/fluent/qp"
+	basicnode "github.com/ipld/go-ipld-prime/node/basic"
+	"github.com/ipld/go-ipld-prime/node/bindnode"
+	"github.com/ipld/go-ipld-prime/schema"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -44,6 +58,8 @@ type ActorEventFilter struct {
 }
 
 type ActorEvent struct {
+	encodeCompact *bool `json:"-"` // shouldn't be exposed publicly for any reason
+
 	// Event entries in log form.
 	Entries []EventEntry `json:"entries"`
 
@@ -65,3 +81,286 @@ type ActorEvent struct {
 	// CID of message that produced this event.
 	MsgCid cid.Cid `json:"msgCid"`
 }
+
+// AsCompactEncoded will trigger alternate JSON encoding for ActorEvents, where the event entries
+// are encoded as a list of tuple representation structs, rather than a list of maps, values are
+// decoded using the specified codec where possible, and they are encoded using dag-json form so
+// bytes are represented using the `{"/":{"bytes":"base64"}}` form rather than Go standard base64
+// encoding.
+func (ae ActorEvent) AsCompactEncoded() ActorEvent {
+	ae.encodeCompact = new(bool)
+	*ae.encodeCompact = true
+	return ae
+}
+
+func (ae *ActorEvent) UnmarshalJSON(b []byte) error {
+	nd, err := ipld.Decode(b, dagjson.Decode)
+	if err != nil {
+		return err
+	}
+	builder := actorEventProto.Representation().NewBuilder()
+	if err := builder.AssignNode(nd); err != nil {
+		return err
+	}
+	aePtr := bindnode.Unwrap(builder.Build())
+	aec, _ := aePtr.(*ActorEvent) // safe to assume type
+	*ae = *aec
+
+	// check if we were encoded in compact form and set the flag accordingly
+	entries, _ := nd.LookupByString("entries")
+	if entries.Length() > 0 {
+		first, _ := entries.LookupByIndex(0)
+		if first.Kind() == datamodel.Kind_List {
+			ae.encodeCompact = new(bool)
+			*ae.encodeCompact = true
+		}
+	}
+
+	return nil
+}
+
+func (ae ActorEvent) MarshalJSON() ([]byte, error) {
+	var entryOpt bindnode.Option = eventEntryBindnodeOption
+	if ae.encodeCompact != nil {
+		if *ae.encodeCompact {
+			entryOpt = eventEntryCompactBindnodeOption
+		}
+		ae.encodeCompact = nil // hide it from this encode
+	}
+	nd := bindnode.Wrap(
+		&ae,
+		actorEventProto.Type(),
+		TipSetKeyAsLinksListBindnodeOption,
+		addressAsStringBindnodeOption,
+		entryOpt,
+	)
+	return ipld.Encode(nd, dagjson.Encode)
+}
+
+// TODO: move this in to go-state-types/ipld with the address "as bytes" form
+var addressAsStringBindnodeOption = bindnode.TypedStringConverter(&address.Address{}, addressFromString, addressToString)
+
+func addressFromString(s string) (interface{}, error) {
+	a, err := address.NewFromString(s)
+	if err != nil {
+		return nil, err
+	}
+	return &a, nil
+}
+
+func addressToString(iface interface{}) (string, error) {
+	addr, ok := iface.(*address.Address)
+	if !ok {
+		return "", errors.New("expected *Address value")
+	}
+	return addr.String(), nil
+}
+
+var eventEntryBindnodeOption = bindnode.TypedAnyConverter(&EventEntry{}, eventEntryFromAny, eventEntryToAny)
+var eventEntryCompactBindnodeOption = bindnode.TypedAnyConverter(&EventEntry{}, eventEntryCompactFromAny, eventEntryCompactToAny)
+
+// eventEntryFromAny will instantiate an EventEntry assuming standard Go JSON form, i.e.:
+// {"Codec":82,"Flags":0,"Key":"key2","Value":"dmFsdWUy"}
+// Where the value is intact as raw bytes but represented as a base64 string, and the object is
+// represented as a map.
+func eventEntryFromAny(n datamodel.Node) (interface{}, error) {
+	if n.Kind() == datamodel.Kind_List {
+		return eventEntryCompactFromAny(n)
+	}
+	if n.Kind() != datamodel.Kind_Map {
+		return nil, errors.New("expected map representation for EventEntry")
+	}
+	if n.Length() != 4 {
+		return nil, errors.New("expected 4 fields for EventEntry")
+	}
+	fn, err := n.LookupByString("Flags")
+	if err != nil {
+		return nil, fmt.Errorf("missing Flags field for EventEntry: %w", err)
+	}
+	flags, err := fn.AsInt()
+	if err != nil {
+		return nil, fmt.Errorf("expected int in Flags field for EventEntry: %w", err)
+	}
+	cn, err := n.LookupByString("Codec")
+	if err != nil {
+		return nil, fmt.Errorf("missing Codec field for EventEntry: %w", err)
+	}
+	codec, err := cn.AsInt()
+	if err != nil {
+		return nil, fmt.Errorf("expected int in Codec field for EventEntry: %w", err)
+	}
+	// it has to fit into a uint8
+	if flags < 0 || flags > 255 {
+		return nil, fmt.Errorf("expected uint8 in Flags field for EventEntry, got %d", flags)
+	}
+	kn, err := n.LookupByString("Key")
+	if err != nil {
+		return nil, fmt.Errorf("missing Key field for EventEntry: %w", err)
+	}
+	key, err := kn.AsString()
+	if err != nil {
+		return nil, fmt.Errorf("expected string in Key field for EventEntry: %w", err)
+	}
+	vn, err := n.LookupByString("Value")
+	if err != nil {
+		return nil, fmt.Errorf("missing Value field for EventEntry: %w", err)
+	}
+	value64, err := vn.AsString() // base64
+	if err != nil {
+		return nil, fmt.Errorf("expected string in Value field for EventEntry: %w", err)
+	}
+	value, err := base64.StdEncoding.DecodeString(value64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode base64 value: %w", err)
+	}
+	return &EventEntry{
+		Flags: uint8(flags),
+		Key:   key,
+		Codec: uint64(codec),
+		Value: value,
+	}, nil
+}
+
+// eventEntryCompactFromAny will instantiate an EventEntry assuming compact form, i.e.:
+// [0,82,"key2",{"/":{"bytes":"dmFsdWUy"}}]
+// Where the value is represented in its decoded IPLD data model form, and the object is represented
+// as a tuple.
+func eventEntryCompactFromAny(n datamodel.Node) (interface{}, error) {
+	if n.Kind() != datamodel.Kind_List {
+		return nil, errors.New("expected list representation for compact EventEntry")
+	}
+	if n.Length() != 4 {
+		return nil, errors.New("expected 4 fields for EventEntry")
+	}
+	// Flags before Codec in this form, sorted Codec before Flags in the non-compact form when dag-json
+	fn, err := n.LookupByIndex(0)
+	if err != nil {
+		return nil, fmt.Errorf("missing Flags field for EventEntry: %w", err)
+	}
+	flags, err := fn.AsInt()
+	if err != nil {
+		return nil, fmt.Errorf("expected int in Flags field for EventEntry: %w", err)
+	}
+	// it has to fit into a uint8
+	if flags < 0 || flags > 255 {
+		return nil, fmt.Errorf("expected uint8 in Flags field for EventEntry, got %d", flags)
+	}
+	cn, err := n.LookupByIndex(1)
+	if err != nil {
+		return nil, fmt.Errorf("missing Codec field for EventEntry: %w", err)
+	}
+	codecCode, err := cn.AsInt()
+	if err != nil {
+		return nil, fmt.Errorf("expected int in Codec field for EventEntry: %w", err)
+	}
+	kn, err := n.LookupByIndex(2)
+	if err != nil {
+		return nil, fmt.Errorf("missing Key field for EventEntry: %w", err)
+	}
+	key, err := kn.AsString()
+	if err != nil {
+		return nil, fmt.Errorf("expected string in Key field for EventEntry: %w", err)
+	}
+	vn, err := n.LookupByIndex(3)
+	if err != nil {
+		return nil, fmt.Errorf("missing Value field for EventEntry: %w", err)
+	}
+	// as of writing only 0x55 and 0x51 are supported here, but we'll treat raw as the default,
+	// regardless, which means that for any unexpected codecs encountered we'll assume that the
+	// encoder also didn't know what to do with it and just treat it as raw bytes.
+	var value []byte
+	switch codecCode {
+	case 0x51: // plain cbor
+		if value, err = ipld.Encode(vn, dagcbor.Encode); err != nil {
+			return nil, fmt.Errorf("failed to encode cbor value: %w", err)
+		}
+	default: // raw (0x55) and all unknowns
+		if vn.Kind() != datamodel.Kind_Bytes {
+			return nil, fmt.Errorf("expected bytes in Value field for EventEntry, got %s", vn.Kind())
+		}
+		if value, err = vn.AsBytes(); err != nil {
+			return nil, err
+		}
+	}
+
+	return &EventEntry{
+		Flags: uint8(flags),
+		Key:   key,
+		Codec: uint64(codecCode),
+		Value: value,
+	}, nil
+}
+
+// eventEntryToAny does the reverse of eventEntryFromAny, converting an EventEntry back to the
+// standard Go JSON form, i.e.:
+// {"Codec":82,"Flags":0,"Key":"key2","Value":"dmFsdWUy"}
+func eventEntryToAny(iface interface{}) (datamodel.Node, error) {
+	ee, ok := iface.(*EventEntry)
+	if !ok {
+		return nil, errors.New("expected *Address value")
+	}
+	return qp.BuildMap(basicnode.Prototype.Map, 4, func(ma datamodel.MapAssembler) {
+		qp.MapEntry(ma, "Flags", qp.Int(int64(ee.Flags)))
+		qp.MapEntry(ma, "Codec", qp.Int(int64(ee.Codec)))
+		qp.MapEntry(ma, "Key", qp.String(ee.Key))
+		qp.MapEntry(ma, "Value", qp.String(base64.StdEncoding.EncodeToString(ee.Value)))
+	})
+}
+
+// eventEntryCompactToAny does the reverse of eventEntryCompactFromAny, converting an EventEntry
+// back to the compact form, i.e.:
+// [0,82,"key2",{"/":{"bytes":"dmFsdWUy"}}]
+func eventEntryCompactToAny(iface interface{}) (datamodel.Node, error) {
+	ee, ok := iface.(*EventEntry)
+	if !ok {
+		return nil, errors.New("expected *Address value")
+	}
+	var decoder codec.Decoder = raw.Decode
+	if ee.Codec == 0x51 {
+		decoder = dagcbor.Decode
+	}
+	valueNode, err := ipld.Decode(ee.Value, decoder)
+	if err != nil {
+		log.Warn("failed to decode event entry value with expected codec", "err", err)
+		valueNode = basicnode.NewBytes(ee.Value)
+	}
+	return qp.BuildList(basicnode.Prototype.List, 4, func(la datamodel.ListAssembler) {
+		qp.ListEntry(la, qp.Int(int64(ee.Flags)))
+		qp.ListEntry(la, qp.Int(int64(ee.Codec)))
+		qp.ListEntry(la, qp.String(ee.Key))
+		qp.ListEntry(la, qp.Node(valueNode))
+	})
+}
+
+var (
+	actorEventProto    schema.TypedPrototype
+	fullFormIpldSchema = `
+type ActorEvent struct {
+	encodeCompact optional Bool
+	Entries       [Any]        (rename "entries")   # EventEntry
+	Emitter       String       (rename "emitter")   # addr.Address
+	Reverted      Bool         (rename "reverted")
+	Height        Int          (rename "height")
+	TipSetKey     Any          (rename "tipsetKey") # types.TipSetKey
+	MsgCid        &Any         (rename "msgCid")
+}
+`
+)
+
+func init() {
+	typeSystem, err := ipld.LoadSchemaBytes([]byte(fullFormIpldSchema))
+	if err != nil {
+		panic(err)
+	}
+	schemaType := typeSystem.TypeByName("ActorEvent")
+	if schemaType == nil {
+		panic(fmt.Errorf("schema for [%T] does not contain that named type [%s]", (*ActorEvent)(nil), "ActorEvent"))
+	}
+	actorEventProto = bindnode.Prototype(
+		(*ActorEvent)(nil),
+		schemaType,
+		TipSetKeyAsLinksListBindnodeOption,
+		addressAsStringBindnodeOption,
+		eventEntryBindnodeOption,
+	)
+}