diff --git a/pkg/codec/config.go b/pkg/codec/config.go index 92c79ceaa..21b6cca04 100644 --- a/pkg/codec/config.go +++ b/pkg/codec/config.go @@ -24,6 +24,7 @@ import ( // - epoch to time -> [EpochToTimeModifierConfig] // - address to string -> [AddressBytesToStringModifierConfig] // - field wrapper -> [WrapperModifierConfig] +// - precodec -> [PrecodecModifierConfig] type ModifiersConfig []ModifierConfig func (m *ModifiersConfig) UnmarshalJSON(data []byte) error { @@ -58,6 +59,8 @@ func (m *ModifiersConfig) UnmarshalJSON(data []byte) error { (*m)[i] = &AddressBytesToStringModifierConfig{} case ModifierWrapper: (*m)[i] = &WrapperModifierConfig{} + case ModifierPreCodec: + (*m)[i] = &PreCodecModifierConfig{} default: return fmt.Errorf("%w: unknown modifier type: %s", types.ErrInvalidConfig, mType) } @@ -84,6 +87,7 @@ func (m *ModifiersConfig) ToModifier(onChainHooks ...mapstructure.DecodeHookFunc type ModifierType string const ( + ModifierPreCodec ModifierType = "precodec" ModifierRename ModifierType = "rename" ModifierDrop ModifierType = "drop" ModifierHardCode ModifierType = "hard code" @@ -199,6 +203,69 @@ func (h *HardCodeModifierConfig) MarshalJSON() ([]byte, error) { }) } +// PreCodec creates a modifier that will transform data using a preliminary encoding/decoding step. +// 'Off-chain' values will be overwritten with the encoded data as a byte array. +// 'On-chain' values will be typed using the optimistic types from the codec. +// This is useful when wanting to move the data as generic bytes. +// +// Example: +// +// Based on this input struct: +// type example struct { +// A []B +// } +// +// type B struct { +// C string +// D string +// } +// +// And the fields config defined as: +// {"A": "string C, string D"} +// +// The codec config gives a map of strings (the values from fields config map) to implementation for encoding/decoding +// +// RemoteCodec { +// func (types.TypeProvider) CreateType(itemType string, forEncoding bool) (any, error) +// func (types.Decoder) Decode(ctx context.Context, raw []byte, into any, itemType string) error +// func (types.Encoder) Encode(ctx context.Context, item any, itemType string) ([]byte, error) +// func (types.Decoder) GetMaxDecodingSize(ctx context.Context, n int, itemType string) (int, error) +// func (types.Encoder) GetMaxEncodingSize(ctx context.Context, n int, itemType string) (int, error) +// } +// +// {"string C, string D": RemoteCodec} +// +// Result: +// type example struct { +// A [][]bytes +// } +// +// Where []bytes are the encoded input struct B +type PreCodecModifierConfig struct { + // A map of a path of properties to encoding scheme. + // If the path leads to an array, encoding will occur on every entry. + // + // Example: "a.b" -> "uint256 Value" + Fields map[string]string + // Codecs is skipped in JSON serialization, it will be injected later. + // The map should be keyed using the value from "Fields" to a corresponding Codec that can encode/decode for it + // This allows encoding and decoding implementations to be handled outside of the modifier. + // + // Example: "uint256 Value" -> a chain specific encoder for "uint256 Value" + Codecs map[string]types.RemoteCodec `json:"-"` +} + +func (c *PreCodecModifierConfig) ToModifier(_ ...mapstructure.DecodeHookFunc) (Modifier, error) { + return NewPreCodec(c.Fields, c.Codecs) +} + +func (c *PreCodecModifierConfig) MarshalJSON() ([]byte, error) { + return json.Marshal(&modifierMarshaller[PreCodecModifierConfig]{ + Type: ModifierPreCodec, + T: c, + }) +} + // EpochToTimeModifierConfig is used to convert epoch seconds as uint64 fields on-chain to time.Time type EpochToTimeModifierConfig struct { Fields []string diff --git a/pkg/codec/precodec.go b/pkg/codec/precodec.go new file mode 100644 index 000000000..447f97dcb --- /dev/null +++ b/pkg/codec/precodec.go @@ -0,0 +1,113 @@ +package codec + +import ( + "context" + "fmt" + "reflect" + + "github.com/go-viper/mapstructure/v2" + "github.com/smartcontractkit/chainlink-common/pkg/types" +) + +// PreCodec creates a modifier that will run a preliminary encoding/decoding step. +// This is useful when wanting to move nested data as generic bytes. +func NewPreCodec(fields map[string]string, codecs map[string]types.RemoteCodec) (Modifier, error) { + m := &preCodec{ + modifierBase: modifierBase[string]{ + fields: fields, + onToOffChainType: map[reflect.Type]reflect.Type{}, + offToOnChainType: map[reflect.Type]reflect.Type{}, + }, + codecs: codecs, + } + + // validate that there is a codec for each unique type definition + for _, typeDef := range fields { + if _, ok := m.codecs[typeDef]; ok { + continue + } + return nil, fmt.Errorf("codec not supplied for: %s", typeDef) + } + + m.modifyFieldForInput = func(_ string, field *reflect.StructField, _ string, typeDef string) error { + if field.Type != reflect.SliceOf(reflect.TypeFor[uint8]()) { + return fmt.Errorf("can only decode []byte from on-chain: %s", field.Type) + } + + codec, ok := m.codecs[typeDef] + if !ok || codec == nil { + return fmt.Errorf("codec not found for type definition: '%s'", typeDef) + } + + newType, err := codec.CreateType("", false) + if err != nil { + return err + } + field.Type = reflect.TypeOf(newType) + + return nil + } + + return m, nil +} + +type preCodec struct { + modifierBase[string] + codecs map[string]types.RemoteCodec +} + +func (pc *preCodec) TransformToOffChain(onChainValue any, _ string) (any, error) { + allHooks := make([]mapstructure.DecodeHookFunc, 1) + allHooks[0] = hardCodeManyHook + + return transformWithMaps(onChainValue, pc.onToOffChainType, pc.fields, pc.decodeFieldMapAction, allHooks...) +} + +func (pc *preCodec) decodeFieldMapAction(extractMap map[string]any, key string, typeDef string) error { + _, exists := extractMap[key] + if !exists { + return fmt.Errorf("field %s does not exist", key) + } + + codec, ok := pc.codecs[typeDef] + if !ok || codec == nil { + return fmt.Errorf("codec not found for type definition: '%s'", typeDef) + } + + to, err := codec.CreateType("", false) + if err != nil { + return err + } + err = codec.Decode(context.Background(), extractMap[key].([]byte), &to, "") + if err != nil { + return err + } + extractMap[key] = to + return nil +} + +func (pc *preCodec) TransformToOnChain(offChainValue any, _ string) (any, error) { + allHooks := make([]mapstructure.DecodeHookFunc, 1) + allHooks[0] = hardCodeManyHook + + return transformWithMaps(offChainValue, pc.offToOnChainType, pc.fields, pc.encodeFieldMapAction, allHooks...) +} + +func (pc *preCodec) encodeFieldMapAction(extractMap map[string]any, key string, typeDef string) error { + _, exists := extractMap[key] + if !exists { + return fmt.Errorf("field %s does not exist", key) + } + + codec, ok := pc.codecs[typeDef] + if !ok || codec == nil { + return fmt.Errorf("codec not found for type definition: '%s'", typeDef) + } + + encoded, err := codec.Encode(context.Background(), extractMap[key], "") + if err != nil { + return err + } + extractMap[key] = encoded + return nil +} diff --git a/pkg/codec/precodec_test.go b/pkg/codec/precodec_test.go new file mode 100644 index 000000000..9911ee877 --- /dev/null +++ b/pkg/codec/precodec_test.go @@ -0,0 +1,393 @@ +package codec_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/codec" + "github.com/smartcontractkit/chainlink-common/pkg/types" +) + +var _ types.RemoteCodec = &ExampleCodec{} + +type ExampleCodec struct { + offChainType any +} + +func (ec ExampleCodec) Encode(_ context.Context, item any, _ string) ([]byte, error) { + return json.Marshal(item) +} + +func (ec ExampleCodec) GetMaxEncodingSize(_ context.Context, n int, _ string) (int, error) { + // not used in the example + return math.MaxInt32, nil +} + +func (ec ExampleCodec) Decode(_ context.Context, raw []byte, into any, _ string) error { + err := json.Unmarshal(raw, into) + if err != nil { + return fmt.Errorf("%w: %w", types.ErrInvalidType, err) + } + return nil +} + +func (ec ExampleCodec) GetMaxDecodingSize(ctx context.Context, n int, _ string) (int, error) { + // not used in the example + return math.MaxInt32, nil +} + +func (ec ExampleCodec) CreateType(_ string, _ bool) (any, error) { + // parameters here are unused in the example, but can be used to determine what type to expect. + // this allows remote execution to know how to decode the incoming message + // and for [codec.NewModifierCodec] to know what type to expect for intermediate phases. + return ec.offChainType, nil +} + +type testStructOff struct { + Ask int + Bid int +} + +type testStructOn struct { + Ask []byte + Bid int +} + +type nestedTestStructOn struct { + Report []byte + FeedID [32]byte + Timestamp int64 +} + +type deepNestedTestStructOn struct { + Reports []nestedTestStructOn +} + +const ( + TestStructOffDef = "uint256 Ask, uint256 Bid" +) + +func TestPreCodec(t *testing.T) { + t.Parallel() + + preCodec, err := codec.NewPreCodec( + map[string]string{"Ask": "uint256"}, + map[string]types.RemoteCodec{"uint256": ExampleCodec{offChainType: int(0)}}, + ) + require.NoError(t, err) + + nestedPreCodec, err := codec.NewPreCodec( + map[string]string{"Report": TestStructOffDef}, + map[string]types.RemoteCodec{TestStructOffDef: ExampleCodec{offChainType: testStructOff{}}}, + ) + require.NoError(t, err) + + deepNestedPreCodec, err := codec.NewPreCodec( + map[string]string{"Reports.Report": TestStructOffDef}, + map[string]types.RemoteCodec{TestStructOffDef: ExampleCodec{offChainType: testStructOff{}}}, + ) + require.NoError(t, err) + + invalidPreCodec, err := codec.NewPreCodec( + map[string]string{"Unknown": TestStructOffDef}, + map[string]types.RemoteCodec{TestStructOffDef: ExampleCodec{offChainType: testStructOff{}}}, + ) + require.NoError(t, err) + + t.Run("NOK codec not supplied", func(t *testing.T) { + _, err := codec.NewPreCodec( + map[string]string{"Unknown": TestStructOffDef}, + map[string]types.RemoteCodec{"invalid def": ExampleCodec{offChainType: testStructOff{}}}, + ) + require.Error(t, err) + }) + + t.Run("RetypeToOffChain converts type to codec.CreateType type", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.TypeOf(testStructOn{}), "") + require.NoError(t, err) + require.Equal(t, 2, offChainType.NumField()) + field0 := offChainType.Field(0) + assert.Equal(t, "Ask", field0.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field0.Type) + field1 := offChainType.Field(1) + assert.Equal(t, "Bid", field1.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field1.Type) + }) + + t.Run("RetypeToOffChain works on pointers", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.PointerTo(reflect.TypeOf(testStructOn{})), "") + require.NoError(t, err) + assert.Equal(t, reflect.Ptr, offChainType.Kind()) + elem := offChainType.Elem() + require.Equal(t, 2, elem.NumField()) + field0 := elem.Field(0) + assert.Equal(t, "Ask", field0.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field0.Type) + field1 := elem.Field(1) + assert.Equal(t, "Bid", field1.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field1.Type) + }) + + t.Run("RetypeToOffChain works on slices", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.SliceOf(reflect.TypeOf(testStructOn{})), "") + require.NoError(t, err) + assert.Equal(t, reflect.Slice, offChainType.Kind()) + elem := offChainType.Elem() + require.Equal(t, 2, elem.NumField()) + field0 := elem.Field(0) + assert.Equal(t, "Ask", field0.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field0.Type) + field1 := elem.Field(1) + assert.Equal(t, "Bid", field1.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field1.Type) + }) + + t.Run("RetypeToOffChain works on arrays", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.ArrayOf(1, reflect.TypeOf(testStructOn{})), "") + require.NoError(t, err) + assert.Equal(t, reflect.Array, offChainType.Kind()) + elem := offChainType.Elem() + require.Equal(t, 2, elem.NumField()) + field0 := elem.Field(0) + assert.Equal(t, "Ask", field0.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field0.Type) + field1 := elem.Field(1) + assert.Equal(t, "Bid", field1.Name) + assert.Equal(t, reflect.TypeOf(int(0)), field1.Type) + }) + + t.Run("RetypeToOffChain converts nested type to codec.CreateType type", func(t *testing.T) { + offChainType, err := nestedPreCodec.RetypeToOffChain(reflect.TypeOf(nestedTestStructOn{}), "") + + require.NoError(t, err) + + require.Equal(t, 3, offChainType.NumField()) + field0 := offChainType.Field(0) + assert.Equal(t, "Report", field0.Name) + assert.Equal(t, reflect.TypeOf(testStructOff{}), field0.Type) + field1 := offChainType.Field(1) + assert.Equal(t, "FeedID", field1.Name) + assert.Equal(t, reflect.TypeOf([32]byte{}), field1.Type) + field2 := offChainType.Field(2) + assert.Equal(t, "Timestamp", field2.Name) + assert.Equal(t, reflect.TypeOf(int64(0)), field2.Type) + }) + + t.Run("RetypeToOffChain converts deep nested type to codec.CreateType type", func(t *testing.T) { + offChainType, err := deepNestedPreCodec.RetypeToOffChain(reflect.TypeOf(deepNestedTestStructOn{}), "") + + require.NoError(t, err) + + reports, exists := offChainType.FieldByName("Reports") + assert.True(t, exists) + report := reports.Type.Elem() + require.Equal(t, 3, report.NumField()) + field0 := report.Field(0) + assert.Equal(t, "Report", field0.Name) + assert.Equal(t, reflect.TypeOf(testStructOff{}), field0.Type) + field1 := report.Field(1) + assert.Equal(t, "FeedID", field1.Name) + assert.Equal(t, reflect.TypeOf([32]byte{}), field1.Type) + field2 := report.Field(2) + assert.Equal(t, "Timestamp", field2.Name) + assert.Equal(t, reflect.TypeOf(int64(0)), field2.Type) + }) + + t.Run("RetypeToOffChain only works on byte arrays", func(t *testing.T) { + _, err := preCodec.RetypeToOffChain(reflect.TypeOf(testStructOff{}), "") + require.Error(t, err) + assert.Equal(t, err.Error(), "can only decode []byte from on-chain: int") + }) + + t.Run("RetypeToOffChain only works with a valid path", func(t *testing.T) { + _, err := invalidPreCodec.RetypeToOffChain(reflect.TypeOf(testStructOn{}), "") + require.Error(t, err) + assert.Equal(t, err.Error(), "invalid type: cannot find Unknown") + }) + + t.Run("TransformToOnChain and TransformToOffChain returns error if input type was not from TransformToOnChain", func(t *testing.T) { + incorrectVal := struct{}{} + _, err := preCodec.TransformToOnChain(incorrectVal, "") + assert.True(t, errors.Is(err, types.ErrInvalidType)) + _, err = preCodec.TransformToOffChain(incorrectVal, "") + assert.True(t, errors.Is(err, types.ErrInvalidType)) + }) + + t.Run("TransformToOnChain and TransformToOffChain works on structs", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.TypeOf(testStructOn{}), "") + require.NoError(t, err) + iOffchain := reflect.Indirect(reflect.New(offChainType)) + iOffchain.FieldByName("Ask").SetInt(20) + iOffchain.FieldByName("Bid").SetInt(10) + + output, err := preCodec.TransformToOnChain(iOffchain.Interface(), "") + require.NoError(t, err) + + jsonEncoded, err := json.Marshal(20) + require.NoError(t, err) + expected := testStructOn{ + Ask: jsonEncoded, + Bid: 10, + } + assert.Equal(t, expected, output) + newInput, err := preCodec.TransformToOffChain(expected, "") + require.NoError(t, err) + assert.Equal(t, iOffchain.Interface(), newInput) + }) + + t.Run("TransformToOnChain and TransformToOffChain works on pointers", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.PointerTo(reflect.TypeOf(testStructOn{})), "") + require.NoError(t, err) + + rOffchain := reflect.New(offChainType.Elem()) + iOffchain := reflect.Indirect(rOffchain) + iOffchain.FieldByName("Ask").SetInt(20) + iOffchain.FieldByName("Bid").SetInt(10) + + output, err := preCodec.TransformToOnChain(rOffchain.Interface(), "") + require.NoError(t, err) + jsonEncoded, err := json.Marshal(20) + require.NoError(t, err) + expected := testStructOn{ + Ask: jsonEncoded, + Bid: 10, + } + assert.Equal(t, &expected, output) + newInput, err := preCodec.TransformToOffChain(expected, "") + require.NoError(t, err) + assert.Equal(t, iOffchain.Interface(), newInput) + }) + + t.Run("TransformToOnChain and TransformToOffChain works on slices", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.SliceOf(reflect.TypeOf(testStructOn{})), "") + require.NoError(t, err) + + iOffchain := reflect.MakeSlice(offChainType, 2, 2) + iElm := iOffchain.Index(0) + iElm.FieldByName("Ask").SetInt(20) + iElm.FieldByName("Bid").SetInt(10) + iElm2 := iOffchain.Index(1) + iElm2.FieldByName("Ask").SetInt(20) + iElm2.FieldByName("Bid").SetInt(30) + + output, err := preCodec.TransformToOnChain(iOffchain.Interface(), "") + require.NoError(t, err) + + jsonEncoded, err := json.Marshal(20) + require.NoError(t, err) + expected := []testStructOn{ + { + Ask: jsonEncoded, + Bid: 10, + }, + { + Ask: jsonEncoded, + Bid: 30, + }, + } + assert.Equal(t, expected, output) + newInput, err := preCodec.TransformToOffChain(expected, "") + require.NoError(t, err) + assert.Equal(t, iOffchain.Interface(), newInput) + }) + + t.Run("TransformToOnChain and TransformToOffChain works on arrays", func(t *testing.T) { + offChainType, err := preCodec.RetypeToOffChain(reflect.ArrayOf(2, reflect.TypeOf(testStructOn{})), "") + require.NoError(t, err) + + iOffchain := reflect.New(offChainType).Elem() + iElm := iOffchain.Index(0) + iElm.FieldByName("Ask").SetInt(20) + iElm.FieldByName("Bid").SetInt(10) + iElm2 := iOffchain.Index(1) + iElm2.FieldByName("Ask").SetInt(20) + iElm2.FieldByName("Bid").SetInt(30) + + output, err := preCodec.TransformToOnChain(iOffchain.Interface(), "") + require.NoError(t, err) + + jsonEncoded, err := json.Marshal(20) + require.NoError(t, err) + expected := [2]testStructOn{ + { + Ask: jsonEncoded, + Bid: 10, + }, + { + Ask: jsonEncoded, + Bid: 30, + }, + } + assert.Equal(t, expected, output) + newInput, err := preCodec.TransformToOffChain(expected, "") + require.NoError(t, err) + assert.Equal(t, iOffchain.Interface(), newInput) + }) + + t.Run("TransformToOnChain and TransformToOffChain works on nested fields", func(t *testing.T) { + offChainType, err := nestedPreCodec.RetypeToOffChain(reflect.TypeOf(nestedTestStructOn{}), "") + require.NoError(t, err) + + iOffchain := reflect.Indirect(reflect.New(offChainType)) + iReport := iOffchain.FieldByName("Report") + iReport.FieldByName("Ask").SetInt(20) + iReport.FieldByName("Bid").SetInt(10) + + output, err := nestedPreCodec.TransformToOnChain(iOffchain.Interface(), "") + require.NoError(t, err) + + report := testStructOff{ + Ask: 20, + Bid: 10, + } + jsonEncoded, err := json.Marshal(report) + require.NoError(t, err) + expected := nestedTestStructOn{ + Report: jsonEncoded, + } + assert.Equal(t, expected, output) + newInput, err := nestedPreCodec.TransformToOffChain(expected, "") + require.NoError(t, err) + assert.Equal(t, iOffchain.Interface(), newInput) + }) + + t.Run("TransformToOnChain and TransformToOffChain works on deeply nested fields", func(t *testing.T) { + offChainType, err := deepNestedPreCodec.RetypeToOffChain(reflect.TypeOf(deepNestedTestStructOn{}), "") + require.NoError(t, err) + + iOffchain := reflect.Indirect(reflect.New(offChainType)) + iReports := iOffchain.FieldByName("Reports") + iReports.Set(reflect.MakeSlice(iReports.Type(), 1, 1)) + iElm := iReports.Index(0) + iReport := iElm.FieldByName("Report") + iReport.FieldByName("Ask").SetInt(20) + iReport.FieldByName("Bid").SetInt(10) + + output, err := deepNestedPreCodec.TransformToOnChain(iOffchain.Interface(), "") + require.NoError(t, err) + + report := testStructOff{ + Ask: 20, + Bid: 10, + } + jsonEncoded, err := json.Marshal(report) + require.NoError(t, err) + expected := deepNestedTestStructOn{ + Reports: []nestedTestStructOn{ + {Report: jsonEncoded}, + }, + } + assert.Equal(t, expected, output) + newInput, err := deepNestedPreCodec.TransformToOffChain(expected, "") + require.NoError(t, err) + assert.Equal(t, iOffchain.Interface(), newInput) + }) +}