(feat) Add PreCodec modifier

pkg/codec/config.go

@@ -24,6 +24,7 @@ import (
 // - epoch to time -> [EpochToTimeModifierConfig]
 // - address to string -> [AddressBytesToStringModifierConfig]
 // - field wrapper -> [WrapperModifierConfig]
+// - precodec -> [PrecodecModifierConfig]
 type ModifiersConfig []ModifierConfig
 
 func (m *ModifiersConfig) UnmarshalJSON(data []byte) error {
@@ -58,6 +59,8 @@ func (m *ModifiersConfig) UnmarshalJSON(data []byte) error {
 			(*m)[i] = &AddressBytesToStringModifierConfig{}
 		case ModifierWrapper:
 			(*m)[i] = &WrapperModifierConfig{}
+		case ModifierPreCodec:
+			(*m)[i] = &PreCodecModifierConfig{}
 		default:
 			return fmt.Errorf("%w: unknown modifier type: %s", types.ErrInvalidConfig, mType)
 		}
@@ -84,6 +87,7 @@ func (m *ModifiersConfig) ToModifier(onChainHooks ...mapstructure.DecodeHookFunc
 type ModifierType string
 
 const (
+	ModifierPreCodec        ModifierType = "precodec"
 	ModifierRename          ModifierType = "rename"
 	ModifierDrop            ModifierType = "drop"
 	ModifierHardCode        ModifierType = "hard code"
@@ -199,6 +203,33 @@ func (h *HardCodeModifierConfig) MarshalJSON() ([]byte, error) {
 	})
 }
 
+// PreCodec creates a modifier that will transform data using a preliminary encoding/decoding step.
+// 'Off-chain' values will be overwritten with the encoded data as a byte array.
+// 'On-chain' values will be typed using the optimistic types from the codec.
+// This is useful when wanting to move the data as generic bytes.
+type PreCodecModifierConfig struct {
+	// A map of a path of properties to encoding scheme.
+	// If the path leads to an array, encoding will occur on every entry.
+	//
+	// Example: "a.b" -> "uint256 Value"
+	Fields map[string]string
+	// A factory function that given an encoding scheme, returns a specific codec.
+	// This allows encoding and decoding implementations to be handled outside of the modifier.
+	// The map value given to "Fields" will be used to initialize the codec.
+	CodecFactory func(typeABI string) types.RemoteCodec
+}
+
+func (c *PreCodecModifierConfig) ToModifier(_ ...mapstructure.DecodeHookFunc) (Modifier, error) {
+	return NewPreCodec(c.Fields, c.CodecFactory), nil
+}
+
+func (c *PreCodecModifierConfig) MarshalJSON() ([]byte, error) {
+	return json.Marshal(&modifierMarshaller[PreCodecModifierConfig]{
+		Type: ModifierPreCodec,
+		T:    c,
+	})
+}
+
 // EpochToTimeModifierConfig is used to convert epoch seconds as uint64 fields on-chain to time.Time
 type EpochToTimeModifierConfig struct {
 	Fields []string

pkg/codec/precodec.go

@@ -0,0 +1,115 @@
+package codec
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	"github.com/go-viper/mapstructure/v2"
+	"github.com/smartcontractkit/chainlink-common/pkg/types"
+)
+
+// PreCodec creates a modifier that will run a preliminary encoding/decoding step.
+// This is useful when wanting to move nested data as generic bytes.
+func NewPreCodec(fields map[string]string, codecFactory func(typeABI string) types.RemoteCodec) Modifier {
+	m := &preCodec{
+		modifierBase: modifierBase[string]{
+			fields:           fields,
+			onToOffChainType: map[reflect.Type]reflect.Type{},
+			offToOnChainType: map[reflect.Type]reflect.Type{},
+		},
+		codecFactory: codecFactory,
+		codecs:       make(map[string]types.RemoteCodec),
+	}
+
+	// set up a codec each unique ABI
+	for _, abi := range fields {
+		if _, ok := m.codecs[abi]; ok {
+			continue
+		}
+		m.codecs[abi] = codecFactory(abi)
+	}
+
+	m.modifyFieldForInput = func(_ string, field *reflect.StructField, _ string, abi string) error {
+		if field.Type != reflect.SliceOf(reflect.TypeFor[uint8]()) {
+			return fmt.Errorf("can only decode []byte from on-chain: %s", field.Type)
+		}
+
+		codec, ok := m.codecs[abi]
+		if !ok || codec == nil {
+			return fmt.Errorf("codec not found for abi: '%s'", abi)
+		}
+
+		newType, err := codec.CreateType("", false)
+		if err != nil {
+			return err
+		}
+		field.Type = reflect.TypeOf(newType)
+
+		return nil
+	}
+
+	return m
+}
+
+type preCodec struct {
+	modifierBase[string]
+	codecFactory func(typeABI string) types.RemoteCodec
+	codecs       map[string]types.RemoteCodec
+}
+
+func (pc *preCodec) TransformToOffChain(onChainValue any, _ string) (any, error) {
+	allHooks := make([]mapstructure.DecodeHookFunc, 1)
+	allHooks[0] = hardCodeManyHook
+
+	return transformWithMaps(onChainValue, pc.onToOffChainType, pc.fields, pc.decodeFieldMapAction, allHooks...)
+}
+
+func (pc *preCodec) decodeFieldMapAction(extractMap map[string]any, key string, abi string) error {
+	_, exists := extractMap[key]
+	if !exists {
+		return fmt.Errorf("field %s does not exist", key)
+	}
+
+	codec, ok := pc.codecs[abi]
+	if !ok || codec == nil {
+		return fmt.Errorf("codec not found for abi: '%s'", abi)
+	}
+
+	to, err := codec.CreateType("", false)
+	if err != nil {
+		return err
+	}
+	err = codec.Decode(context.TODO(), extractMap[key].([]byte), to, "")
+	if err != nil {
+		return err
+	}
+	extractMap[key] = to
+	return nil
+}
+
+func (pc *preCodec) TransformToOnChain(offChainValue any, _ string) (any, error) {
+	allHooks := make([]mapstructure.DecodeHookFunc, 1)
+	allHooks[0] = hardCodeManyHook
+
+	return transformWithMaps(offChainValue, pc.offToOnChainType, pc.fields, pc.encodeFieldMapAction, allHooks...)
+}
+
+func (pc *preCodec) encodeFieldMapAction(extractMap map[string]any, key string, abi string) error {
+	_, exists := extractMap[key]
+	if !exists {
+		return fmt.Errorf("field %s does not exist", key)
+	}
+
+	codec, ok := pc.codecs[abi]
+	if !ok || codec == nil {
+		return fmt.Errorf("codec not found for abi: '%s'", abi)
+	}
+
+	encoded, err := codec.Encode(context.TODO(), extractMap[key], "")
+	if err != nil {
+		return err
+	}
+	extractMap[key] = encoded
+	return nil
+}

pkg/codec/precodec_test.go

@@ -0,0 +1,183 @@
+package codec_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/codec"
+	"github.com/smartcontractkit/chainlink-common/pkg/types"
+)
+
+var _ types.RemoteCodec = &ExampleCodec{}
+
+type ExampleCodec struct {
+	offChainType any
+}
+
+func (ec ExampleCodec) Encode(_ context.Context, item any, _ string) ([]byte, error) {
+	return json.Marshal(item)
+}
+
+func (ec ExampleCodec) GetMaxEncodingSize(_ context.Context, n int, _ string) (int, error) {
+	// not used in the example
+	return math.MaxInt32, nil
+}
+
+func (ec ExampleCodec) Decode(_ context.Context, raw []byte, into any, _ string) error {
+	err := json.Unmarshal(raw, into)
+	if err != nil {
+		return fmt.Errorf("%w: %w", types.ErrInvalidType, err)
+	}
+	return nil
+}
+
+func (ec ExampleCodec) GetMaxDecodingSize(ctx context.Context, n int, _ string) (int, error) {
+	// not used in the example
+	return math.MaxInt32, nil
+}
+
+func (ec ExampleCodec) CreateType(_ string, _ bool) (any, error) {
+	// parameters here are unused in the example, but can be used to determine what type to expect.
+	// this allows remote execution to know how to decode the incoming message
+	// and for [codec.NewModifierCodec] to know what type to expect for intermediate phases.
+	return ec.offChainType, nil
+}
+
+type testStructOff struct {
+	Ask int
+	Bid int
+}
+
+type nestedTestStructOff struct {
+	Report    testStructOff
+	FeedID    [32]byte
+	Timestamp int64
+}
+
+type deepNestedTestStructOff struct {
+	Reports []nestedTestStructOff
+}
+
+type testStructOn struct {
+	Ask []byte
+	Bid int
+}
+
+type nestedTestStructOn struct {
+	Report    []byte
+	FeedID    [32]byte
+	Timestamp int64
+}
+
+type deepNestedTestStructOn struct {
+	Reports []nestedTestStructOn
+}
+
+const (
+	TestStructOffABI = "uint256 Ask, uint256 Bid"
+)
+
+func TestPreCodec(t *testing.T) {
+	t.Parallel()
+
+	preCodec := codec.NewPreCodec(
+		map[string]string{"Ask": "uint256"},
+		func(typeABI string) types.RemoteCodec {
+			return ExampleCodec{offChainType: int(0)}
+		},
+	)
+	nestedPreCodec := codec.NewPreCodec(
+		map[string]string{"Report": TestStructOffABI},
+		func(typeABI string) types.RemoteCodec { return ExampleCodec{offChainType: testStructOff{}} },
+	)
+	deepNestedPreCodec := codec.NewPreCodec(
+		map[string]string{"Reports.Report": TestStructOffABI},
+		func(typeABI string) types.RemoteCodec { return ExampleCodec{offChainType: testStructOff{}} },
+	)
+	invalidPreCodec := codec.NewPreCodec(
+		map[string]string{"Unknown": TestStructOffABI},
+		func(typeABI string) types.RemoteCodec { return ExampleCodec{offChainType: testStructOff{}} },
+	)
+
+	t.Run("RetypeToOffChain converts type to codec.CreateType type", func(t *testing.T) {
+		offChainType, err := preCodec.RetypeToOffChain(reflect.TypeOf(testStructOn{}), "")
+
+		require.NoError(t, err)
+
+		require.Equal(t, 2, offChainType.NumField())
+		field0 := offChainType.Field(0)
+		assert.Equal(t, "Ask", field0.Name)
+		assert.Equal(t, reflect.TypeOf(int(0)), field0.Type)
+		field1 := offChainType.Field(1)
+		assert.Equal(t, "Bid", field1.Name)
+		assert.Equal(t, reflect.TypeOf(int(0)), field1.Type)
+	})
+
+	t.Run("RetypeToOffChain converts nested type to codec.CreateType type", func(t *testing.T) {
+		offChainType, err := nestedPreCodec.RetypeToOffChain(reflect.TypeOf(nestedTestStructOn{}), "")
+
+		require.NoError(t, err)
+
+		require.Equal(t, 3, offChainType.NumField())
+		field0 := offChainType.Field(0)
+		assert.Equal(t, "Report", field0.Name)
+		assert.Equal(t, reflect.TypeOf(testStructOff{}), field0.Type)
+		field1 := offChainType.Field(1)
+		assert.Equal(t, "FeedID", field1.Name)
+		assert.Equal(t, reflect.TypeOf([32]byte{}), field1.Type)
+		field2 := offChainType.Field(2)
+		assert.Equal(t, "Timestamp", field2.Name)
+		assert.Equal(t, reflect.TypeOf(int64(0)), field2.Type)
+	})
+
+	t.Run("RetypeToOffChain converts deep nested type to codec.CreateType type", func(t *testing.T) {
+		offChainType, err := deepNestedPreCodec.RetypeToOffChain(reflect.TypeOf(deepNestedTestStructOn{}), "")
+
+		require.NoError(t, err)
+
+		reports, exists := offChainType.FieldByName("Reports")
+		assert.True(t, exists)
+		report := reports.Type.Elem()
+		require.Equal(t, 3, report.NumField())
+		field0 := report.Field(0)
+		assert.Equal(t, "Report", field0.Name)
+		assert.Equal(t, reflect.TypeOf(testStructOff{}), field0.Type)
+		field1 := report.Field(1)
+		assert.Equal(t, "FeedID", field1.Name)
+		assert.Equal(t, reflect.TypeOf([32]byte{}), field1.Type)
+		field2 := report.Field(2)
+		assert.Equal(t, "Timestamp", field2.Name)
+		assert.Equal(t, reflect.TypeOf(int64(0)), field2.Type)
+	})
+
+	t.Run("RetypeToOffChain only works on byte arrays", func(t *testing.T) {
+		_, err := preCodec.RetypeToOffChain(reflect.TypeOf(testStructOff{}), "")
+		require.Error(t, err)
+		assert.Equal(t, err.Error(), "can only decode []byte from on-chain: int")
+	})
+
+	t.Run("RetypeToOffChain only works with valid path", func(t *testing.T) {
+		_, err := invalidPreCodec.RetypeToOffChain(reflect.TypeOf(testStructOn{}), "")
+		require.Error(t, err)
+		assert.Equal(t, err.Error(), "invalid type: cannot find Unknown")
+	})
+}
+
+func assertPreCodecTransform(t *testing.T, offChainType reflect.Type) {
+	require.Equal(t, 2, offChainType.NumField())
+	field0 := offChainType.Field(0)
+	fmt.Println(offChainType)
+	fmt.Println(field0.Type)
+	assert.Equal(t, "Ask", field0.Name)
+	assert.Equal(t, reflect.TypeOf(int(0)), field0.Type)
+	field1 := offChainType.Field(1)
+	assert.Equal(t, "Bid", field1.Name)
+	assert.Equal(t, reflect.TypeOf(int(0)), field1.Type)
+}