(refactor): Filter out destination chain bridgeable tokens that are n…

…ot configured on pricegetter (#686)

## Motivation
Self service token pools will enable many new TransferTokens to be used
across CCIP, most of which may not have a readily available price.

## Context
The current procedure (as of [batched price
updates](#623)) is for a
single lane to be designated as the "leader lane". It reports all prices
that other lanes use. Other lanes have their price reporting disabled.
To enable this, the leader lane has all supported tokens configured in
the CommitJobSpec.

The source of truth for which tokens are supported comes from on-chain:
queried from the leader lane's destination price registry (FeeTokens)
and from each OffRamp's `supportedTokens`. The combined set is given to
the `price getter`, and if the number of resulting prices is different
from the CommitJobSpec's tokens then the observation is thrown out.

## Solution

### _For v1.4 (current) contracts:_
Use the `CommitJobSpec`'s configured tokens as the source of truth for
which TransferTokens need a price update. Filter out TransferTokens that
are not configured on the `pricegetter` before they reach
`pricegetter.TokenPricesUSD()`.

Risks:
- Compatability with [shared job
specs](#683)

### _For v1.5 (next) contracts:_
Return to on-chain being the source of truth for which tokens need a
price update.
The list of TransferTokens that need pricing will be created from
querying the OnRamps for tokens that have either BPS set and/or
Aggregate Rate Limits. FeeTokens will still be taken from the
destination Price Registry.

core/services/ocr2/plugins/ccip/ccipcommit/ocr2.go

@@ -184,7 +184,9 @@ func (r *CommitReportingPlugin) observePriceUpdates(
 		return nil, nil, nil
 	}
 
-	sortedChainTokens, err := ccipcommon.GetSortedChainTokens(ctx, r.offRampReaders, r.destPriceRegistryReader)
+	sortedChainTokens, filteredChainTokens, err := ccipcommon.GetFilteredSortedChainTokens(ctx, r.offRampReaders, r.destPriceRegistryReader, r.priceGetter)
+	lggr.Debugw("Filtered bridgeable tokens with no configured price getter", filteredChainTokens)
+
 	if err != nil {
 		return nil, nil, fmt.Errorf("get destination tokens: %w", err)
 	}
@@ -324,7 +326,7 @@ func (r *CommitReportingPlugin) Report(ctx context.Context, epochAndRound types.
 
 	parsableObservations := ccip.GetParsableObservations[ccip.CommitObservation](lggr, observations)
 
-	sortedChainTokens, err := ccipcommon.GetSortedChainTokens(ctx, r.offRampReaders, r.destPriceRegistryReader)
+	sortedChainTokens, _, err := ccipcommon.GetFilteredSortedChainTokens(ctx, r.offRampReaders, r.destPriceRegistryReader, r.priceGetter)
 	if err != nil {
 		return false, nil, fmt.Errorf("get destination tokens: %w", err)
 	}

core/services/ocr2/plugins/ccip/ccipcommit/ocr2_test.go

@@ -211,6 +211,10 @@ func TestCommitReportingPlugin_Observation(t *testing.T) {
 			if !tc.priceReportingDisabled && len(tc.tokenPrices) > 0 {
 				queryTokens := ccipcommon.FlattenUniqueSlice([]cciptypes.Address{sourceNativeTokenAddr}, destTokens)
 				priceGet.On("TokenPricesUSD", mock.Anything, queryTokens).Return(tc.tokenPrices, nil)
+				priceGet.On("FilterConfiguredTokens", mock.Anything, destTokens).Return([]cciptypes.Address{
+					bridgedTokens[0],
+					bridgedTokens[1],
+				}, []cciptypes.Address{}, nil)
 			}
 
 			gasPriceEstimator := prices.NewMockGasPriceEstimatorCommit(t)
@@ -303,6 +307,9 @@ func TestCommitReportingPlugin_Report(t *testing.T) {
 		chainHealthcheck := ccipcachemocks.NewChainHealthcheck(t)
 		chainHealthcheck.On("IsHealthy", ctx).Return(true, nil).Maybe()
 		p.chainHealthcheck = chainHealthcheck
+		pricegetter := pricegetter.NewMockPriceGetter(t)
+		pricegetter.On("FilterConfiguredTokens", mock.Anything, mock.Anything).Return([]cciptypes.Address{}, []cciptypes.Address{}, nil)
+		p.priceGetter = pricegetter
 
 		o := ccip.CommitObservation{Interval: cciptypes.CommitStoreInterval{Min: 1, Max: 1}, SourceGasPriceUSD: big.NewInt(0)}
 		obs, err := o.Marshal()
@@ -510,7 +517,7 @@ func TestCommitReportingPlugin_Report(t *testing.T) {
 				gasPriceEstimator.On("Deviates", mock.Anything, mock.Anything, mock.Anything).Return(false, nil)
 			}
 
-			var destTokens []cciptypes.Address
+			destTokens := []cciptypes.Address{}
 			for tk := range tc.tokenDecimals {
 				destTokens = append(destTokens, tk)
 			}
@@ -559,6 +566,9 @@ func TestCommitReportingPlugin_Report(t *testing.T) {
 			healthCheck := ccipcachemocks.NewChainHealthcheck(t)
 			healthCheck.On("IsHealthy", ctx).Return(true, nil)
 
+			pricegetter := pricegetter.NewMockPriceGetter(t)
+			pricegetter.On("FilterConfiguredTokens", mock.Anything, destTokens).Return(destTokens, []cciptypes.Address{}, nil)
+
 			p := &CommitReportingPlugin{}
 			p.lggr = logger.TestLogger(t)
 			p.destPriceRegistryReader = destPriceRegistryReader
@@ -569,6 +579,7 @@ func TestCommitReportingPlugin_Report(t *testing.T) {
 			p.offchainConfig.GasPriceHeartBeat = gasPriceHeartBeat.Duration()
 			p.commitStoreReader = commitStoreReader
 			p.F = tc.f
+			p.priceGetter = pricegetter
 			p.metricsCollector = ccip.NoopMetricsCollector
 			p.offchainConfig.PriceReportingDisabled = tc.priceReportingDisabled
 			p.chainHealthcheck = healthCheck

core/services/ocr2/plugins/ccip/internal/ccipcommon/shortcuts.go

@@ -33,22 +33,56 @@ type BackfillArgs struct {
 	SourceStartBlock, DestStartBlock uint64
 }
 
+// GetChainTokens returns union of all tokens supported on the destination chain, including fee tokens from the provided price registry
+// and the bridgeable tokens from all the offRamps living on the chain.
 func GetSortedChainTokens(ctx context.Context, offRamps []ccipdata.OffRampReader, priceRegistry cciptypes.PriceRegistryReader) (chainTokens []cciptypes.Address, err error) {
-	return getSortedChainTokensWithBatchLimit(ctx, offRamps, priceRegistry, offRampBatchSizeLimit)
+	destFeeTokens, destBridgeableTokens, err := getTokensWithBatchLimit(ctx, offRamps, priceRegistry, offRampBatchSizeLimit)
+	if err != nil {
+		return nil, fmt.Errorf("get tokens with batch limit: %w", err)
+	}
+	// fee token can overlap with bridgeable tokens
+	// we need to dedup them to arrive at chain token set
+	return flattenedAndSortedChainTokens(destFeeTokens, destBridgeableTokens), nil
 }
 
-// GetChainTokens returns union of all tokens supported on the destination chain, including fee tokens from the provided price registry
-// and the bridgeable tokens from all the offRamps living on the chain.
-func getSortedChainTokensWithBatchLimit(ctx context.Context, offRamps []ccipdata.OffRampReader, priceRegistry cciptypes.PriceRegistryReader, batchSize int) (chainTokens []cciptypes.Address, err error) {
+// GetFilteredSortedChainTokens returns union of all tokens supported on the destination chain, including fee tokens from the provided price registry
+// and the bridgeable tokens from all the offRamps living on the chain. Bridgeable tokens are only included if they are configured on the pricegetter
+// Fee tokens are not filtered as they must always be priced
+func GetFilteredSortedChainTokens(ctx context.Context, offRamps []ccipdata.OffRampReader, priceRegistry cciptypes.PriceRegistryReader, priceGetter cciptypes.PriceGetter) (chainTokens []cciptypes.Address, excludedTokens []cciptypes.Address, err error) {
+	destFeeTokens, destBridgeableTokens, err := getTokensWithBatchLimit(ctx, offRamps, priceRegistry, offRampBatchSizeLimit)
+	if err != nil {
+		return nil, nil, fmt.Errorf("get tokens with batch limit: %w", err)
+	}
+
+	destTokensWithPrice, destTokensWithoutPrice, err := priceGetter.FilterConfiguredTokens(ctx, destBridgeableTokens)
+	if err != nil {
+		return nil, nil, fmt.Errorf("filter for priced tokens: %w", err)
+	}
+
+	return flattenedAndSortedChainTokens(destFeeTokens, destTokensWithPrice), destTokensWithoutPrice, nil
+}
+
+func flattenedAndSortedChainTokens(slices ...[]cciptypes.Address) (chainTokens []cciptypes.Address) {
+	// same token can be returned by multiple offRamps, and fee token can overlap with bridgeable tokens,
+	// we need to dedup them to arrive at chain token set
+	chainTokens = FlattenUniqueSlice(slices...)
+
+	// return the tokens in deterministic order to aid with testing and debugging
+	sort.Slice(chainTokens, func(i, j int) bool {
+		return chainTokens[i] < chainTokens[j]
+	})
+
+	return chainTokens
+}
+
+func getTokensWithBatchLimit(ctx context.Context, offRamps []ccipdata.OffRampReader, priceRegistry cciptypes.PriceRegistryReader, batchSize int) (destFeeTokens []cciptypes.Address, destBridgeableTokens []cciptypes.Address, err error) {
 	if batchSize == 0 {
-		return nil, fmt.Errorf("batch size must be greater than 0")
+		return nil, nil, fmt.Errorf("batch size must be greater than 0")
 	}
 
 	eg := new(errgroup.Group)
 	eg.SetLimit(batchSize)
 
-	var destFeeTokens []cciptypes.Address
-	var destBridgeableTokens []cciptypes.Address
 	mu := &sync.RWMutex{}
 
 	eg.Go(func() error {
@@ -75,19 +109,11 @@ func getSortedChainTokensWithBatchLimit(ctx context.Context, offRamps []ccipdata
 	}
 
 	if err := eg.Wait(); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	// same token can be returned by multiple offRamps, and fee token can overlap with bridgeable tokens,
-	// we need to dedup them to arrive at chain token set
-	chainTokens = FlattenUniqueSlice(destFeeTokens, destBridgeableTokens)
-
-	// return the tokens in deterministic order to aid with testing and debugging
-	sort.Slice(chainTokens, func(i, j int) bool {
-		return chainTokens[i] < chainTokens[j]
-	})
-
-	return chainTokens, nil
+	// same token can be returned by multiple offRamps
+	return destFeeTokens, flattenedAndSortedChainTokens(destBridgeableTokens), nil
 }
 
 // GetDestinationTokens returns the destination chain fee tokens from the provided price registry

core/services/ocr2/plugins/ccip/internal/ccipcommon/shortcuts_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 
 	cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccip"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipcalc"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata"
 	ccipdatamocks "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata/mocks"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/pricegetter"
 )
 
 func TestGetMessageIDsAsHexString(t *testing.T) {
@@ -141,23 +143,120 @@ func TestGetChainTokens(t *testing.T) {
 	}
 }
 
+func TestGetFilteredChainTokens(t *testing.T) {
+	const numTokens = 6
+	var tokens []cciptypes.Address
+	for i := 0; i < numTokens; i++ {
+		tokens = append(tokens, ccipcalc.EvmAddrToGeneric(utils.RandomAddress()))
+	}
+
+	testCases := []struct {
+		name                   string
+		feeTokens              []cciptypes.Address
+		destTokens             [][]cciptypes.Address
+		expectedChainTokens    []cciptypes.Address
+		expectedFilteredTokens []cciptypes.Address
+	}{
+		{
+			name:                   "empty",
+			feeTokens:              []cciptypes.Address{},
+			destTokens:             [][]cciptypes.Address{{}},
+			expectedChainTokens:    []cciptypes.Address{},
+			expectedFilteredTokens: []cciptypes.Address{},
+		},
+		{
+			name:      "single offRamp",
+			feeTokens: []cciptypes.Address{tokens[0]},
+			destTokens: [][]cciptypes.Address{
+				{tokens[1], tokens[2], tokens[3]},
+			},
+			expectedChainTokens:    []cciptypes.Address{tokens[0], tokens[1], tokens[2], tokens[3]},
+			expectedFilteredTokens: []cciptypes.Address{tokens[4], tokens[5]},
+		},
+		{
+			name:      "multiple offRamps with distinct tokens",
+			feeTokens: []cciptypes.Address{tokens[0]},
+			destTokens: [][]cciptypes.Address{
+				{tokens[1], tokens[2]},
+				{tokens[3], tokens[4]},
+				{tokens[5]},
+			},
+			expectedChainTokens:    []cciptypes.Address{tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5]},
+			expectedFilteredTokens: []cciptypes.Address{},
+		},
+		{
+			name:      "overlapping tokens",
+			feeTokens: []cciptypes.Address{tokens[0]},
+			destTokens: [][]cciptypes.Address{
+				{tokens[0], tokens[1], tokens[2], tokens[3]},
+				{tokens[0], tokens[2], tokens[3], tokens[4], tokens[5]},
+				{tokens[5]},
+			},
+			expectedChainTokens:    []cciptypes.Address{tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5]},
+			expectedFilteredTokens: []cciptypes.Address{},
+		},
+		{
+			name:      "unconfigured tokens",
+			feeTokens: []cciptypes.Address{tokens[0]},
+			destTokens: [][]cciptypes.Address{
+				{tokens[0], tokens[1], tokens[2], tokens[3]},
+				{tokens[0], tokens[2], tokens[3], tokens[4], tokens[5]},
+				{tokens[5]},
+			},
+			expectedChainTokens:    []cciptypes.Address{tokens[0], tokens[1], tokens[2], tokens[3], tokens[4]},
+			expectedFilteredTokens: []cciptypes.Address{tokens[5]},
+		},
+	}
+
+	ctx := testutils.Context(t)
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+
+			priceRegistry := ccipdatamocks.NewPriceRegistryReader(t)
+			priceRegistry.On("GetFeeTokens", ctx).Return(tc.feeTokens, nil).Once()
+
+			priceGet := pricegetter.NewMockPriceGetter(t)
+			priceGet.On("FilterConfiguredTokens", mock.Anything, mock.Anything).Return(tc.expectedChainTokens, tc.expectedFilteredTokens, nil)
+
+			var offRamps []ccipdata.OffRampReader
+			for _, destTokens := range tc.destTokens {
+				offRamp := ccipdatamocks.NewOffRampReader(t)
+				offRamp.On("GetTokens", ctx).Return(cciptypes.OffRampTokens{DestinationTokens: destTokens}, nil).Once()
+				offRamps = append(offRamps, offRamp)
+			}
+
+			chainTokens, filteredTokens, err := GetFilteredSortedChainTokens(ctx, offRamps, priceRegistry, priceGet)
+			assert.NoError(t, err)
+
+			sort.Slice(tc.expectedChainTokens, func(i, j int) bool {
+				return tc.expectedChainTokens[i] < tc.expectedChainTokens[j]
+			})
+			assert.Equal(t, tc.expectedChainTokens, chainTokens)
+			assert.Equal(t, tc.expectedFilteredTokens, filteredTokens)
+		})
+	}
+}
+
 func TestGetChainTokensWithBatchLimit(t *testing.T) {
 	numTokens := 100
+	numFeeTokens := 10
 	var tokens []cciptypes.Address
 	for i := 0; i < numTokens; i++ {
 		tokens = append(tokens, ccipcalc.EvmAddrToGeneric(utils.RandomAddress()))
 	}
 
-	expectedTokens := make([]cciptypes.Address, numTokens)
-	copy(expectedTokens, tokens)
-	sort.Slice(expectedTokens, func(i, j int) bool {
-		return expectedTokens[i] < expectedTokens[j]
+	expectedFeeTokens := make([]cciptypes.Address, numFeeTokens)
+	copy(expectedFeeTokens, tokens[0:numFeeTokens])
+	expectedBridgeableTokens := make([]cciptypes.Address, numTokens)
+	copy(expectedBridgeableTokens, tokens)
+	sort.Slice(expectedBridgeableTokens, func(i, j int) bool {
+		return expectedBridgeableTokens[i] < expectedBridgeableTokens[j]
 	})
 
 	testCases := []struct {
 		name        string
 		batchSize   int
-		numOffRamps uint
+		numOffRamps int
 		expectError bool
 	}{
 		{
@@ -203,23 +302,24 @@ func TestGetChainTokensWithBatchLimit(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 
 			priceRegistry := ccipdatamocks.NewPriceRegistryReader(t)
-			priceRegistry.On("GetFeeTokens", ctx).Return(tokens[0:10], nil).Maybe()
+			priceRegistry.On("GetFeeTokens", ctx).Return(expectedFeeTokens, nil).Maybe()
 
 			var offRamps []ccipdata.OffRampReader
-			for i := 0; i < int(tc.numOffRamps); i++ {
+			for i := 0; i < tc.numOffRamps; i++ {
 				offRamp := ccipdatamocks.NewOffRampReader(t)
 				offRamp.On("GetTokens", ctx).Return(cciptypes.OffRampTokens{DestinationTokens: tokens[i%numTokens:]}, nil).Maybe()
 				offRamps = append(offRamps, offRamp)
 			}
 
-			chainTokens, err := getSortedChainTokensWithBatchLimit(ctx, offRamps, priceRegistry, tc.batchSize)
+			destFeeTokens, destBridgeableTokens, err := getTokensWithBatchLimit(ctx, offRamps, priceRegistry, tc.batchSize)
 			if tc.expectError {
 				assert.Error(t, err)
 				return
 			}
 
 			assert.NoError(t, err)
-			assert.Equal(t, expectedTokens, chainTokens)
+			assert.Equal(t, expectedFeeTokens, destFeeTokens)
+			assert.Equal(t, expectedBridgeableTokens, destBridgeableTokens)
 		})
 	}
 }

core/services/ocr2/plugins/ccip/internal/pricegetter/evm.go

@@ -74,6 +74,28 @@ func NewDynamicPriceGetter(cfg config.DynamicPriceGetterConfig, evmClients map[u
 	return &priceGetter, nil
 }
 
+// FilterForConfiguredTokens implements the PriceGetter interface.
+// It filters a list of token addresses for only those that have a price resolution rule configured on the PriceGetterConfig
+func (d *DynamicPriceGetter) FilterConfiguredTokens(ctx context.Context, tokens []cciptypes.Address) (configured []cciptypes.Address, unconfigured []cciptypes.Address, err error) {
+	configured = []cciptypes.Address{}
+	unconfigured = []cciptypes.Address{}
+	for _, tk := range tokens {
+		evmAddr, err := ccipcalc.GenericAddrToEvm(tk)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if _, isAgg := d.cfg.AggregatorPrices[evmAddr]; isAgg {
+			configured = append(configured, tk)
+		} else if _, isStatic := d.cfg.StaticPrices[evmAddr]; isStatic {
+			configured = append(configured, tk)
+		} else {
+			unconfigured = append(unconfigured, tk)
+		}
+	}
+	return configured, unconfigured, nil
+}
+
 // TokenPricesUSD implements the PriceGetter interface.
 // It returns static prices stored in the price getter, and batch calls to aggregators (on per chain) for aggregator-based prices.
 func (d *DynamicPriceGetter) TokenPricesUSD(ctx context.Context, tokens []cciptypes.Address) (map[cciptypes.Address]*big.Int, error) {

core/services/ocr2/plugins/ccip/internal/pricegetter/evm_test.go

@@ -62,10 +62,17 @@ func TestDynamicPriceGetter(t *testing.T) {
 			}
 			require.NoError(t, err)
 			ctx := testutils.Context(t)
+			// Check configured token
+			unconfiguredTk := cciptypes.Address(utils.RandomAddress().String())
+			cfgTokens, uncfgTokens, err := pg.FilterConfiguredTokens(ctx, []cciptypes.Address{unconfiguredTk})
+			require.NoError(t, err)
+			assert.Equal(t, []cciptypes.Address{}, cfgTokens)
+			assert.Equal(t, []cciptypes.Address{unconfiguredTk}, uncfgTokens)
 			// Build list of tokens to query.
 			tokens := make([]cciptypes.Address, 0, len(test.param.expectedTokenPrices))
 			for tk := range test.param.expectedTokenPrices {
-				tokens = append(tokens, cciptypes.Address(tk.String()))
+				tokenAddr := cciptypes.Address(tk.String())
+				tokens = append(tokens, tokenAddr)
 			}
 			prices, err := pg.TokenPricesUSD(ctx, tokens)
 			if test.param.priceResolutionErrorExpected {