From b5f1c48fe0b5115e5b472d8c5e546db4a8158975 Mon Sep 17 00:00:00 2001 From: connorwstein Date: Wed, 18 Sep 2024 16:42:31 -0400 Subject: [PATCH] Keep multiclient simple --- .../ccip/ccip_integration_tests/helpers.go | 2 - integration-tests/deployment/evm_kmsclient.go | 42 +++-- integration-tests/deployment/multiclient.go | 174 ++++-------------- .../deployment/multiclient_test.go | 29 +++ integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- 6 files changed, 98 insertions(+), 155 deletions(-) create mode 100644 integration-tests/deployment/multiclient_test.go diff --git a/core/capabilities/ccip/ccip_integration_tests/helpers.go b/core/capabilities/ccip/ccip_integration_tests/helpers.go index 25baddfb48e..4670333e391 100644 --- a/core/capabilities/ccip/ccip_integration_tests/helpers.go +++ b/core/capabilities/ccip/ccip_integration_tests/helpers.go @@ -406,7 +406,6 @@ func (h *homeChain) AddNodes( p2pIDs [][32]byte, capabilityIDs [][32]byte, ) { - // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail sortP2PIDS(p2pIDs) var nodeParams []kcr.CapabilitiesRegistryNodeParams for _, p2pID := range p2pIDs { @@ -430,7 +429,6 @@ func AddChainConfig( p2pIDs [][32]byte, f uint8, ) ccip_config.CCIPConfigTypesChainConfigInfo { - // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail sortP2PIDS(p2pIDs) // First Add ChainConfig that includes all p2pIDs as readers encodedExtraChainConfig, err := chainconfig.EncodeChainConfig(chainconfig.ChainConfig{ diff --git a/integration-tests/deployment/evm_kmsclient.go b/integration-tests/deployment/evm_kmsclient.go index b7fb1b6a4b9..8e2de18d789 100644 --- a/integration-tests/deployment/evm_kmsclient.go +++ b/integration-tests/deployment/evm_kmsclient.go @@ -44,24 +44,44 @@ type asn1ECDSASig struct { S asn1.RawValue } +// TODO: Mockery gen then test with a regular eth key behind the interface. type KMSClient interface { GetPublicKey(input *kms.GetPublicKeyInput) (*kms.GetPublicKeyOutput, error) Sign(input *kms.SignInput) (*kms.SignOutput, error) } -type evmKMSClient struct { +type KMS struct { + KmsDeployerKeyId string + KmsDeployerKeyRegion string + AwsProfileName string +} + +func NewKMSClient(config KMS) KMSClient { + if config.KmsDeployerKeyId != "" && config.KmsDeployerKeyRegion != "" { + var awsSessionFn AwsSessionFn + if config.AwsProfileName != "" { + awsSessionFn = awsSessionFromProfileFn + } else { + awsSessionFn = awsSessionFromEnvVarsFn + } + return kms.New(awsSessionFn(config)) + } + return nil +} + +type EVMKMSClient struct { Client KMSClient KeyID string } -func NewEVMKMSClient(client KMSClient, keyID string) *evmKMSClient { - return &evmKMSClient{ +func NewEVMKMSClient(client KMSClient, keyID string) *EVMKMSClient { + return &EVMKMSClient{ Client: client, KeyID: keyID, } } -func (c *evmKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { +func (c *EVMKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { ecdsaPublicKey, err := c.GetECDSAPublicKey() if err != nil { return nil, err @@ -110,7 +130,7 @@ func (c *evmKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) } // GetECDSAPublicKey retrieves the public key from KMS and converts it to its ECDSA representation. -func (c *evmKMSClient) GetECDSAPublicKey() (*ecdsa.PublicKey, error) { +func (c *EVMKMSClient) GetECDSAPublicKey() (*ecdsa.PublicKey, error) { getPubKeyOutput, err := c.Client.GetPublicKey(&kms.GetPublicKeyInput{ KeyId: aws.String(c.KeyID), }) @@ -187,23 +207,23 @@ func padTo32Bytes(buffer []byte) []byte { return buffer } -type AwsSessionFn func(config Config) *session.Session +type AwsSessionFn func(config KMS) *session.Session -var awsSessionFromEnvVarsFn = func(config Config) *session.Session { +var awsSessionFromEnvVarsFn = func(config KMS) *session.Session { return session.Must( session.NewSession(&aws.Config{ - Region: aws.String(config.EnvConfig.KmsDeployerKeyRegion), + Region: aws.String(config.KmsDeployerKeyRegion), CredentialsChainVerboseErrors: aws.Bool(true), })) } -var awsSessionFromProfileFn = func(config Config) *session.Session { +var awsSessionFromProfileFn = func(config KMS) *session.Session { return session.Must( session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, - Profile: config.EnvConfig.AwsProfileName, + Profile: config.AwsProfileName, Config: aws.Config{ - Region: aws.String(config.EnvConfig.KmsDeployerKeyRegion), + Region: aws.String(config.KmsDeployerKeyRegion), CredentialsChainVerboseErrors: aws.Bool(true), }, })) diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go index 4d3d90c87a0..927df43e9ae 100644 --- a/integration-tests/deployment/multiclient.go +++ b/integration-tests/deployment/multiclient.go @@ -4,173 +4,67 @@ import ( "context" "fmt" "math/big" - "os" "time" "github.com/avast/retry-go/v4" - "github.com/aws/aws-sdk-go/service/kms" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" - "github.com/pelletier/go-toml/v2" - "github.com/smartcontractkit/chainlink-testing-framework/seth" + "github.com/pkg/errors" ) const ( - RPC_RETRY_ATTEMPTS = 10 - RPC_RETRY_DELAY = 1000 * time.Millisecond + RPC_DEFAULT_RETRY_ATTEMPTS = 10 + RPC_DEFAULT_RETRY_DELAY = 1000 * time.Millisecond ) -// MultiClient should comply with the OnchainClient interface -var _ OnchainClient = &MultiClient{} +type RetryConfig struct { + Attempts uint + Delay time.Duration +} -type MultiClient struct { - *ethclient.Client - backup []*ethclient.Client - // we will use Seth only for gas estimations, confirming and tracing the transactions, but for sending transactions we will use pure ethclient - // so that MultiClient conforms to the OnchainClient interface - SethClient *seth.Client - EvmKMSClient *evmKMSClient - chainId uint64 +func defaultRetryConfig() RetryConfig { + return RetryConfig{ + Attempts: RPC_DEFAULT_RETRY_ATTEMPTS, + Delay: RPC_DEFAULT_RETRY_DELAY, + } } type RPC struct { - RPCName string HTTPURL string - WSURL string + // TODO: ws support? } -type Config struct { - EnvConfig EnvConfig -} - -type EnvConfig struct { - TestWalletKey string - KmsDeployerKeyId string - KmsDeployerKeyRegion string - AwsProfileName string - EvmNetworks []EvmNetwork - // Seth-related - GethWrappersDirs []string - SethConfigFile string -} +// MultiClient should comply with the OnchainClient interface +var _ OnchainClient = &MultiClient{} -type EvmNetwork struct { - ChainID uint64 - EtherscanAPIKey string - EtherscanUrl string - RPCs []RPC +type MultiClient struct { + *ethclient.Client + Backups []*ethclient.Client + RetryConfig RetryConfig } -func initRpcClients(rpcs []RPC) (*ethclient.Client, []*ethclient.Client) { +func NewMultiClient(rpcs []RPC, opts ...func(client *MultiClient)) (*MultiClient, error) { if len(rpcs) == 0 { - panic("No RPCs provided") + return nil, fmt.Errorf("No RPCs provided, need at least one") } + var mc MultiClient clients := make([]*ethclient.Client, 0, len(rpcs)) - for _, rpc := range rpcs { client, err := ethclient.Dial(rpc.HTTPURL) if err != nil { - panic(err) + return nil, errors.Wrapf(err, "failed to dial %s", rpc.HTTPURL) } clients = append(clients, client) } - return clients[0], clients[1:] -} - -func NewMultiClientWithSeth(rpcs []RPC, chainId uint64, config Config) *MultiClient { - mainClient, backupClients := initRpcClients(rpcs) - mc := &MultiClient{ - Client: mainClient, - backup: backupClients, - chainId: chainId, - } - - sethClient, err := buildSethClient(rpcs[0].HTTPURL, chainId, config) - if err != nil { - panic(err) - } - - mc.SethClient = sethClient - mc.EvmKMSClient = initialiseKMSClient(config) - - return mc -} - -func buildSethClient(rpc string, chainId uint64, config Config) (*seth.Client, error) { - var sethClient *seth.Client - var err error - - // if config path is provided use the TOML file to configure Seth to provide maximum flexibility - if config.EnvConfig.SethConfigFile != "" { - sethConfig, readErr := readSethConfigFromFile(config.EnvConfig.SethConfigFile) - if readErr != nil { - return nil, readErr - } - - sethClient, err = seth.NewClientBuilderWithConfig(sethConfig). - UseNetworkWithChainId(chainId). - WithRpcUrl(rpc). - WithPrivateKeys([]string{config.EnvConfig.TestWalletKey}). - Build() - } else { - // if full flexibility is not needed we create a client with reasonable defaults - // if you need to further tweak them, please refer to https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/seth/README.md - sethClient, err = seth.NewClientBuilder(). - WithRpcUrl(rpc). - WithPrivateKeys([]string{config.EnvConfig.TestWalletKey}). - WithProtections(true, true, seth.MustMakeDuration(1*time.Minute)). - WithGethWrappersFolders(config.EnvConfig.GethWrappersDirs). - // Fast priority will add a 20% buffer on top of what the node suggests - // we will use last 20 block to estimate block congestion and further bump gas price suggested by the node - WithGasPriceEstimations(true, 20, seth.Priority_Fast). - Build() - } - - return sethClient, err -} - -func readSethConfigFromFile(configPath string) (*seth.Config, error) { - d, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } + mc.Client = clients[0] + mc.Backups = clients[1:] + mc.RetryConfig = defaultRetryConfig() - var sethConfig seth.Config - err = toml.Unmarshal(d, &sethConfig) - if err != nil { - return nil, err + for _, opt := range opts { + opt(&mc) } - - return &sethConfig, nil -} - -func initialiseKMSClient(config Config) *evmKMSClient { - if config.EnvConfig.KmsDeployerKeyId != "" && config.EnvConfig.KmsDeployerKeyRegion != "" { - var awsSessionFn AwsSessionFn - if config.EnvConfig.AwsProfileName != "" { - awsSessionFn = awsSessionFromProfileFn - } else { - awsSessionFn = awsSessionFromEnvVarsFn - } - return NewEVMKMSClient(kms.New(awsSessionFn(config)), config.EnvConfig.KmsDeployerKeyId) - } - return nil -} - -func (mc *MultiClient) GetKMSKey() *bind.TransactOpts { - kmsTxOpts, err := mc.EvmKMSClient.GetKMSTransactOpts(context.Background(), big.NewInt(int64(mc.chainId))) - if err != nil { - panic(err) - } - // nonce needs to be `nil` so that RPC node sets it, otherwise Seth would set it to whatever it was, when we requested the key - return mc.SethClient.NewTXOpts(seth.WithNonce(nil), seth.WithFrom(kmsTxOpts.From), seth.WithSignerFn(kmsTxOpts.Signer)) -} - -func (mc *MultiClient) GetTestWalletKey() *bind.TransactOpts { - // nonce needs to be `nil` so that RPC node sets it, otherwise Seth would set it to whatever it was, when we requested the key - return mc.SethClient.NewTXOpts(seth.WithNonce(nil)) + return &mc, nil } func (mc *MultiClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { @@ -211,18 +105,20 @@ func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address) (uin func (mc *MultiClient) retryWithBackups(op func(*ethclient.Client) error) error { var err error - for _, client := range append([]*ethclient.Client{mc.Client}, mc.backup...) { + for _, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) { err2 := retry.Do(func() error { err = op(client) if err != nil { - fmt.Printf(" [MultiClient RPC] Retrying with new client, error: %v\n", err) + // TODO: logger? + fmt.Printf("Error %v with client %v\n", err, client) return err } return nil - }, retry.Attempts(RPC_RETRY_ATTEMPTS), retry.Delay(RPC_RETRY_DELAY)) + }, retry.Attempts(mc.RetryConfig.Attempts), retry.Delay(mc.RetryConfig.Delay)) if err2 == nil { return nil } + fmt.Println("Client %v failed, trying next client", client) } - return err + return errors.Wrapf(err, "All backup clients %v failed", mc.Backups) } diff --git a/integration-tests/deployment/multiclient_test.go b/integration-tests/deployment/multiclient_test.go new file mode 100644 index 00000000000..40b0b8e6946 --- /dev/null +++ b/integration-tests/deployment/multiclient_test.go @@ -0,0 +1,29 @@ +package deployment + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMultiClient(t *testing.T) { + // Expect an error if no RPCs supplied. + _, err := NewMultiClient([]RPC{}) + require.Error(t, err) + + // Expect defaults to be set if not provided. + mc, err := NewMultiClient([]RPC{{HTTPURL: "http://localhost:8545"}}) + require.NoError(t, err) + assert.Equal(t, mc.RetryConfig.Attempts, uint(RPC_DEFAULT_RETRY_ATTEMPTS)) + assert.Equal(t, mc.RetryConfig.Delay, RPC_DEFAULT_RETRY_DELAY) + + // Expect second client to be set as backup. + mc, err = NewMultiClient([]RPC{ + {HTTPURL: "http://localhost:8545"}, + {HTTPURL: "http://localhost:8546"}, + }) + require.NoError(t, err) + require.Equal(t, len(mc.Backups), 1) + assert.Equal(t, mc.Backups[0], "http://localhost:8546") +} diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 673191e145b..adbc5d36f9a 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240916150342-36cb47701edf github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 - github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 + github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index ee64962cb1c..708f9726936 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1417,8 +1417,8 @@ github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 h1:Owb1MQZn0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5/go.mod h1:hS4yNF94C1lkS9gvtFXW8Km8K9NzGeR20aNfkqo5qbE= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 h1:/wGR8PUytZBze4DPnhzF+V7IVl/EslDmDC2SsamXjqw= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 h1:gfhfTn7HkbUHNooSF3c9vzQyN8meWJVGt6G/pNUbpYk= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0/go.mod h1:tqajhpUJA/9OaMCLitghBXjAgqYO4i27St0F4TUO3+M= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs=