From 1a26acd5515ddeaf14e17f98cfbf474249d05157 Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Wed, 13 Dec 2023 08:35:28 -0600 Subject: [PATCH] fix health monitoring (#11558) --- core/chains/evm/gas/models.go | 31 +++- core/chains/evm/gas/models_test.go | 71 +++++---- core/chains/evm/txmgr/broadcaster_test.go | 12 +- core/chains/evm/txmgr/confirmer_test.go | 13 +- core/services/chainlink/relayer_factory.go | 6 +- core/services/relay/evm/evm.go | 4 +- core/web/health_controller.go | 11 +- core/web/health_controller_test.go | 70 ++++++++- core/web/testdata/body/health.json | 166 +++++++++++++++++++++ 9 files changed, 333 insertions(+), 51 deletions(-) create mode 100644 core/web/testdata/body/health.json diff --git a/core/chains/evm/gas/models.go b/core/chains/evm/gas/models.go index c7476d58ba4..8d977df0991 100644 --- a/core/chains/evm/gas/models.go +++ b/core/chains/evm/gas/models.go @@ -71,19 +71,31 @@ func NewEstimator(lggr logger.Logger, ethClient evmclient.Client, cfg Config, ge if rollups.IsRollupWithL1Support(cfg.ChainType()) { l1Oracle = rollups.NewL1GasPriceOracle(lggr, ethClient, cfg.ChainType()) } + var newEstimator func(logger.Logger) EvmEstimator switch s { case "Arbitrum": - return NewWrappedEvmEstimator(NewArbitrumEstimator(lggr, geCfg, ethClient, ethClient), df, l1Oracle) + newEstimator = func(l logger.Logger) EvmEstimator { + return NewArbitrumEstimator(lggr, geCfg, ethClient, ethClient) + } case "BlockHistory": - return NewWrappedEvmEstimator(NewBlockHistoryEstimator(lggr, ethClient, cfg, geCfg, bh, *ethClient.ConfiguredChainID()), df, l1Oracle) + newEstimator = func(l logger.Logger) EvmEstimator { + return NewBlockHistoryEstimator(lggr, ethClient, cfg, geCfg, bh, *ethClient.ConfiguredChainID()) + } case "FixedPrice": - return NewWrappedEvmEstimator(NewFixedPriceEstimator(geCfg, bh, lggr), df, l1Oracle) + newEstimator = func(l logger.Logger) EvmEstimator { + return NewFixedPriceEstimator(geCfg, bh, lggr) + } case "L2Suggested", "SuggestedPrice": - return NewWrappedEvmEstimator(NewSuggestedPriceEstimator(lggr, ethClient), df, l1Oracle) + newEstimator = func(l logger.Logger) EvmEstimator { + return NewSuggestedPriceEstimator(lggr, ethClient) + } default: lggr.Warnf("GasEstimator: unrecognised mode '%s', falling back to FixedPriceEstimator", s) - return NewWrappedEvmEstimator(NewFixedPriceEstimator(geCfg, bh, lggr), df, l1Oracle) + newEstimator = func(l logger.Logger) EvmEstimator { + return NewFixedPriceEstimator(geCfg, bh, lggr) + } } + return NewWrappedEvmEstimator(lggr, newEstimator, df, l1Oracle) } // DynamicFee encompasses both FeeCap and TipCap for EIP1559 transactions @@ -150,6 +162,7 @@ func (fee EvmFee) ValidDynamic() bool { // WrappedEvmEstimator provides a struct that wraps the EVM specific dynamic and legacy estimators into one estimator that conforms to the generic FeeEstimator type WrappedEvmEstimator struct { services.StateMachine + lggr logger.Logger EvmEstimator EIP1559Enabled bool l1Oracle rollups.L1Oracle @@ -157,16 +170,18 @@ type WrappedEvmEstimator struct { var _ EvmFeeEstimator = (*WrappedEvmEstimator)(nil) -func NewWrappedEvmEstimator(e EvmEstimator, eip1559Enabled bool, l1Oracle rollups.L1Oracle) EvmFeeEstimator { +func NewWrappedEvmEstimator(lggr logger.Logger, newEstimator func(logger.Logger) EvmEstimator, eip1559Enabled bool, l1Oracle rollups.L1Oracle) EvmFeeEstimator { + lggr = logger.Named(lggr, "WrappedEvmEstimator") return &WrappedEvmEstimator{ - EvmEstimator: e, + lggr: lggr, + EvmEstimator: newEstimator(lggr), EIP1559Enabled: eip1559Enabled, l1Oracle: l1Oracle, } } func (e *WrappedEvmEstimator) Name() string { - return fmt.Sprintf("WrappedEvmEstimator(%s)", e.EvmEstimator.Name()) + return e.lggr.Name() } func (e *WrappedEvmEstimator) Start(ctx context.Context) error { diff --git a/core/chains/evm/gas/models_test.go b/core/chains/evm/gas/models_test.go index a2dce58ee3f..95a7a471eba 100644 --- a/core/chains/evm/gas/models_test.go +++ b/core/chains/evm/gas/models_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas/mocks" @@ -27,39 +28,41 @@ func TestWrappedEvmEstimator(t *testing.T) { FeeCap: assets.NewWeiI(20), TipCap: assets.NewWeiI(1), } - - e := mocks.NewEvmEstimator(t) - e.On("GetDynamicFee", mock.Anything, mock.Anything, mock.Anything). + est := mocks.NewEvmEstimator(t) + est.On("GetDynamicFee", mock.Anything, mock.Anything, mock.Anything). Return(dynamicFee, gasLimit, nil).Twice() - e.On("GetLegacyGas", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + est.On("GetLegacyGas", mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(legacyFee, gasLimit, nil).Twice() - e.On("BumpDynamicFee", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + est.On("BumpDynamicFee", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(dynamicFee, gasLimit, nil).Once() - e.On("BumpLegacyGas", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). + est.On("BumpLegacyGas", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything). Return(legacyFee, gasLimit, nil).Once() + getRootEst := func(logger.Logger) gas.EvmEstimator { return est } - mockEvmEstimatorName := "MockEstimator" - mockEstimatorName := "WrappedEvmEstimator(MockEstimator)" + mockEstimatorName := "WrappedEvmEstimator" + mockEvmEstimatorName := "WrappedEvmEstimator.MockEstimator" // L1Oracle returns the correct L1Oracle interface t.Run("L1Oracle", func(t *testing.T) { + lggr := logger.Test(t) // expect nil - estimator := gas.NewWrappedEvmEstimator(e, false, nil) + estimator := gas.NewWrappedEvmEstimator(lggr, getRootEst, false, nil) l1Oracle := estimator.L1Oracle() assert.Nil(t, l1Oracle) // expect l1Oracle oracle := rollupMocks.NewL1Oracle(t) - estimator = gas.NewWrappedEvmEstimator(e, false, oracle) + estimator = gas.NewWrappedEvmEstimator(lggr, getRootEst, false, oracle) l1Oracle = estimator.L1Oracle() assert.Equal(t, oracle, l1Oracle) }) // GetFee returns gas estimation based on configuration value t.Run("GetFee", func(t *testing.T) { + lggr := logger.Test(t) // expect legacy fee data dynamicFees := false - estimator := gas.NewWrappedEvmEstimator(e, dynamicFees, nil) + estimator := gas.NewWrappedEvmEstimator(lggr, getRootEst, dynamicFees, nil) fee, max, err := estimator.GetFee(ctx, nil, 0, nil) require.NoError(t, err) assert.Equal(t, gasLimit, max) @@ -69,7 +72,7 @@ func TestWrappedEvmEstimator(t *testing.T) { // expect dynamic fee data dynamicFees = true - estimator = gas.NewWrappedEvmEstimator(e, dynamicFees, nil) + estimator = gas.NewWrappedEvmEstimator(lggr, getRootEst, dynamicFees, nil) fee, max, err = estimator.GetFee(ctx, nil, 0, nil) require.NoError(t, err) assert.Equal(t, gasLimit, max) @@ -80,8 +83,9 @@ func TestWrappedEvmEstimator(t *testing.T) { // BumpFee returns bumped fee type based on original fee calculation t.Run("BumpFee", func(t *testing.T) { + lggr := logger.Test(t) dynamicFees := false - estimator := gas.NewWrappedEvmEstimator(e, dynamicFees, nil) + estimator := gas.NewWrappedEvmEstimator(lggr, getRootEst, dynamicFees, nil) // expect legacy fee data fee, max, err := estimator.BumpFee(ctx, gas.EvmFee{Legacy: assets.NewWeiI(0)}, 0, nil, nil) @@ -114,11 +118,12 @@ func TestWrappedEvmEstimator(t *testing.T) { }) t.Run("GetMaxCost", func(t *testing.T) { + lggr := logger.Test(t) val := assets.NewEthValue(1) // expect legacy fee data dynamicFees := false - estimator := gas.NewWrappedEvmEstimator(e, dynamicFees, nil) + estimator := gas.NewWrappedEvmEstimator(lggr, getRootEst, dynamicFees, nil) total, err := estimator.GetMaxCost(ctx, val, nil, gasLimit, nil) require.NoError(t, err) fee := new(big.Int).Mul(legacyFee.ToInt(), big.NewInt(int64(gasLimit))) @@ -126,7 +131,7 @@ func TestWrappedEvmEstimator(t *testing.T) { // expect dynamic fee data dynamicFees = true - estimator = gas.NewWrappedEvmEstimator(e, dynamicFees, nil) + estimator = gas.NewWrappedEvmEstimator(lggr, getRootEst, dynamicFees, nil) total, err = estimator.GetMaxCost(ctx, val, nil, gasLimit, nil) require.NoError(t, err) fee = new(big.Int).Mul(dynamicFee.FeeCap.ToInt(), big.NewInt(int64(gasLimit))) @@ -134,33 +139,38 @@ func TestWrappedEvmEstimator(t *testing.T) { }) t.Run("Name", func(t *testing.T) { - evmEstimator := mocks.NewEvmEstimator(t) - oracle := rollupMocks.NewL1Oracle(t) + lggr := logger.Test(t) + oracle := rollupMocks.NewL1Oracle(t) + evmEstimator := mocks.NewEvmEstimator(t) evmEstimator.On("Name").Return(mockEvmEstimatorName, nil).Once() - estimator := gas.NewWrappedEvmEstimator(evmEstimator, false, oracle) - name := estimator.Name() - require.Equal(t, mockEstimatorName, name) + estimator := gas.NewWrappedEvmEstimator(lggr, func(logger.Logger) gas.EvmEstimator { + return evmEstimator + }, false, oracle) + + require.Equal(t, mockEstimatorName, estimator.Name()) + require.Equal(t, mockEvmEstimatorName, evmEstimator.Name()) }) t.Run("Start and stop calls both EVM estimator and L1Oracle", func(t *testing.T) { - evmEstimator := mocks.NewEvmEstimator(t) + lggr := logger.Test(t) oracle := rollupMocks.NewL1Oracle(t) + evmEstimator := mocks.NewEvmEstimator(t) - evmEstimator.On("Name").Return(mockEvmEstimatorName, nil).Times(4) evmEstimator.On("Start", mock.Anything).Return(nil).Twice() evmEstimator.On("Close").Return(nil).Twice() oracle.On("Start", mock.Anything).Return(nil).Once() oracle.On("Close").Return(nil).Once() + getEst := func(logger.Logger) gas.EvmEstimator { return evmEstimator } - estimator := gas.NewWrappedEvmEstimator(evmEstimator, false, nil) + estimator := gas.NewWrappedEvmEstimator(lggr, getEst, false, nil) err := estimator.Start(ctx) require.NoError(t, err) err = estimator.Close() require.NoError(t, err) - estimator = gas.NewWrappedEvmEstimator(evmEstimator, false, oracle) + estimator = gas.NewWrappedEvmEstimator(lggr, getEst, false, oracle) err = estimator.Start(ctx) require.NoError(t, err) err = estimator.Close() @@ -168,22 +178,25 @@ func TestWrappedEvmEstimator(t *testing.T) { }) t.Run("Read calls both EVM estimator and L1Oracle", func(t *testing.T) { + lggr := logger.Test(t) evmEstimator := mocks.NewEvmEstimator(t) oracle := rollupMocks.NewL1Oracle(t) evmEstimator.On("Ready").Return(nil).Twice() oracle.On("Ready").Return(nil).Once() + getEst := func(logger.Logger) gas.EvmEstimator { return evmEstimator } - estimator := gas.NewWrappedEvmEstimator(evmEstimator, false, nil) + estimator := gas.NewWrappedEvmEstimator(lggr, getEst, false, nil) err := estimator.Ready() require.NoError(t, err) - estimator = gas.NewWrappedEvmEstimator(evmEstimator, false, oracle) + estimator = gas.NewWrappedEvmEstimator(lggr, getEst, false, oracle) err = estimator.Ready() require.NoError(t, err) }) t.Run("HealthReport merges report from EVM estimator and L1Oracle", func(t *testing.T) { + lggr := logger.Test(t) evmEstimator := mocks.NewEvmEstimator(t) oracle := rollupMocks.NewL1Oracle(t) @@ -192,17 +205,17 @@ func TestWrappedEvmEstimator(t *testing.T) { oracleKey := "oracle" oracleError := errors.New("oracle error") - evmEstimator.On("Name").Return(mockEvmEstimatorName, nil).Twice() evmEstimator.On("HealthReport").Return(map[string]error{evmEstimatorKey: evmEstimatorError}).Twice() oracle.On("HealthReport").Return(map[string]error{oracleKey: oracleError}).Once() + getEst := func(logger.Logger) gas.EvmEstimator { return evmEstimator } - estimator := gas.NewWrappedEvmEstimator(evmEstimator, false, nil) + estimator := gas.NewWrappedEvmEstimator(lggr, getEst, false, nil) report := estimator.HealthReport() require.True(t, errors.Is(report[evmEstimatorKey], evmEstimatorError)) require.Nil(t, report[oracleKey]) require.NotNil(t, report[mockEstimatorName]) - estimator = gas.NewWrappedEvmEstimator(evmEstimator, false, oracle) + estimator = gas.NewWrappedEvmEstimator(lggr, getEst, false, oracle) report = estimator.HealthReport() require.True(t, errors.Is(report[evmEstimatorKey], evmEstimatorError)) require.True(t, errors.Is(report[oracleKey], oracleError)) diff --git a/core/chains/evm/txmgr/broadcaster_test.go b/core/chains/evm/txmgr/broadcaster_test.go index b9e8fe99e31..f676d3d18e2 100644 --- a/core/chains/evm/txmgr/broadcaster_test.go +++ b/core/chains/evm/txmgr/broadcaster_test.go @@ -64,7 +64,9 @@ func NewTestEthBroadcaster( lggr := logger.Test(t) ge := config.EVM().GasEstimator() - estimator := gas.NewWrappedEvmEstimator(gas.NewFixedPriceEstimator(config.EVM().GasEstimator(), ge.BlockHistory(), lggr), ge.EIP1559DynamicFees(), nil) + estimator := gas.NewWrappedEvmEstimator(lggr, func(lggr logger.Logger) gas.EvmEstimator { + return gas.NewFixedPriceEstimator(config.EVM().GasEstimator(), ge.BlockHistory(), lggr) + }, ge.EIP1559DynamicFees(), nil) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, keyStore, estimator) txNonceSyncer := txmgr.NewNonceSyncer(txStore, lggr, ethClient) ethBroadcaster := txmgr.NewEvmBroadcaster(txStore, txmgr.NewEvmTxmClient(ethClient), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(config.EVM().GasEstimator()), config.EVM().Transactions(), config.Database().Listener(), keyStore, txBuilder, txNonceSyncer, lggr, checkerFactory, nonceAutoSync) @@ -1134,7 +1136,9 @@ func TestEthBroadcaster_ProcessUnstartedEthTxs_Errors(t *testing.T) { // same as the parent test, but callback is set by ctor t.Run("callback set by ctor", func(t *testing.T) { lggr := logger.Test(t) - estimator := gas.NewWrappedEvmEstimator(gas.NewFixedPriceEstimator(evmcfg.EVM().GasEstimator(), evmcfg.EVM().GasEstimator().BlockHistory(), lggr), evmcfg.EVM().GasEstimator().EIP1559DynamicFees(), nil) + estimator := gas.NewWrappedEvmEstimator(lggr, func(lggr logger.Logger) gas.EvmEstimator { + return gas.NewFixedPriceEstimator(evmcfg.EVM().GasEstimator(), evmcfg.EVM().GasEstimator().BlockHistory(), lggr) + }, evmcfg.EVM().GasEstimator().EIP1559DynamicFees(), nil) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), evmcfg.EVM().GasEstimator(), ethKeyStore, estimator) localNextNonce = getLocalNextNonce(t, eb, fromAddress) ethClient.On("PendingNonceAt", mock.Anything, fromAddress).Return(uint64(localNextNonce), nil).Once() @@ -1759,7 +1763,9 @@ func TestEthBroadcaster_SyncNonce(t *testing.T) { ethNodeNonce := uint64(22) - estimator := gas.NewWrappedEvmEstimator(gas.NewFixedPriceEstimator(evmcfg.EVM().GasEstimator(), evmcfg.EVM().GasEstimator().BlockHistory(), lggr), evmcfg.EVM().GasEstimator().EIP1559DynamicFees(), nil) + estimator := gas.NewWrappedEvmEstimator(lggr, func(lggr logger.Logger) gas.EvmEstimator { + return gas.NewFixedPriceEstimator(evmcfg.EVM().GasEstimator(), evmcfg.EVM().GasEstimator().BlockHistory(), lggr) + }, evmcfg.EVM().GasEstimator().EIP1559DynamicFees(), nil) checkerFactory := &testCheckerFactory{} ge := evmcfg.EVM().GasEstimator() diff --git a/core/chains/evm/txmgr/confirmer_test.go b/core/chains/evm/txmgr/confirmer_test.go index 3acbfe9800c..30b2a391a7f 100644 --- a/core/chains/evm/txmgr/confirmer_test.go +++ b/core/chains/evm/txmgr/confirmer_test.go @@ -122,9 +122,10 @@ func TestEthConfirmer_Lifecycle(t *testing.T) { cltest.MustInsertRandomKey(t, ethKeyStore) cltest.MustInsertRandomKey(t, ethKeyStore) estimator := gasmocks.NewEvmEstimator(t) + newEst := func(logger.Logger) gas.EvmEstimator { return estimator } lggr := logger.Test(t) ge := config.EVM().GasEstimator() - feeEstimator := gas.NewWrappedEvmEstimator(estimator, ge.EIP1559DynamicFees(), nil) + feeEstimator := gas.NewWrappedEvmEstimator(lggr, newEst, ge.EIP1559DynamicFees(), nil) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ethKeyStore, feeEstimator) ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), config.Database(), ethKeyStore, txBuilder, lggr) ctx := testutils.Context(t) @@ -1639,9 +1640,10 @@ func TestEthConfirmer_RebroadcastWhereNecessary_WithConnectivityCheck(t *testing kst := ksmocks.NewEth(t) estimator := gasmocks.NewEvmEstimator(t) + newEst := func(logger.Logger) gas.EvmEstimator { return estimator } estimator.On("BumpLegacyGas", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, uint32(0), pkgerrors.Wrapf(commonfee.ErrConnectivity, "transaction...")) ge := ccfg.EVM().GasEstimator() - feeEstimator := gas.NewWrappedEvmEstimator(estimator, ge.EIP1559DynamicFees(), nil) + feeEstimator := gas.NewWrappedEvmEstimator(lggr, newEst, ge.EIP1559DynamicFees(), nil) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, kst, feeEstimator) addresses := []gethCommon.Address{fromAddress} kst.On("EnabledAddressesForChain", &cltest.FixtureChainID).Return(addresses, nil).Maybe() @@ -1685,9 +1687,10 @@ func TestEthConfirmer_RebroadcastWhereNecessary_WithConnectivityCheck(t *testing estimator := gasmocks.NewEvmEstimator(t) estimator.On("BumpDynamicFee", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(gas.DynamicFee{}, uint32(0), pkgerrors.Wrapf(commonfee.ErrConnectivity, "transaction...")) + newEst := func(logger.Logger) gas.EvmEstimator { return estimator } // Create confirmer with necessary state ge := ccfg.EVM().GasEstimator() - feeEstimator := gas.NewWrappedEvmEstimator(estimator, ge.EIP1559DynamicFees(), nil) + feeEstimator := gas.NewWrappedEvmEstimator(lggr, newEst, ge.EIP1559DynamicFees(), nil) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, kst, feeEstimator) addresses := []gethCommon.Address{fromAddress} kst.On("EnabledAddressesForChain", &cltest.FixtureChainID).Return(addresses, nil).Maybe() @@ -3079,7 +3082,9 @@ func ptr[T any](t T) *T { return &t } func newEthConfirmer(t testing.TB, txStore txmgr.EvmTxStore, ethClient client.Client, config evmconfig.ChainScopedConfig, ks keystore.Eth, fn txmgrcommon.ResumeCallback) *txmgr.Confirmer { lggr := logger.Test(t) ge := config.EVM().GasEstimator() - estimator := gas.NewWrappedEvmEstimator(gas.NewFixedPriceEstimator(ge, ge.BlockHistory(), lggr), ge.EIP1559DynamicFees(), nil) + estimator := gas.NewWrappedEvmEstimator(lggr, func(lggr logger.Logger) gas.EvmEstimator { + return gas.NewFixedPriceEstimator(ge, ge.BlockHistory(), lggr) + }, ge.EIP1559DynamicFees(), nil) txBuilder := txmgr.NewEvmTxAttemptBuilder(*ethClient.ConfiguredChainID(), ge, ks, estimator) ec := txmgr.NewEvmConfirmer(txStore, txmgr.NewEvmTxmClient(ethClient), txmgr.NewEvmTxmConfig(config.EVM()), txmgr.NewEvmTxmFeeConfig(ge), config.EVM().Transactions(), config.Database(), ks, txBuilder, lggr) ec.SetResumeCallback(fn) diff --git a/core/services/chainlink/relayer_factory.go b/core/services/chainlink/relayer_factory.go index 4ed73d8e53b..b4bd530d080 100644 --- a/core/services/chainlink/relayer_factory.go +++ b/core/services/chainlink/relayer_factory.go @@ -45,9 +45,11 @@ func (r *RelayerFactory) NewEVM(ctx context.Context, config EVMFactoryConfig) (m relayers := make(map[relay.ID]evmrelay.LoopRelayAdapter) + lggr := r.Logger.Named("EVM") + // override some common opts with the factory values. this seems weird... maybe other signatures should change, or this should take a different type... ccOpts := legacyevm.ChainRelayExtenderConfig{ - Logger: r.Logger.Named("EVM"), + Logger: lggr, KeyStore: config.CSAETHKeystore.Eth(), ChainOpts: config.ChainOpts, } @@ -71,7 +73,7 @@ func (r *RelayerFactory) NewEVM(ctx context.Context, config EVMFactoryConfig) (m EventBroadcaster: ccOpts.EventBroadcaster, MercuryPool: r.MercuryPool, } - relayer, err2 := evmrelay.NewRelayer(r.Logger.Named("EVM").Named(relayID.ChainID), chain, relayerOpts) + relayer, err2 := evmrelay.NewRelayer(lggr.Named(relayID.ChainID), chain, relayerOpts) if err2 != nil { err = errors.Join(err, err2) continue diff --git a/core/services/relay/evm/evm.go b/core/services/relay/evm/evm.go index 83540e22bb7..303cdd3ba0e 100644 --- a/core/services/relay/evm/evm.go +++ b/core/services/relay/evm/evm.go @@ -13,6 +13,7 @@ import ( "github.com/jmoiron/sqlx" pkgerrors "github.com/pkg/errors" "go.uber.org/multierr" + "golang.org/x/exp/maps" "github.com/smartcontractkit/libocr/gethwrappers2/ocr2aggregator" "github.com/smartcontractkit/libocr/offchainreporting2/reportingplugin/median" @@ -121,11 +122,12 @@ func (r *Relayer) Close() error { // Ready does noop: always ready func (r *Relayer) Ready() error { - return nil + return r.chain.Ready() } func (r *Relayer) HealthReport() (report map[string]error) { report = make(map[string]error) + maps.Copy(report, r.chain.HealthReport()) return } diff --git a/core/web/health_controller.go b/core/web/health_controller.go index d6a7edb2340..d6490e5542a 100644 --- a/core/web/health_controller.go +++ b/core/web/health_controller.go @@ -1,7 +1,10 @@ package web import ( + "cmp" "net/http" + "slices" + "testing" "github.com/gin-gonic/gin" @@ -94,6 +97,12 @@ func (hc *HealthController) Health(c *gin.Context) { }) } + if testing.Testing() { + slices.SortFunc(checks, func(a, b presenters.Check) int { + return cmp.Compare(a.Name, b.Name) + }) + } + // return a json description of all the checks - jsonAPIResponse(c, checks, "checks") + jsonAPIResponseWithStatus(c, checks, "checks", status) } diff --git a/core/web/health_controller_test.go b/core/web/health_controller_test.go index d380b279d00..7e7c42141ca 100644 --- a/core/web/health_controller_test.go +++ b/core/web/health_controller_test.go @@ -1,15 +1,19 @@ package web_test import ( + "bytes" + _ "embed" + "encoding/json" + "io" "net/http" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/services/mocks" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestHealthController_Readyz(t *testing.T) { @@ -47,3 +51,63 @@ func TestHealthController_Readyz(t *testing.T) { }) } } + +func TestHealthController_Health_status(t *testing.T) { + var tt = []struct { + name string + ready bool + status int + }{ + { + name: "not ready", + ready: false, + status: http.StatusServiceUnavailable, + }, + { + name: "ready", + ready: true, + status: http.StatusOK, + }, + } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + app := cltest.NewApplicationWithKey(t) + healthChecker := new(mocks.Checker) + healthChecker.On("Start").Return(nil).Once() + healthChecker.On("IsHealthy").Return(tc.ready, nil).Once() + healthChecker.On("Close").Return(nil).Once() + + app.HealthChecker = healthChecker + require.NoError(t, app.Start(testutils.Context(t))) + + client := app.NewHTTPClient(nil) + resp, cleanup := client.Get("/health") + t.Cleanup(cleanup) + assert.Equal(t, tc.status, resp.StatusCode) + }) + } +} + +var ( + //go:embed testdata/body/health.json + healthJSON string +) + +func TestHealthController_Health_body(t *testing.T) { + app := cltest.NewApplicationWithKey(t) + require.NoError(t, app.Start(testutils.Context(t))) + + client := app.NewHTTPClient(nil) + resp, cleanup := client.Get("/health") + t.Cleanup(cleanup) + assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + // pretty print for comparison + var b bytes.Buffer + require.NoError(t, json.Indent(&b, body, "", " ")) + body = b.Bytes() + + assert.Equal(t, healthJSON, string(body)) +} diff --git a/core/web/testdata/body/health.json b/core/web/testdata/body/health.json new file mode 100644 index 00000000000..d8418560543 --- /dev/null +++ b/core/web/testdata/body/health.json @@ -0,0 +1,166 @@ +{ + "data": [ + { + "type": "checks", + "id": "EVM.0", + "attributes": { + "name": "EVM.0", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.BalanceMonitor", + "attributes": { + "name": "EVM.0.BalanceMonitor", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.HeadBroadcaster", + "attributes": { + "name": "EVM.0.HeadBroadcaster", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.HeadTracker", + "attributes": { + "name": "EVM.0.HeadTracker", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.HeadTracker.HeadListener", + "attributes": { + "name": "EVM.0.HeadTracker.HeadListener", + "status": "failing", + "output": "Listener is not connected" + } + }, + { + "type": "checks", + "id": "EVM.0.LogBroadcaster", + "attributes": { + "name": "EVM.0.LogBroadcaster", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.Txm", + "attributes": { + "name": "EVM.0.Txm", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.Txm.BlockHistoryEstimator", + "attributes": { + "name": "EVM.0.Txm.BlockHistoryEstimator", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.Txm.Broadcaster", + "attributes": { + "name": "EVM.0.Txm.Broadcaster", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.Txm.Confirmer", + "attributes": { + "name": "EVM.0.Txm.Confirmer", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "EVM.0.Txm.WrappedEvmEstimator", + "attributes": { + "name": "EVM.0.Txm.WrappedEvmEstimator", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "JobSpawner", + "attributes": { + "name": "JobSpawner", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "Mercury.WSRPCPool", + "attributes": { + "name": "Mercury.WSRPCPool", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "Monitor", + "attributes": { + "name": "Monitor", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "PipelineORM", + "attributes": { + "name": "PipelineORM", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "PipelineRunner", + "attributes": { + "name": "PipelineRunner", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "PromReporter", + "attributes": { + "name": "PromReporter", + "status": "passing", + "output": "" + } + }, + { + "type": "checks", + "id": "TelemetryManager", + "attributes": { + "name": "TelemetryManager", + "status": "passing", + "output": "" + } + } + ] +} \ No newline at end of file