Merge branch 'develop' into ocrv2_multiple_el_clients

.github/workflows/ci-chaincli.yml

@@ -2,11 +2,7 @@ name: chaincli CI
 
 on:
   push:
-    paths:
-      - "core/scripts/chaincli/**"
   pull_request:
-    paths:
-      - "core/scripts/chaincli/**"
 
 jobs:
   golangci:

.github/workflows/integration-tests.yml

@@ -498,6 +498,13 @@ jobs:
           PYROSCOPE_ENVIRONMENT: ${{ matrix.product.pyroscope_env }}
           PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
           ETH2_EL_CLIENT: ${{matrix.product.client}}
+          LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }}
+          LOKI_URL: ${{ secrets.LOKI_URL }}
+          LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
+          LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
+          GRAFANA_URL: ${{ vars.GRAFANA_URL }}
+          GRAFANA_DATASOURCE: ${{ vars.GRAFANA_DATASOURCE }}
+          RUN_ID: ${{ github.run_id }}
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestfmt
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -550,6 +557,27 @@ jobs:
         with:
           name: trace-data
           path: ./integration-tests/smoke/traces/trace-data.json
+      - name: Print failed test summary
+        if: always()
+        run: |
+          directory="./integration-tests/smoke/.test_summary"
+          files=("$directory"/*)
+          if [ -d "$directory" ]; then
+            echo "Test summary folder found"
+            if [ ${#files[@]} -gt 0 ]; then
+              first_file="${files[0]}"
+              echo "Name of the first test summary file: $(basename "$first_file")"
+              echo "### Failed Test Execution Logs Dashboard (over VPN):" >> $GITHUB_STEP_SUMMARY
+              cat "$first_file" | jq -r '.loki[] | "* [\(.test_name)](\(.value))"' >> $GITHUB_STEP_SUMMARY
+              if [ ${#files[@]} -gt 1 ]; then
+                echo "Found more than one test summary file. This is incorrect, there should be only one file"
+              fi
+            else
+                echo "Test summary directory is empty. This should not happen"
+            fi
+          else
+            echo "No test summary folder found. If no test failed or log collection wasn't explicitly requested this is correct. Exiting"
+          fi
 
   ### Used to check the required checks box when the matrix completes
   eth-smoke-tests:

.gitignore

@@ -83,5 +83,5 @@ go.work*
 # This sometimes shows up for some reason
 tools/flakeytests/coverage.txt
 
+.test_summary/
 .run.id
-test_summary/

.tool-versions

@@ -1,4 +1,4 @@
-golang 1.21.4
+golang 1.21.5
 mockery 2.38.0
 nodejs 16.16.0
 postgres 13.3

GNUmakefile

@@ -147,6 +147,10 @@ goreleaser-dev-build: ## Run goreleaser snapshot build
 goreleaser-dev-release: ## run goreleaser snapshot release
 	./tools/bin/goreleaser_wrapper release --snapshot --rm-dist --config ${GORELEASER_CONFIG}
 
+.PHONY: modgraph
+modgraph:
+	./tools/bin/modgraph > go.md
+
 help:
 	@echo ""
 	@echo "         .__           .__       .__  .__        __"

contracts/gas-snapshots/functions.gas-snapshot

@@ -142,11 +142,11 @@ FunctionsSubscriptions_GetSubscriptionsInRange:test_GetSubscriptionsInRange_Reve
 FunctionsSubscriptions_GetSubscriptionsInRange:test_GetSubscriptionsInRange_RevertIfStartIsAfterEnd() (gas: 13459)
 FunctionsSubscriptions_GetSubscriptionsInRange:test_GetSubscriptionsInRange_Success() (gas: 59592)
 FunctionsSubscriptions_GetTotalBalance:test_GetTotalBalance_Success() (gas: 15010)
-FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfCallerIsNoCalldata(uint96) (runs: 256, μ: 43774, ~: 45548)
-FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfCallerIsNoSubscription(uint96) (runs: 256, μ: 46286, ~: 48060)
-FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfCallerIsNotLink(uint96) (runs: 256, μ: 14295, ~: 14295)
-FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfPaused(uint96) (runs: 256, μ: 51443, ~: 53040)
-FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_Success(uint96) (runs: 256, μ: 86057, ~: 89604)
+FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfCallerIsNoCalldata() (gas: 39939)
+FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfCallerIsNoSubscription() (gas: 42404)
+FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfCallerIsNotLink() (gas: 13441)
+FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_RevertIfPaused() (gas: 47347)
+FunctionsSubscriptions_OnTokenTransfer:test_OnTokenTransfer_Success(uint96) (runs: 256, μ: 81598, ~: 81598)
 FunctionsSubscriptions_OracleWithdraw:test_OracleWithdraw_RevertIfAmountMoreThanBalance() (gas: 20745)
 FunctionsSubscriptions_OracleWithdraw:test_OracleWithdraw_RevertIfBalanceInvariant() (gas: 189)
 FunctionsSubscriptions_OracleWithdraw:test_OracleWithdraw_RevertIfNoAmount() (gas: 15638)
@@ -177,7 +177,7 @@ FunctionsSubscriptions_ProposeSubscriptionOwnerTransfer:test_ProposeSubscription
 FunctionsSubscriptions_ProposeSubscriptionOwnerTransfer:test_ProposeSubscriptionOwnerTransfer_Success() (gas: 68196)
 FunctionsSubscriptions_ProposeSubscriptionOwnerTransfer:test_ProposeSubscriptionOwnerTransfer_SuccessChangeProposedOwner() (gas: 82749)
 FunctionsSubscriptions_RecoverFunds:test_OwnerCancelSubscription_RevertIfNotOwner() (gas: 15554)
-FunctionsSubscriptions_RecoverFunds:test_RecoverFunds_Success(uint64) (runs: 256, μ: 41717, ~: 41721)
+FunctionsSubscriptions_RecoverFunds:test_RecoverFunds_Success() (gas: 41111)
 FunctionsSubscriptions_RemoveConsumer:test_RemoveConsumer_RevertIfInvalidConsumer() (gas: 30260)
 FunctionsSubscriptions_RemoveConsumer:test_RemoveConsumer_RevertIfNoSubscription() (gas: 15019)
 FunctionsSubscriptions_RemoveConsumer:test_RemoveConsumer_RevertIfNotAllowedSender() (gas: 57800)

contracts/src/v0.8/functions/tests/v1_X/FunctionsSubscriptions.t.sol

@@ -130,12 +130,8 @@ contract FunctionsSubscriptions_OwnerCancelSubscription is FunctionsSubscription
 contract FunctionsSubscriptions_RecoverFunds is FunctionsRouterSetup {
   event FundsRecovered(address to, uint256 amount);
 
-  function test_RecoverFunds_Success(uint64 fundsTransferred) public {
-    //amount must be less than LINK total supply
-    vm.assume(fundsTransferred < 1_000_000_000 * 1e18);
-    vm.assume(fundsTransferred > 0);
-
-    // uint256 fundsTransferred = 1 * 1e18; // 1 LINK
+  function test_RecoverFunds_Success() public {
+    uint256 fundsTransferred = 1 * 1e18; // 1 LINK
     s_linkToken.transfer(address(s_functionsRouter), fundsTransferred);
 
     // topic0 (function signature, always checked), NOT topic1 (false), NOT topic2 (false), NOT topic3 (false), and data (true).
@@ -320,58 +316,43 @@ contract FunctionsSubscriptions_OnTokenTransfer is FunctionsClientSetup {
     s_functionsRouter.addConsumer(s_subscriptionId, address(s_functionsClient));
   }
 
-  function test_OnTokenTransfer_RevertIfPaused(uint96 fundingAmount) public {
-    // Funding amount must be less than LINK total supply
+  function test_OnTokenTransfer_RevertIfPaused() public {
+    // Funding amount must be less than or equal to LINK total supply
     uint256 totalSupplyJuels = 1_000_000_000 * 1e18;
-    vm.assume(fundingAmount <= totalSupplyJuels);
-    vm.assume(fundingAmount >= 0);
-
     s_functionsRouter.pause();
     vm.expectRevert("Pausable: paused");
-    s_linkToken.transferAndCall(address(s_functionsRouter), fundingAmount, abi.encode(s_subscriptionId));
+    s_linkToken.transferAndCall(address(s_functionsRouter), totalSupplyJuels, abi.encode(s_subscriptionId));
   }
 
-  function test_OnTokenTransfer_RevertIfCallerIsNotLink(uint96 fundingAmount) public {
-    // Funding amount must be less than LINK total supply
+  function test_OnTokenTransfer_RevertIfCallerIsNotLink() public {
+    // Funding amount must be less than or equal to LINK total supply
     uint256 totalSupplyJuels = 1_000_000_000 * 1e18;
-    vm.assume(fundingAmount <= totalSupplyJuels);
-    vm.assume(fundingAmount >= 0);
-
     vm.expectRevert(FunctionsSubscriptions.OnlyCallableFromLink.selector);
-    s_functionsRouter.onTokenTransfer(address(s_functionsRouter), fundingAmount, abi.encode(s_subscriptionId));
+    s_functionsRouter.onTokenTransfer(address(s_functionsRouter), totalSupplyJuels, abi.encode(s_subscriptionId));
   }
 
-  function test_OnTokenTransfer_RevertIfCallerIsNoCalldata(uint96 fundingAmount) public {
-    // Funding amount must be less than LINK total supply
+  function test_OnTokenTransfer_RevertIfCallerIsNoCalldata() public {
+    // Funding amount must be less than or equal to LINK total supply
     uint256 totalSupplyJuels = 1_000_000_000 * 1e18;
-    vm.assume(fundingAmount <= totalSupplyJuels);
-    vm.assume(fundingAmount >= 0);
-
     vm.expectRevert(FunctionsSubscriptions.InvalidCalldata.selector);
-    s_linkToken.transferAndCall(address(s_functionsRouter), fundingAmount, new bytes(0));
+    s_linkToken.transferAndCall(address(s_functionsRouter), totalSupplyJuels, new bytes(0));
   }
 
-  function test_OnTokenTransfer_RevertIfCallerIsNoSubscription(uint96 fundingAmount) public {
-    // Funding amount must be less than LINK total supply
+  function test_OnTokenTransfer_RevertIfCallerIsNoSubscription() public {
+    // Funding amount must be less than or equal to LINK total supply
     uint256 totalSupplyJuels = 1_000_000_000 * 1e18;
-    vm.assume(fundingAmount <= totalSupplyJuels);
-    vm.assume(fundingAmount >= 0);
-
     vm.expectRevert(FunctionsSubscriptions.InvalidSubscription.selector);
     uint64 invalidSubscriptionId = 123456789;
-    s_linkToken.transferAndCall(address(s_functionsRouter), fundingAmount, abi.encode(invalidSubscriptionId));
+    s_linkToken.transferAndCall(address(s_functionsRouter), totalSupplyJuels, abi.encode(invalidSubscriptionId));
   }
 
   function test_OnTokenTransfer_Success(uint96 fundingAmount) public {
     // Funding amount must be less than LINK total supply
     uint256 totalSupplyJuels = 1_000_000_000 * 1e18;
     // Some of the total supply is already in the subscription account
-    vm.assume(fundingAmount <= totalSupplyJuels);
-    vm.assume(fundingAmount >= 0);
-
-    s_linkToken.transferAndCall(address(s_functionsRouter), fundingAmount, abi.encode(s_subscriptionId));
+    s_linkToken.transferAndCall(address(s_functionsRouter), totalSupplyJuels, abi.encode(s_subscriptionId));
     uint96 subscriptionBalanceAfter = s_functionsRouter.getSubscription(s_subscriptionId).balance;
-    assertEq(fundingAmount, subscriptionBalanceAfter);
+    assertEq(totalSupplyJuels, subscriptionBalanceAfter);
   }
 }
 

core/chains/evm/gas/arbitrum_estimator_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas/mocks"
@@ -66,8 +67,7 @@ func TestArbitrumEstimator(t *testing.T) {
 		}).Return(zeros.Bytes(), nil)
 
 		o := gas.NewArbitrumEstimator(logger.Test(t), &arbConfig{v: maxGasLimit}, rpcClient, ethClient)
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, maxGasPrice)
 		require.NoError(t, err)
 		// Expected price for a standard l2_suggested_estimator would be 42, but we add a fixed gasPriceBufferPercentage.
@@ -92,8 +92,7 @@ func TestArbitrumEstimator(t *testing.T) {
 			assert.Equal(t, big.NewInt(-1), blockNumber)
 		}).Return(zeros.Bytes(), nil)
 
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, assets.NewWeiI(40))
 		require.Error(t, err)
 		assert.EqualError(t, err, "estimated gas price: 42 wei is greater than the maximum gas price configured: 40 wei")
@@ -118,8 +117,7 @@ func TestArbitrumEstimator(t *testing.T) {
 			assert.Equal(t, big.NewInt(-1), blockNumber)
 		}).Return(zeros.Bytes(), nil)
 
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, assets.NewWeiI(110))
 		assert.EqualError(t, err, "estimated gas price: 120 wei is greater than the maximum gas price configured: 110 wei")
 		assert.Nil(t, gasPrice)
@@ -148,8 +146,7 @@ func TestArbitrumEstimator(t *testing.T) {
 			assert.Equal(t, big.NewInt(-1), blockNumber)
 		}).Return(zeros.Bytes(), nil)
 
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 
 		_, _, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, maxGasPrice)
 		assert.EqualError(t, err, "failed to estimate gas; gas price not set")
@@ -181,8 +178,7 @@ func TestArbitrumEstimator(t *testing.T) {
 		}).Return(b.Bytes(), nil)
 
 		o := gas.NewArbitrumEstimator(logger.Test(t), &arbConfig{v: maxGasLimit}, rpcClient, ethClient)
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, maxGasPrice)
 		require.NoError(t, err)
 		require.NotNil(t, gasPrice)
@@ -216,8 +212,7 @@ func TestArbitrumEstimator(t *testing.T) {
 		}).Return(b.Bytes(), nil)
 
 		o := gas.NewArbitrumEstimator(logger.Test(t), &arbConfig{v: maxGasLimit}, rpcClient, ethClient)
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, maxGasPrice)
 		require.Error(t, err, "expected error but got (%s, %d)", gasPrice, chainSpecificGasLimit)
 	})

core/chains/evm/gas/rollups/l1_gas_price_oracle_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
 
 	"github.com/smartcontractkit/chainlink/v2/common/config"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
@@ -50,8 +51,7 @@ func TestL1GasPriceOracle(t *testing.T) {
 		}).Return(common.BigToHash(l1BaseFee).Bytes(), nil)
 
 		oracle := NewL1GasPriceOracle(logger.Test(t), ethClient, config.ChainArbitrum)
-		require.NoError(t, oracle.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, oracle.Close()) })
+		servicetest.RunHealthy(t, oracle)
 
 		gasPrice, err := oracle.GasPrice(testutils.Context(t))
 		require.NoError(t, err)
@@ -72,8 +72,7 @@ func TestL1GasPriceOracle(t *testing.T) {
 		}).Return(common.BigToHash(l1BaseFee).Bytes(), nil)
 
 		oracle := NewL1GasPriceOracle(logger.Test(t), ethClient, config.ChainKroma)
-		require.NoError(t, oracle.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, oracle.Close()) })
+		servicetest.RunHealthy(t, oracle)
 
 		gasPrice, err := oracle.GasPrice(testutils.Context(t))
 		require.NoError(t, err)
@@ -94,8 +93,7 @@ func TestL1GasPriceOracle(t *testing.T) {
 		}).Return(common.BigToHash(l1BaseFee).Bytes(), nil)
 
 		oracle := NewL1GasPriceOracle(logger.Test(t), ethClient, config.ChainOptimismBedrock)
-		require.NoError(t, oracle.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, oracle.Close()) })
+		servicetest.RunHealthy(t, oracle)
 
 		gasPrice, err := oracle.GasPrice(testutils.Context(t))
 		require.NoError(t, err)

core/chains/evm/gas/suggested_price_estimator_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/gas/mocks"
@@ -40,8 +41,7 @@ func TestSuggestedPriceEstimator(t *testing.T) {
 		})
 
 		o := gas.NewSuggestedPriceEstimator(logger.Test(t), client)
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, maxGasPrice)
 		require.NoError(t, err)
 		assert.Equal(t, assets.NewWeiI(42), gasPrice)
@@ -57,8 +57,7 @@ func TestSuggestedPriceEstimator(t *testing.T) {
 			(*big.Int)(res).SetInt64(42)
 		})
 
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, assets.NewWeiI(40))
 		require.Error(t, err)
 		assert.EqualError(t, err, "estimated gas price: 42 wei is greater than the maximum gas price configured: 40 wei")
@@ -75,8 +74,7 @@ func TestSuggestedPriceEstimator(t *testing.T) {
 			(*big.Int)(res).SetInt64(120)
 		})
 
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 		gasPrice, chainSpecificGasLimit, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, assets.NewWeiI(110))
 		assert.EqualError(t, err, "estimated gas price: 120 wei is greater than the maximum gas price configured: 110 wei")
 		assert.Nil(t, gasPrice)
@@ -96,8 +94,7 @@ func TestSuggestedPriceEstimator(t *testing.T) {
 
 		client.On("CallContext", mock.Anything, mock.Anything, "eth_gasPrice").Return(errors.New("kaboom"))
 
-		require.NoError(t, o.Start(testutils.Context(t)))
-		t.Cleanup(func() { assert.NoError(t, o.Close()) })
+		servicetest.RunHealthy(t, o)
 
 		_, _, err := o.GetLegacyGas(testutils.Context(t), calldata, gasLimit, maxGasPrice)
 		assert.EqualError(t, err, "failed to estimate gas; gas price not set")

core/chains/evm/headtracker/head_broadcaster_test.go

@@ -11,6 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
+	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
+
 	commonhtrk "github.com/smartcontractkit/chainlink/v2/common/headtracker"
 	commonmocks "github.com/smartcontractkit/chainlink/v2/common/types/mocks"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker"
@@ -21,7 +23,6 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils/evmtest"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest"
-	"github.com/smartcontractkit/chainlink/v2/core/services"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
@@ -68,14 +69,14 @@ func TestHeadBroadcaster_Subscribe(t *testing.T) {
 	checker1 := &cltest.MockHeadTrackable{}
 	checker2 := &cltest.MockHeadTrackable{}
 
-	hb := headtracker.NewHeadBroadcaster(logger)
 	orm := headtracker.NewORM(db, logger, cfg.Database(), *ethClient.ConfiguredChainID())
 	hs := headtracker.NewHeadSaver(logger, orm, evmCfg.EVM(), evmCfg.EVM().HeadTracker())
 	mailMon := utils.NewMailboxMonitor(t.Name())
+	servicetest.Run(t, mailMon)
+	hb := headtracker.NewHeadBroadcaster(logger)
+	servicetest.Run(t, hb)
 	ht := headtracker.NewHeadTracker(logger, ethClient, evmCfg.EVM(), evmCfg.EVM().HeadTracker(), hb, hs, mailMon)
-	var ms services.MultiStart
-	require.NoError(t, ms.Start(testutils.Context(t), mailMon, hb, ht))
-	t.Cleanup(func() { require.NoError(t, services.CloseAll(mailMon, hb, ht)) })
+	servicetest.Run(t, ht)
 
 	latest1, unsubscribe1 := hb.Subscribe(checker1)
 	// "latest head" is nil here because we didn't receive any yet

core/chains/evm/log/helpers_test.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink-common/pkg/services/servicetest"
+
 	evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
 	evmclimocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client/mocks"
 	evmconfig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config"