[TT-591] Parallel Test Logging E2E Fixes (#10610)

* [TT-591] Fix E2E Test Logging In Parallel Tests

* triage why logs are not getting the correct test struct

* More logging missing in parallel e2e tests

* More improvements to the logging

* merge conflict fixes

* Cover more missing cases

* More and cleanup

* Bump ctf and merge conflict fixes

integration-tests/actions/actions.go

@@ -19,6 +19,7 @@ import (
 	"github.com/smartcontractkit/chainlink-env/environment"
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 	ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client"
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/smartcontractkit/chainlink-testing-framework/testreporters"
 	"github.com/smartcontractkit/chainlink-testing-framework/utils"
 
@@ -251,7 +252,7 @@ func TeardownSuite(
 	failingLogLevel zapcore.Level, // Examines logs after the test, and fails the test if any Chainlink logs are found at or above provided level
 	clients ...blockchain.EVMClient,
 ) error {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	if err := testreporters.WriteTeardownLogs(t, env, optionalTestReporter, failingLogLevel); err != nil {
 		return errors.Wrap(err, "Error dumping environment logs, leaving environment running for manual retrieval")
 	}
@@ -295,7 +296,7 @@ func TeardownRemoteSuite(
 	optionalTestReporter testreporters.TestReporter, // Optionally pass in a test reporter to log further metrics
 	client blockchain.EVMClient,
 ) error {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	var err error
 	if err = testreporters.SendReport(t, namespace, "./", optionalTestReporter); err != nil {
 		l.Warn().Err(err).Msg("Error writing test report")

integration-tests/actions/automation_ocr_helpers.go

@@ -13,7 +13,7 @@ import (
 	"github.com/lib/pq"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
-	"github.com/smartcontractkit/chainlink-testing-framework/utils"
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/smartcontractkit/chainlink/v2/core/services/job"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
@@ -48,7 +48,7 @@ func BuildAutoOCR2ConfigVarsWithKeyIndex(
 	deltaStage time.Duration,
 	keyIndex int,
 ) (contracts.OCRv2Config, error) {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	S, oracleIdentities, err := GetOracleIdentitiesWithKeyIndex(chainlinkNodes, keyIndex)
 	if err != nil {
 		return contracts.OCRv2Config{}, err
@@ -172,7 +172,7 @@ func CreateOCRKeeperJobs(
 	keyIndex int,
 	registryVersion ethereum.KeeperRegistryVersion,
 ) {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	bootstrapNode := chainlinkNodes[0]
 	bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys()
 	require.NoError(t, err, "Shouldn't fail reading P2P keys from bootstrap node")

integration-tests/actions/automation_ocr_helpers_local.go

@@ -4,17 +4,14 @@ package actions
 import (
 	"encoding/json"
 	"fmt"
-	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/lib/pq"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
-	"github.com/stretchr/testify/require"
+	"github.com/rs/zerolog"
 	"gopkg.in/guregu/null.v4"
 
-	"github.com/smartcontractkit/chainlink-testing-framework/utils"
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum"
@@ -30,24 +27,23 @@ import (
 )
 
 func BuildAutoOCR2ConfigVarsLocal(
-	t *testing.T,
+	l zerolog.Logger,
 	chainlinkNodes []*client.ChainlinkClient,
 	registryConfig contracts.KeeperRegistrySettings,
 	registrar string,
 	deltaStage time.Duration,
 ) (contracts.OCRv2Config, error) {
-	return BuildAutoOCR2ConfigVarsWithKeyIndexLocal(t, chainlinkNodes, registryConfig, registrar, deltaStage, 0)
+	return BuildAutoOCR2ConfigVarsWithKeyIndexLocal(l, chainlinkNodes, registryConfig, registrar, deltaStage, 0)
 }
 
 func BuildAutoOCR2ConfigVarsWithKeyIndexLocal(
-	t *testing.T,
+	l zerolog.Logger,
 	chainlinkNodes []*client.ChainlinkClient,
 	registryConfig contracts.KeeperRegistrySettings,
 	registrar string,
 	deltaStage time.Duration,
 	keyIndex int,
 ) (contracts.OCRv2Config, error) {
-	l := utils.GetTestLogger(t)
 	S, oracleIdentities, err := GetOracleIdentitiesWithKeyIndexLocal(chainlinkNodes, keyIndex)
 	if err != nil {
 		return contracts.OCRv2Config{}, err
@@ -136,13 +132,17 @@ func BuildAutoOCR2ConfigVarsWithKeyIndexLocal(
 
 	var signers []common.Address
 	for _, signer := range signerOnchainPublicKeys {
-		require.Equal(t, 20, len(signer), "OnChainPublicKey '%v' has wrong length for address", signer)
+		if len(signer) != 20 {
+			return contracts.OCRv2Config{}, fmt.Errorf("OnChainPublicKey '%v' has wrong length for address", signer)
+		}
 		signers = append(signers, common.BytesToAddress(signer))
 	}
 
 	var transmitters []common.Address
 	for _, transmitter := range transmitterAccounts {
-		require.True(t, common.IsHexAddress(string(transmitter)), "TransmitAccount '%s' is not a valid Ethereum address", string(transmitter))
+		if !common.IsHexAddress(string(transmitter)) {
+			return contracts.OCRv2Config{}, fmt.Errorf("TransmitAccount '%s' is not a valid Ethereum address", string(transmitter))
+		}
 		transmitters = append(transmitters, common.HexToAddress(string(transmitter)))
 	}
 
@@ -164,6 +164,7 @@ func BuildAutoOCR2ConfigVarsWithKeyIndexLocal(
 
 // CreateOCRKeeperJobs bootstraps the first node and to the other nodes sends ocr jobs
 func CreateOCRKeeperJobsLocal(
+	l zerolog.Logger,
 	chainlinkNodes []*client.ChainlinkClient,
 	registryAddr string,
 	chainID int64,
@@ -173,7 +174,7 @@ func CreateOCRKeeperJobsLocal(
 	bootstrapNode := chainlinkNodes[0]
 	bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys()
 	if err != nil {
-		log.Error().Err(err).Msg("Shouldn't fail reading P2P keys from bootstrap node")
+		l.Error().Err(err).Msg("Shouldn't fail reading P2P keys from bootstrap node")
 		return err
 	}
 	bootstrapP2PId := bootstrapP2PIds.Data[0].Attributes.PeerID
@@ -201,20 +202,20 @@ func CreateOCRKeeperJobsLocal(
 	}
 	_, err = bootstrapNode.MustCreateJob(bootstrapSpec)
 	if err != nil {
-		log.Error().Err(err).Msg("Shouldn't fail creating bootstrap job on bootstrap node")
+		l.Error().Err(err).Msg("Shouldn't fail creating bootstrap job on bootstrap node")
 		return err
 	}
 
 	P2Pv2Bootstrapper := fmt.Sprintf("%s@%s:%d", bootstrapP2PId, bootstrapNode.InternalIP(), 6690)
 	for nodeIndex := 1; nodeIndex < len(chainlinkNodes); nodeIndex++ {
 		nodeTransmitterAddress, err := chainlinkNodes[nodeIndex].EthAddresses()
 		if err != nil {
-			log.Error().Err(err).Msgf("Shouldn't fail getting primary ETH address from OCR node %d", nodeIndex+1)
+			l.Error().Err(err).Msgf("Shouldn't fail getting primary ETH address from OCR node %d", nodeIndex+1)
 			return err
 		}
 		nodeOCRKeys, err := chainlinkNodes[nodeIndex].MustReadOCR2Keys()
 		if err != nil {
-			log.Error().Err(err).Msgf("Shouldn't fail getting OCR keys from OCR node %d", nodeIndex+1)
+			l.Error().Err(err).Msgf("Shouldn't fail getting OCR keys from OCR node %d", nodeIndex+1)
 			return err
 		}
 		var nodeOCRKeyId []string
@@ -248,11 +249,11 @@ func CreateOCRKeeperJobsLocal(
 
 		_, err = chainlinkNodes[nodeIndex].MustCreateJob(&autoOCR2JobSpec)
 		if err != nil {
-			log.Error().Err(err).Msgf("Shouldn't fail creating OCR Task job on OCR node %d err: %+v", nodeIndex+1, err)
+			l.Error().Err(err).Msgf("Shouldn't fail creating OCR Task job on OCR node %d err: %+v", nodeIndex+1, err)
 			return err
 		}
 
 	}
-	log.Info().Msg("Done creating OCR automation jobs")
+	l.Info().Msg("Done creating OCR automation jobs")
 	return nil
 }

integration-tests/actions/keeper_helpers.go

@@ -13,7 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
-	"github.com/smartcontractkit/chainlink-testing-framework/utils"
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
@@ -339,7 +339,7 @@ func RegisterUpkeepContracts(t *testing.T, linkToken contracts.LinkToken, linkFu
 }
 
 func RegisterUpkeepContractsWithCheckData(t *testing.T, linkToken contracts.LinkToken, linkFunds *big.Int, client blockchain.EVMClient, upkeepGasLimit uint32, registry contracts.KeeperRegistry, registrar contracts.KeeperRegistrar, numberOfContracts int, upkeepAddresses []string, checkData [][]byte, isLogTrigger bool) []*big.Int {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	registrationTxHashes := make([]common.Hash, 0)
 	upkeepIds := make([]*big.Int, 0)
 	for contractCount, upkeepAddress := range upkeepAddresses {
@@ -398,7 +398,7 @@ func RegisterUpkeepContractsWithCheckData(t *testing.T, linkToken contracts.Link
 }
 
 func DeployKeeperConsumers(t *testing.T, contractDeployer contracts.ContractDeployer, client blockchain.EVMClient, numberOfContracts int, isLogTrigger bool) []contracts.KeeperConsumer {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	keeperConsumerContracts := make([]contracts.KeeperConsumer, 0)
 
 	for contractCount := 0; contractCount < numberOfContracts; contractCount++ {
@@ -441,7 +441,7 @@ func DeployKeeperConsumersPerformance(
 	checkGasToBurn, // How much gas should be burned on checkUpkeep() calls
 	performGasToBurn int64, // How much gas should be burned on performUpkeep() calls
 ) []contracts.KeeperConsumerPerformance {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	upkeeps := make([]contracts.KeeperConsumerPerformance, 0)
 
 	for contractCount := 0; contractCount < numberOfContracts; contractCount++ {
@@ -478,7 +478,7 @@ func DeployPerformDataChecker(
 	numberOfContracts int,
 	expectedData []byte,
 ) []contracts.KeeperPerformDataChecker {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	upkeeps := make([]contracts.KeeperPerformDataChecker, 0)
 
 	for contractCount := 0; contractCount < numberOfContracts; contractCount++ {
@@ -510,7 +510,7 @@ func DeployUpkeepCounters(
 	testRange *big.Int,
 	interval *big.Int,
 ) []contracts.UpkeepCounter {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	upkeepCounters := make([]contracts.UpkeepCounter, 0)
 
 	for contractCount := 0; contractCount < numberOfContracts; contractCount++ {
@@ -543,7 +543,7 @@ func DeployUpkeepPerformCounterRestrictive(
 	testRange *big.Int,
 	averageEligibilityCadence *big.Int,
 ) []contracts.UpkeepPerformCounterRestrictive {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	upkeepCounters := make([]contracts.UpkeepPerformCounterRestrictive, 0)
 
 	for contractCount := 0; contractCount < numberOfContracts; contractCount++ {

integration-tests/actions/keeper_helpers_local.go

@@ -3,13 +3,14 @@ package actions
 import (
 	"fmt"
 
-	"github.com/rs/zerolog/log"
+	"github.com/rs/zerolog"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 )
 
 func CreateKeeperJobsLocal(
+	l zerolog.Logger,
 	chainlinkNodes []*client.ChainlinkClient,
 	keeperRegistry contracts.KeeperRegistry,
 	ocrConfig contracts.OCRv2Config,
@@ -19,12 +20,12 @@ func CreateKeeperJobsLocal(
 	primaryNode := chainlinkNodes[0]
 	primaryNodeAddress, err := primaryNode.PrimaryEthAddress()
 	if err != nil {
-		log.Error().Err(err).Msg("Reading ETH Keys from Chainlink Client shouldn't fail")
+		l.Error().Err(err).Msg("Reading ETH Keys from Chainlink Client shouldn't fail")
 		return nil, err
 	}
 	nodeAddresses, err := ChainlinkNodeAddressesLocal(chainlinkNodes)
 	if err != nil {
-		log.Error().Err(err).Msg("Retrieving on-chain wallet addresses for chainlink nodes shouldn't fail")
+		l.Error().Err(err).Msg("Retrieving on-chain wallet addresses for chainlink nodes shouldn't fail")
 		return nil, err
 	}
 	nodeAddressesStr, payees := make([]string, 0), make([]string, 0)
@@ -34,14 +35,14 @@ func CreateKeeperJobsLocal(
 	}
 	err = keeperRegistry.SetKeepers(nodeAddressesStr, payees, ocrConfig)
 	if err != nil {
-		log.Error().Err(err).Msg("Setting keepers in the registry shouldn't fail")
+		l.Error().Err(err).Msg("Setting keepers in the registry shouldn't fail")
 		return nil, err
 	}
 	jobs := []*client.Job{}
 	for _, chainlinkNode := range chainlinkNodes {
 		chainlinkNodeAddress, err := chainlinkNode.PrimaryEthAddress()
 		if err != nil {
-			log.Error().Err(err).Msg("Error retrieving chainlink node address")
+			l.Error().Err(err).Msg("Error retrieving chainlink node address")
 			return nil, err
 		}
 		job, err := chainlinkNode.MustCreateJob(&client.KeeperJobSpec{
@@ -52,7 +53,7 @@ func CreateKeeperJobsLocal(
 			MinIncomingConfirmations: 1,
 		})
 		if err != nil {
-			log.Error().Err(err).Msg("Creating KeeperV2 Job shouldn't fail")
+			l.Error().Err(err).Msg("Creating KeeperV2 Job shouldn't fail")
 			return nil, err
 		}
 		jobs = append(jobs, job)

integration-tests/actions/ocr2_helpers.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/lib/pq"
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/guregu/null.v4"
@@ -360,13 +361,14 @@ func StartNewOCR2Round(
 	ocrInstances []contracts.OffchainAggregatorV2,
 	client blockchain.EVMClient,
 	timeout time.Duration,
+	logger zerolog.Logger,
 ) error {
 	for i := 0; i < len(ocrInstances); i++ {
 		err := ocrInstances[i].RequestNewRound()
 		if err != nil {
 			return fmt.Errorf("requesting new OCR round %d have failed: %w", i+1, err)
 		}
-		ocrRound := contracts.NewOffchainAggregatorV2RoundConfirmer(ocrInstances[i], big.NewInt(roundNumber), timeout)
+		ocrRound := contracts.NewOffchainAggregatorV2RoundConfirmer(ocrInstances[i], big.NewInt(roundNumber), timeout, logger)
 		client.AddHeaderEventSubscription(ocrInstances[i].Address(), ocrRound)
 		err = client.WaitForEvents()
 		if err != nil {

integration-tests/actions/ocr2vrf_actions/ocr2vrf_config_helpers.go

@@ -16,7 +16,7 @@ import (
 	"go.dedis.ch/kyber/v3/group/edwards25519"
 	"gopkg.in/guregu/null.v4"
 
-	"github.com/smartcontractkit/chainlink-testing-framework/utils"
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/smartcontractkit/chainlink/v2/core/services/job"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
 	"github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
@@ -39,7 +39,7 @@ func CreateOCR2VRFJobs(
 	chainID int64,
 	keyIndex int,
 ) {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	p2pV2Bootstrapper := createBootstrapJob(t, bootstrapNode, OCR2VRFPluginConfig.DKGConfig.DKGContractAddress, chainID)
 
 	createNonBootstrapJobs(t, nonBootstrapNodes, OCR2VRFPluginConfig, chainID, keyIndex, p2pV2Bootstrapper)
@@ -120,7 +120,7 @@ func BuildOCR2DKGConfigVars(
 	t *testing.T,
 	ocr2VRFPluginConfig *OCR2VRFPluginConfig,
 ) contracts.OCRv2Config {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	var onchainPublicKeys []common.Address
 	for _, onchainPublicKey := range ocr2VRFPluginConfig.OCR2Config.OnchainPublicKeys {
 		onchainPublicKeys = append(onchainPublicKeys, common.HexToAddress(onchainPublicKey))
@@ -272,7 +272,7 @@ func BuildOCR2VRFConfigVars(
 	t *testing.T,
 	ocr2VRFPluginConfig *OCR2VRFPluginConfig,
 ) contracts.OCRv2Config {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	var onchainPublicKeys []common.Address
 	for _, onchainPublicKey := range ocr2VRFPluginConfig.OCR2Config.OnchainPublicKeys {
 		onchainPublicKeys = append(onchainPublicKeys, common.HexToAddress(onchainPublicKey))

integration-tests/actions/ocr2vrf_actions/ocr2vrf_steps.go

@@ -13,7 +13,7 @@ import (
 	ocr2vrftypes "github.com/smartcontractkit/ocr2vrf/types"
 
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
-	"github.com/smartcontractkit/chainlink-testing-framework/utils"
+	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
 	chainlinkutils "github.com/smartcontractkit/chainlink/v2/core/utils"
@@ -25,7 +25,7 @@ import (
 )
 
 func SetAndWaitForVRFBeaconProcessToFinish(t *testing.T, ocr2VRFPluginConfig *OCR2VRFPluginConfig, vrfBeacon contracts.VRFBeacon) {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	ocr2VrfConfig := BuildOCR2VRFConfigVars(t, ocr2VRFPluginConfig)
 	l.Debug().Interface("OCR2 VRF Config", ocr2VrfConfig).Msg("OCR2 VRF Config prepared")
 
@@ -45,7 +45,7 @@ func SetAndWaitForVRFBeaconProcessToFinish(t *testing.T, ocr2VRFPluginConfig *OC
 }
 
 func SetAndWaitForDKGProcessToFinish(t *testing.T, ocr2VRFPluginConfig *OCR2VRFPluginConfig, dkg contracts.DKG) {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	ocr2DkgConfig := BuildOCR2DKGConfigVars(t, ocr2VRFPluginConfig)
 
 	// set config for DKG OCR
@@ -208,7 +208,7 @@ func RequestAndRedeemRandomness(
 	confirmationDelay *big.Int,
 	randomnessTransmissionEventTimeout time.Duration,
 ) *big.Int {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	receipt, err := consumer.RequestRandomness(
 		numberOfRandomWordsToRequest,
 		subscriptionID,
@@ -244,7 +244,7 @@ func RequestRandomnessFulfillmentAndWaitForFulfilment(
 	confirmationDelay *big.Int,
 	randomnessTransmissionEventTimeout time.Duration,
 ) *big.Int {
-	l := utils.GetTestLogger(t)
+	l := logging.GetTestLogger(t)
 	receipt, err := consumer.RequestRandomnessFulfillment(
 		numberOfRandomWordsToRequest,
 		subscriptionID,