From 26d43f4c4a8af7d9133acf2c8250345e70d71044 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 15 Apr 2024 14:44:56 +0200
Subject: [PATCH 01/32] Get E2E test coverage of chainlink node binary

---
 .../actions/build-chainlink-image/action.yml  |  3 +-
 GNUmakefile                                   |  4 +
 core/chainlink.Dockerfile                     | 13 ++-
 .../docker/test_env/cl_node_cluster.go        | 91 +++++++++++++++++++
 .../docker/test_env/test_env_builder.go       | 14 +++
 5 files changed, 123 insertions(+), 2 deletions(-)

diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml
index f75bd68a87e..0c7deccd8d2 100644
--- a/.github/actions/build-chainlink-image/action.yml
+++ b/.github/actions/build-chainlink-image/action.yml
@@ -33,7 +33,7 @@ runs:
         AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
     - name: Build Image
       if: steps.check-image.outputs.exists != 'true'
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@f9f5d5e01a6c15764206f732f34d6cc0a85ad964
       with:
         cl_repo: smartcontractkit/chainlink
         cl_ref: ${{ inputs.git_commit_sha }}
@@ -41,6 +41,7 @@ runs:
         push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
         QA_AWS_REGION: ${{ inputs.AWS_REGION }}
         QA_AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
+        COVER_FLAG: true
         dep_evm_sha: ${{ inputs.dep_evm_sha }}
     - name: Print Chainlink Image Built
       shell: sh
diff --git a/GNUmakefile b/GNUmakefile
index 10d72cb724c..74a770b53c2 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -45,6 +45,10 @@ docs: ## Install and run pkgsite to view Go docs
 install-chainlink: operator-ui ## Install the chainlink binary.
 	go install $(GOFLAGS) .
 
+.PHONY: install-chainlink-cover
+install-chainlink: operator-ui ## Install the chainlink binary with cover flag.
+	go install -cover $(GOFLAGS) .
+
 .PHONY: chainlink
 chainlink: ## Build the chainlink binary.
 	go build $(GOFLAGS) .
diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile
index e82a4cd662c..c171079613d 100644
--- a/core/chainlink.Dockerfile
+++ b/core/chainlink.Dockerfile
@@ -12,12 +12,19 @@ RUN go mod download
 # Env vars needed for chainlink build
 ARG COMMIT_SHA
 
+# Build chainlink bin with cover flag
+ARG COVER_FLAG=false
+
 COPY . .
 
 RUN apt-get update && apt-get install -y jq
 
 # Build the golang binary
-RUN make install-chainlink
+RUN if [ "$COVER_FLAG" = "true" ]; then \
+        make install-chainlink-cover; \
+    else \
+        make install-chainlink; \
+    fi
 
 # Link LOOP Plugin source dirs with simple names
 RUN go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds | xargs -I % ln -s % /chainlink-feeds
@@ -68,6 +75,10 @@ WORKDIR /home/${CHAINLINK_USER}
 ENV XDG_CACHE_HOME /home/${CHAINLINK_USER}/.cache
 RUN mkdir -p ${XDG_CACHE_HOME}
 
+# Set up coverage dir env variable
+ENV GOCOVERDIR="coverage"
+RUN mkdir -p coverage
+
 EXPOSE 6688
 ENTRYPOINT ["chainlink"]
 
diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index 08122b5744d..848b64d44a6 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -1,12 +1,19 @@
 package test_env
 
 import (
+	"context"
 	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
+	tc "github.com/testcontainers/testcontainers-go"
 )
 
 var (
@@ -68,3 +75,87 @@ func (c *ClCluster) NodeCSAKeys() ([]string, error) {
 	}
 	return keys, nil
 }
+
+func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath string) error {
+	var wg sync.WaitGroup
+	errors := make(chan error, len(c.Nodes))
+
+	for i, node := range c.Nodes {
+		wg.Add(1)
+		go func(n *ClNode) {
+			defer wg.Done()
+			// Create a unique subdirectory for each node based on an identifier
+			destPath := filepath.Join(destPath, fmt.Sprintf("node_%d", i)) // Use node ID or another unique identifier
+			if err := os.MkdirAll(destPath, 0755); err != nil {
+				errors <- fmt.Errorf("failed to create directory for node %d: %w", i, err)
+				return
+			}
+			err := copyFolderFromContainer(ctx, n.Container, srcPath, destPath)
+			if err != nil {
+				errors <- fmt.Errorf("failed to copy folder for node %d: %w", i, err)
+			}
+		}(node)
+	}
+
+	wg.Wait()
+	close(errors)
+
+	for err := range errors {
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func copyFolderFromContainer(ctx context.Context, container tc.Container, srcPath, destPath string) error {
+	// Tar the source directory inside the container
+	tarCmd := []string{"tar", "-czf", "/tmp/archive.tar.gz", "-C", srcPath, "."}
+	_, _, err := container.Exec(ctx, tarCmd)
+	if err != nil {
+		return fmt.Errorf("failed to tar folder in container: %w", err)
+	}
+
+	reader, err := container.CopyFileFromContainer(ctx, "/tmp/archive.tar.gz")
+	if err != nil {
+		return fmt.Errorf("failed to copy from container: %w", err)
+	}
+	defer reader.Close()
+
+	// Ensure destination path exists
+	if info, err := os.Stat(destPath); err == nil {
+		if !info.IsDir() {
+			return fmt.Errorf("destination path %s is not a directory", destPath)
+		}
+	} else if os.IsNotExist(err) {
+		return fmt.Errorf("destination path %s does not exist", destPath)
+	} else {
+		return fmt.Errorf("error checking destination directory: %w", err)
+	}
+
+	if err := os.MkdirAll(destPath, 0755); err != nil {
+		return fmt.Errorf("failed to create destination directory: %w", err)
+	}
+
+	// Create the tar file on the host
+	destTarPath := filepath.Join(destPath, "archive.tar.gz")
+	localTarFile, err := os.Create(destTarPath)
+	if err != nil {
+		return fmt.Errorf("failed to create tar file on host: %w", err)
+	}
+	defer localTarFile.Close()
+
+	// Copy tar data from the container to the host file
+	if _, err := io.Copy(localTarFile, reader); err != nil {
+		return fmt.Errorf("failed to copy tar file content: %w", err)
+	}
+
+	// Extract the tar file
+	cmd := exec.Command("tar", "-xzf", destTarPath, "-C", destPath)
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to extract tar file: %w", err)
+	}
+
+	return nil
+}
diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
index 224c364b2d8..932c7315328 100644
--- a/integration-tests/docker/test_env/test_env_builder.go
+++ b/integration-tests/docker/test_env/test_env_builder.go
@@ -1,6 +1,7 @@
 package test_env
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math/big"
@@ -256,6 +257,19 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 	switch b.cleanUpType {
 	case CleanUpTypeStandard:
 		b.t.Cleanup(func() {
+			// Save test coverage profile from node containers
+			coverageDir := fmt.Sprintf("/tmp/coverage/%s", b.t.Name())
+			if err := os.MkdirAll(coverageDir, 0755); err != nil {
+				b.l.Error().Err(err).Str("coverageDir", coverageDir).Msg("Failed to create test coverage directory")
+			}
+			err = b.te.ClCluster.CopyFolderFromNodes(context.Background(), "/home/root/coverage", coverageDir)
+			if err != nil {
+				b.l.Error().Err(err).Msg("Failed to copy test coverage files from nodes")
+			} else {
+				b.l.Info().Str("coverageDir", coverageDir).Msg("Node test coverage files saved")
+			}
+
+			// Cleanup test environment
 			if err := b.te.Cleanup(); err != nil {
 				b.l.Error().Err(err).Msg("Error cleaning up test environment")
 			}

From b01b883fec57e5241c6f54e17f3c68fe8d64ad3a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 12:52:49 +0200
Subject: [PATCH 02/32] Fix

---
 .github/workflows/integration-tests.yml       |  3 ++-
 GNUmakefile                                   |  2 +-
 .../docker/test_env/test_env_builder.go       | 21 +++++++++++--------
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 1386bfef8f3..b399752e0bc 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -372,7 +372,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@08210ca4289f40ae5f4d6b88bb9b7715a5864c37
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -385,6 +385,7 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
+          go_coverage_dir: ./integration-tests/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
diff --git a/GNUmakefile b/GNUmakefile
index 74a770b53c2..19b70e277b5 100644
--- a/GNUmakefile
+++ b/GNUmakefile
@@ -46,7 +46,7 @@ install-chainlink: operator-ui ## Install the chainlink binary.
 	go install $(GOFLAGS) .
 
 .PHONY: install-chainlink-cover
-install-chainlink: operator-ui ## Install the chainlink binary with cover flag.
+install-chainlink-cover: operator-ui ## Install the chainlink binary with cover flag.
 	go install -cover $(GOFLAGS) .
 
 .PHONY: chainlink
diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
index 932c7315328..27154e07b69 100644
--- a/integration-tests/docker/test_env/test_env_builder.go
+++ b/integration-tests/docker/test_env/test_env_builder.go
@@ -258,15 +258,18 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 	case CleanUpTypeStandard:
 		b.t.Cleanup(func() {
 			// Save test coverage profile from node containers
-			coverageDir := fmt.Sprintf("/tmp/coverage/%s", b.t.Name())
-			if err := os.MkdirAll(coverageDir, 0755); err != nil {
-				b.l.Error().Err(err).Str("coverageDir", coverageDir).Msg("Failed to create test coverage directory")
-			}
-			err = b.te.ClCluster.CopyFolderFromNodes(context.Background(), "/home/root/coverage", coverageDir)
-			if err != nil {
-				b.l.Error().Err(err).Msg("Failed to copy test coverage files from nodes")
-			} else {
-				b.l.Info().Str("coverageDir", coverageDir).Msg("Node test coverage files saved")
+			baseCoverageDir, isSet := os.LookupEnv("GO_COVERAGE_DIR")
+			if isSet {
+				testCoverageDir := fmt.Sprintf("%s/%s", baseCoverageDir, b.t.Name())
+				if err := os.MkdirAll(testCoverageDir, 0755); err != nil {
+					b.l.Error().Err(err).Str("coverageDir", testCoverageDir).Msg("Failed to create test coverage directory")
+				}
+				err = b.te.ClCluster.CopyFolderFromNodes(context.Background(), "/home/root/coverage", testCoverageDir)
+				if err != nil {
+					b.l.Error().Err(err).Msg("Failed to copy test coverage files from nodes")
+				} else {
+					b.l.Info().Str("coverageDir", testCoverageDir).Msg("Node test coverage files saved")
+				}
 			}
 
 			// Cleanup test environment

From 57a1d883396ce2cd9ce6e21f5cbfceb46e1ecef1 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 13:23:19 +0200
Subject: [PATCH 03/32] Fix lint

---
 integration-tests/docker/cmd/test_env.go        |  2 +-
 .../docker/test_env/cl_node_cluster.go          | 17 +++++++++--------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/integration-tests/docker/cmd/test_env.go b/integration-tests/docker/cmd/test_env.go
index bbe86128330..ec41c776577 100644
--- a/integration-tests/docker/cmd/test_env.go
+++ b/integration-tests/docker/cmd/test_env.go
@@ -33,7 +33,7 @@ func main() {
 	startFullEnvCmd := &cobra.Command{
 		Use:   "cl-cluster",
 		Short: "Basic CL cluster",
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 			log.Logger = logging.GetLogger(nil, "CORE_DOCKER_ENV_LOG_LEVEL")
 			log.Info().Msg("Starting CL cluster test environment..")
 
diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index 848b64d44a6..72dfe2f5acf 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -12,8 +12,9 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	tc "github.com/testcontainers/testcontainers-go"
+
+	"github.com/smartcontractkit/chainlink/integration-tests/client"
 )
 
 var (
@@ -82,19 +83,19 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 
 	for i, node := range c.Nodes {
 		wg.Add(1)
-		go func(n *ClNode) {
+		go func(n *ClNode, id int) {
 			defer wg.Done()
 			// Create a unique subdirectory for each node based on an identifier
-			destPath := filepath.Join(destPath, fmt.Sprintf("node_%d", i)) // Use node ID or another unique identifier
-			if err := os.MkdirAll(destPath, 0755); err != nil {
-				errors <- fmt.Errorf("failed to create directory for node %d: %w", i, err)
+			finalDestPath := filepath.Join(destPath, fmt.Sprintf("node_%d", id)) // Use node ID or another unique identifier
+			if err := os.MkdirAll(finalDestPath, 0755); err != nil {
+				errors <- fmt.Errorf("failed to create directory for node %d: %w", id, err)
 				return
 			}
-			err := copyFolderFromContainer(ctx, n.Container, srcPath, destPath)
+			err := copyFolderFromContainer(ctx, n.Container, srcPath, finalDestPath)
 			if err != nil {
-				errors <- fmt.Errorf("failed to copy folder for node %d: %w", i, err)
+				errors <- fmt.Errorf("failed to copy folder for node %d: %w", id, err)
 			}
-		}(node)
+		}(node, i)
 	}
 
 	wg.Wait()

From 69debaa9c275621ab832fd3fdb9b1ca618337056 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 14:37:53 +0200
Subject: [PATCH 04/32] Add step to upload go coverage folder

---
 .github/workflows/integration-tests.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index b399752e0bc..ff234c48a42 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -392,6 +392,13 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+
+      - name: Upload Go Coverage Directory
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-coverage-${{ matrix.product.name }}
+          path: ./integration-tests/go-coverage/
+          
       - name: Print failed test summary
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11

From c04cbffac0df578ece0f4effe41bd64fdb08ea3c Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 15:01:37 +0200
Subject: [PATCH 05/32] Fix

---
 .github/workflows/integration-tests.yml              | 12 ++++++++++--
 .../docker/test_env/test_env_builder.go              |  4 ++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index ff234c48a42..5f5cac34724 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -372,7 +372,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@08210ca4289f40ae5f4d6b88bb9b7715a5864c37
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@1877ccce17d03bd8cb3a0644df598833d5e5470f
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -672,7 +672,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@1877ccce17d03bd8cb3a0644df598833d5e5470f
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -686,12 +686,20 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
+          go_coverage_dir: ./integration-tests/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          
+      - name: Upload Go Coverage Directory
+        uses: actions/upload-artifact@v3
+        with:
+          name: go-coverage-${{ matrix.product.name }}
+          path: ./integration-tests/go-coverage/
+
       # Run this step when changes that do not need the test to run are made
       - name: Run Setup
         if: needs.changes.outputs.src == 'false'
diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
index ac1bf71af88..6dd53eb156b 100644
--- a/integration-tests/docker/test_env/test_env_builder.go
+++ b/integration-tests/docker/test_env/test_env_builder.go
@@ -258,8 +258,8 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 	case CleanUpTypeStandard:
 		b.t.Cleanup(func() {
 			// Save test coverage profile from node containers
-			baseCoverageDir, isSet := os.LookupEnv("GO_COVERAGE_DIR")
-			if isSet {
+			baseCoverageDir := os.Getenv("GO_COVERAGE_DIR")
+			if baseCoverageDir != "" {
 				testCoverageDir := fmt.Sprintf("%s/%s", baseCoverageDir, b.t.Name())
 				if err := os.MkdirAll(testCoverageDir, 0755); err != nil {
 					b.l.Error().Err(err).Str("coverageDir", testCoverageDir).Msg("Failed to create test coverage directory")

From 3f0c6f1080c4e279593fcc2e43c365138d88e1bd Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 15:25:09 +0200
Subject: [PATCH 06/32] debug

---
 .github/workflows/integration-tests.yml               | 6 +++---
 integration-tests/docker/test_env/test_env_builder.go | 6 ++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 5f5cac34724..deb0409cc50 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -674,7 +674,7 @@ jobs:
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@1877ccce17d03bd8cb3a0644df598833d5e5470f
         with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
+          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -onlyerrors=false
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }}
@@ -686,14 +686,14 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
-          go_coverage_dir: ./integration-tests/go-coverage
+          go_coverage_dir: ${{ github.workspace }}/integration-tests/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
-          
+
       - name: Upload Go Coverage Directory
         uses: actions/upload-artifact@v3
         with:
diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
index 6dd53eb156b..0a0f7b9d55f 100644
--- a/integration-tests/docker/test_env/test_env_builder.go
+++ b/integration-tests/docker/test_env/test_env_builder.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/rs/zerolog"
@@ -261,6 +262,11 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 			baseCoverageDir := os.Getenv("GO_COVERAGE_DIR")
 			if baseCoverageDir != "" {
 				testCoverageDir := fmt.Sprintf("%s/%s", baseCoverageDir, b.t.Name())
+				absolutePath, err := filepath.Abs(testCoverageDir)
+				if err != nil {
+					b.l.Err(err).Msg("Error getting absolute path")
+				}
+				b.l.Info().Str("testCoverageDir", testCoverageDir).Str("absolutePath", absolutePath).Msg("Saving coverage files for chainlink nodes")
 				if err := os.MkdirAll(testCoverageDir, 0755); err != nil {
 					b.l.Error().Err(err).Str("coverageDir", testCoverageDir).Msg("Failed to create test coverage directory")
 				}

From a3315f715f925af89d8cb3b9cb6c2c4f642c1075 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 16:24:18 +0200
Subject: [PATCH 07/32] Fix

---
 .../docker/test_env/cl_node_cluster.go        | 95 ++++++++++++-------
 .../docker/test_env/test_env_builder.go       |  9 +-
 2 files changed, 64 insertions(+), 40 deletions(-)

diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index 72dfe2f5acf..eec3eba9de2 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -91,7 +92,7 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 				errors <- fmt.Errorf("failed to create directory for node %d: %w", id, err)
 				return
 			}
-			err := copyFolderFromContainer(ctx, n.Container, srcPath, finalDestPath)
+			err := copyFolderFromContainerUsingDockerCP(ctx, n.Container.GetContainerID(), srcPath, finalDestPath)
 			if err != nil {
 				errors <- fmt.Errorf("failed to copy folder for node %d: %w", id, err)
 			}
@@ -111,51 +112,79 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 }
 
 func copyFolderFromContainer(ctx context.Context, container tc.Container, srcPath, destPath string) error {
-	// Tar the source directory inside the container
-	tarCmd := []string{"tar", "-czf", "/tmp/archive.tar.gz", "-C", srcPath, "."}
-	_, _, err := container.Exec(ctx, tarCmd)
+	// List all files and directories recursively inside the container
+	lsCmd := []string{"find", srcPath, "-type", "f"} // Lists only files, omitting directories
+	outputCode, outputReader, err := container.Exec(ctx, lsCmd)
 	if err != nil {
-		return fmt.Errorf("failed to tar folder in container: %w", err)
+		return fmt.Errorf("failed to list files in container: %w", err)
 	}
-
-	reader, err := container.CopyFileFromContainer(ctx, "/tmp/archive.tar.gz")
-	if err != nil {
-		return fmt.Errorf("failed to copy from container: %w", err)
+	if outputCode != 0 {
+		return fmt.Errorf("could not list files in the container. Command exited with code: %d", outputCode)
 	}
-	defer reader.Close()
 
-	// Ensure destination path exists
-	if info, err := os.Stat(destPath); err == nil {
-		if !info.IsDir() {
-			return fmt.Errorf("destination path %s is not a directory", destPath)
-		}
-	} else if os.IsNotExist(err) {
-		return fmt.Errorf("destination path %s does not exist", destPath)
-	} else {
-		return fmt.Errorf("error checking destination directory: %w", err)
+	// Read the output into a slice of file paths
+	output, err := io.ReadAll(outputReader)
+	if err != nil {
+		return fmt.Errorf("failed to read command output: %w", err)
 	}
+	outStr := string(output)
+	files := strings.Split(outStr, "\n")
 
+	// Ensure destination path exists or create it
 	if err := os.MkdirAll(destPath, 0755); err != nil {
 		return fmt.Errorf("failed to create destination directory: %w", err)
 	}
 
-	// Create the tar file on the host
-	destTarPath := filepath.Join(destPath, "archive.tar.gz")
-	localTarFile, err := os.Create(destTarPath)
-	if err != nil {
-		return fmt.Errorf("failed to create tar file on host: %w", err)
-	}
-	defer localTarFile.Close()
+	// Iterate over each file path
+	for _, file := range files {
+		if file == "" {
+			continue
+		}
+
+		// Define the path for the file on the host
+		relPath, err := filepath.Rel(srcPath, file)
+		if err != nil {
+			return fmt.Errorf("failed to compute relative path: %w", err)
+		}
+		hostPath := filepath.Join(destPath, relPath)
+
+		// Ensure the subdirectory exists
+		if err := os.MkdirAll(filepath.Dir(hostPath), 0755); err != nil {
+			return fmt.Errorf("failed to create subdirectory: %w", err)
+		}
 
-	// Copy tar data from the container to the host file
-	if _, err := io.Copy(localTarFile, reader); err != nil {
-		return fmt.Errorf("failed to copy tar file content: %w", err)
+		// Copy the file from the container
+		reader, err := container.CopyFileFromContainer(ctx, file)
+		if err != nil {
+			return fmt.Errorf("failed to copy file %s from container: %w", file, err)
+		}
+		defer reader.Close()
+
+		// Create the file on the host
+		localFile, err := os.Create(hostPath)
+		if err != nil {
+			return fmt.Errorf("failed to create file on host: %w", err)
+		}
+		defer localFile.Close()
+
+		// Copy data from reader to local file
+		if _, err := io.Copy(localFile, reader); err != nil {
+			return fmt.Errorf("failed to copy file content: %w", err)
+		}
 	}
 
-	// Extract the tar file
-	cmd := exec.Command("tar", "-xzf", destTarPath, "-C", destPath)
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("failed to extract tar file: %w", err)
+	return nil
+}
+
+func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error {
+	source := fmt.Sprintf("%s:%s", containerID, srcPath)
+
+	// Prepare the docker cp command
+	cmd := exec.CommandContext(ctx, "docker", "cp", source, destPath)
+
+	// Execute the docker cp command
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("docker cp command failed: %s, output: %s", err, string(output))
 	}
 
 	return nil
diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
index 0a0f7b9d55f..41744a69540 100644
--- a/integration-tests/docker/test_env/test_env_builder.go
+++ b/integration-tests/docker/test_env/test_env_builder.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"math/big"
 	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/rs/zerolog"
@@ -262,11 +261,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 			baseCoverageDir := os.Getenv("GO_COVERAGE_DIR")
 			if baseCoverageDir != "" {
 				testCoverageDir := fmt.Sprintf("%s/%s", baseCoverageDir, b.t.Name())
-				absolutePath, err := filepath.Abs(testCoverageDir)
-				if err != nil {
-					b.l.Err(err).Msg("Error getting absolute path")
-				}
-				b.l.Info().Str("testCoverageDir", testCoverageDir).Str("absolutePath", absolutePath).Msg("Saving coverage files for chainlink nodes")
+				b.l.Info().Str("testCoverageDir", testCoverageDir).Msg("Saving coverage files for chainlink nodes")
 				if err := os.MkdirAll(testCoverageDir, 0755); err != nil {
 					b.l.Error().Err(err).Str("coverageDir", testCoverageDir).Msg("Failed to create test coverage directory")
 				}
@@ -274,7 +269,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 				if err != nil {
 					b.l.Error().Err(err).Msg("Failed to copy test coverage files from nodes")
 				} else {
-					b.l.Info().Str("coverageDir", testCoverageDir).Msg("Node test coverage files saved")
+					b.l.Info().Str("coverageDir", testCoverageDir).Msg("Chainlink node coverage files saved")
 				}
 			}
 

From 27c51764b04da47d4ca2daf9b7ccf2976f651340 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 16:57:00 +0200
Subject: [PATCH 08/32] Fix lint

---
 .../docker/test_env/cl_node_cluster.go        | 72 +------------------
 1 file changed, 2 insertions(+), 70 deletions(-)

diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index eec3eba9de2..8f45ce56861 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -3,18 +3,15 @@ package test_env
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
-	tc "github.com/testcontainers/testcontainers-go"
-
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 )
 
@@ -111,71 +108,6 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 	return nil
 }
 
-func copyFolderFromContainer(ctx context.Context, container tc.Container, srcPath, destPath string) error {
-	// List all files and directories recursively inside the container
-	lsCmd := []string{"find", srcPath, "-type", "f"} // Lists only files, omitting directories
-	outputCode, outputReader, err := container.Exec(ctx, lsCmd)
-	if err != nil {
-		return fmt.Errorf("failed to list files in container: %w", err)
-	}
-	if outputCode != 0 {
-		return fmt.Errorf("could not list files in the container. Command exited with code: %d", outputCode)
-	}
-
-	// Read the output into a slice of file paths
-	output, err := io.ReadAll(outputReader)
-	if err != nil {
-		return fmt.Errorf("failed to read command output: %w", err)
-	}
-	outStr := string(output)
-	files := strings.Split(outStr, "\n")
-
-	// Ensure destination path exists or create it
-	if err := os.MkdirAll(destPath, 0755); err != nil {
-		return fmt.Errorf("failed to create destination directory: %w", err)
-	}
-
-	// Iterate over each file path
-	for _, file := range files {
-		if file == "" {
-			continue
-		}
-
-		// Define the path for the file on the host
-		relPath, err := filepath.Rel(srcPath, file)
-		if err != nil {
-			return fmt.Errorf("failed to compute relative path: %w", err)
-		}
-		hostPath := filepath.Join(destPath, relPath)
-
-		// Ensure the subdirectory exists
-		if err := os.MkdirAll(filepath.Dir(hostPath), 0755); err != nil {
-			return fmt.Errorf("failed to create subdirectory: %w", err)
-		}
-
-		// Copy the file from the container
-		reader, err := container.CopyFileFromContainer(ctx, file)
-		if err != nil {
-			return fmt.Errorf("failed to copy file %s from container: %w", file, err)
-		}
-		defer reader.Close()
-
-		// Create the file on the host
-		localFile, err := os.Create(hostPath)
-		if err != nil {
-			return fmt.Errorf("failed to create file on host: %w", err)
-		}
-		defer localFile.Close()
-
-		// Copy data from reader to local file
-		if _, err := io.Copy(localFile, reader); err != nil {
-			return fmt.Errorf("failed to copy file content: %w", err)
-		}
-	}
-
-	return nil
-}
-
 func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error {
 	source := fmt.Sprintf("%s:%s", containerID, srcPath)
 
@@ -184,7 +116,7 @@ func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcP
 
 	// Execute the docker cp command
 	if output, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("docker cp command failed: %s, output: %s", err, string(output))
+		return errors.Wrapf(err, "docker cp command failed: %s, output: %s", cmd, string(output))
 	}
 
 	return nil

From c4617a42c60ae8cb1463c9ebe02e1b576cd1d67e Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 17:24:02 +0200
Subject: [PATCH 09/32] Debug

---
 .../docker/test_env/cl_node_cluster.go        | 35 ++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index 8f45ce56861..ff2e7edd265 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -3,15 +3,19 @@ package test_env
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
+	tc "github.com/testcontainers/testcontainers-go"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 )
 
@@ -89,9 +93,17 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 				errors <- fmt.Errorf("failed to create directory for node %d: %w", id, err)
 				return
 			}
-			err := copyFolderFromContainerUsingDockerCP(ctx, n.Container.GetContainerID(), srcPath, finalDestPath)
+			covFiles, err := lsFilesInContainer(ctx, n.Container, srcPath)
+			if err != nil {
+				errors <- fmt.Errorf("failed to list files in container for node %d: %w", id, err)
+				return
+			}
+			fmt.Printf("Coverage files in chainlink node container: %v\n", covFiles)
+
+			err = copyFolderFromContainerUsingDockerCP(ctx, n.Container.GetContainerID(), srcPath, finalDestPath)
 			if err != nil {
 				errors <- fmt.Errorf("failed to copy folder for node %d: %w", id, err)
+				return
 			}
 		}(node, i)
 	}
@@ -108,6 +120,27 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 	return nil
 }
 
+func lsFilesInContainer(ctx context.Context, container tc.Container, srcPath string) ([]string, error) {
+	// List all files and directories recursively inside the container
+	lsCmd := []string{"find", srcPath, "-type", "f"} // Lists only files, omitting directories
+	outputCode, outputReader, err := container.Exec(ctx, lsCmd)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list files in container: %w", err)
+	}
+	// Read the output into a slice of file paths
+	output, err := io.ReadAll(outputReader)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read command output: %w", err)
+	}
+	if outputCode != 0 {
+		return nil, fmt.Errorf("failed to list files in container. Command exited with code: %d. Output: %s", outputCode, output)
+	}
+	outStr := string(output)
+	files := strings.Split(outStr, "\n")
+
+	return files, nil
+}
+
 func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error {
 	source := fmt.Sprintf("%s:%s", containerID, srcPath)
 

From 7d8aed8a5b294a2bcdd01446578e53430db5a705 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 16 Apr 2024 22:17:51 +0200
Subject: [PATCH 10/32] Fix

---
 .github/workflows/integration-tests.yml               | 2 +-
 core/chainlink.Dockerfile                             | 4 ++--
 integration-tests/docker/test_env/cl_node_cluster.go  | 2 +-
 integration-tests/docker/test_env/test_env_builder.go | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index deb0409cc50..b071006215b 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -385,7 +385,7 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
-          go_coverage_dir: ./integration-tests/go-coverage
+          go_coverage_dir: ${{ github.workspace }}/integration-tests/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile
index c171079613d..f6f18273ffc 100644
--- a/core/chainlink.Dockerfile
+++ b/core/chainlink.Dockerfile
@@ -76,8 +76,8 @@ ENV XDG_CACHE_HOME /home/${CHAINLINK_USER}/.cache
 RUN mkdir -p ${XDG_CACHE_HOME}
 
 # Set up coverage dir env variable
-ENV GOCOVERDIR="coverage"
-RUN mkdir -p coverage
+ENV GOCOVERDIR="/var/tmp/go-coverage"
+RUN mkdir -p "/var/tmp/go-coverage"
 
 EXPOSE 6688
 ENTRYPOINT ["chainlink"]
diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index ff2e7edd265..ef5871d7ebd 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -98,7 +98,7 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 				errors <- fmt.Errorf("failed to list files in container for node %d: %w", id, err)
 				return
 			}
-			fmt.Printf("Coverage files in chainlink node container: %v\n", covFiles)
+			fmt.Printf("Coverage files for chainlink node: %v\n", covFiles)
 
 			err = copyFolderFromContainerUsingDockerCP(ctx, n.Container.GetContainerID(), srcPath, finalDestPath)
 			if err != nil {
diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
index 41744a69540..529da85e638 100644
--- a/integration-tests/docker/test_env/test_env_builder.go
+++ b/integration-tests/docker/test_env/test_env_builder.go
@@ -265,7 +265,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 				if err := os.MkdirAll(testCoverageDir, 0755); err != nil {
 					b.l.Error().Err(err).Str("coverageDir", testCoverageDir).Msg("Failed to create test coverage directory")
 				}
-				err = b.te.ClCluster.CopyFolderFromNodes(context.Background(), "/home/root/coverage", testCoverageDir)
+				err = b.te.ClCluster.CopyFolderFromNodes(context.Background(), "/var/tmp/go-coverage", testCoverageDir)
 				if err != nil {
 					b.l.Error().Err(err).Msg("Failed to copy test coverage files from nodes")
 				} else {

From 958916c5e2753b18393fb86ed16d760774352521 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 14:02:31 +0200
Subject: [PATCH 11/32] Fix

---
 .github/workflows/integration-tests.yml       | 10 +++---
 core/chainlink.Dockerfile                     |  2 +-
 .../docker/test_env/cl_node_cluster.go        | 32 +++++++++++++------
 integration-tests/docker/test_env/test_env.go | 30 ++++++++++++++++-
 .../docker/test_env/test_env_builder.go       | 19 +----------
 integration-tests/load/vrfv2/vrfv2_test.go    |  4 +--
 .../load/vrfv2plus/vrfv2plus_test.go          |  4 +--
 integration-tests/smoke/vrfv2_test.go         |  8 ++---
 integration-tests/smoke/vrfv2plus_test.go     | 14 ++++----
 9 files changed, 74 insertions(+), 49 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index b071006215b..9bdc91cf699 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -372,7 +372,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@1877ccce17d03bd8cb3a0644df598833d5e5470f
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@be4b0f4b2e3bb5f9e56255f19e5a4809c515f954
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -385,7 +385,8 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
-          go_coverage_dir: ${{ github.workspace }}/integration-tests/go-coverage
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/integration-tests/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
@@ -672,7 +673,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@1877ccce17d03bd8cb3a0644df598833d5e5470f
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@be4b0f4b2e3bb5f9e56255f19e5a4809c515f954
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -onlyerrors=false
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -686,7 +687,8 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
-          go_coverage_dir: ${{ github.workspace }}/integration-tests/go-coverage
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: ${{ github.workspace }}/integration-tests/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile
index f6f18273ffc..96ca2147754 100644
--- a/core/chainlink.Dockerfile
+++ b/core/chainlink.Dockerfile
@@ -75,7 +75,7 @@ WORKDIR /home/${CHAINLINK_USER}
 ENV XDG_CACHE_HOME /home/${CHAINLINK_USER}/.cache
 RUN mkdir -p ${XDG_CACHE_HOME}
 
-# Set up coverage dir env variable
+# Set up go coverage dir
 ENV GOCOVERDIR="/var/tmp/go-coverage"
 RUN mkdir -p "/var/tmp/go-coverage"
 
diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index ef5871d7ebd..abd949c596a 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
@@ -46,6 +47,25 @@ func (c *ClCluster) Start() error {
 	return eg.Wait()
 }
 
+func (c *ClCluster) Stop() error {
+	eg := &errgroup.Group{}
+	nodes := c.Nodes
+	timeout := time.Minute * 1
+
+	for i := 0; i < len(nodes); i++ {
+		nodeIndex := i
+		eg.Go(func() error {
+			err := nodes[nodeIndex].Container.Stop(context.Background(), &timeout)
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+	}
+
+	return eg.Wait()
+}
+
 func (c *ClCluster) NodeAPIs() []*client.ChainlinkClient {
 	clients := make([]*client.ChainlinkClient, 0)
 	for _, c := range c.Nodes {
@@ -93,14 +113,7 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 				errors <- fmt.Errorf("failed to create directory for node %d: %w", id, err)
 				return
 			}
-			covFiles, err := lsFilesInContainer(ctx, n.Container, srcPath)
-			if err != nil {
-				errors <- fmt.Errorf("failed to list files in container for node %d: %w", id, err)
-				return
-			}
-			fmt.Printf("Coverage files for chainlink node: %v\n", covFiles)
-
-			err = copyFolderFromContainerUsingDockerCP(ctx, n.Container.GetContainerID(), srcPath, finalDestPath)
+			err := copyFolderFromContainerUsingDockerCP(ctx, n.Container.GetContainerID(), srcPath, finalDestPath)
 			if err != nil {
 				errors <- fmt.Errorf("failed to copy folder for node %d: %w", id, err)
 				return
@@ -135,8 +148,7 @@ func lsFilesInContainer(ctx context.Context, container tc.Container, srcPath str
 	if outputCode != 0 {
 		return nil, fmt.Errorf("failed to list files in container. Command exited with code: %d. Output: %s", outputCode, output)
 	}
-	outStr := string(output)
-	files := strings.Split(outStr, "\n")
+	files := strings.Split(string(output), "\n")
 
 	return files, nil
 }
diff --git a/integration-tests/docker/test_env/test_env.go b/integration-tests/docker/test_env/test_env.go
index cbcb943e695..85b784b4aab 100644
--- a/integration-tests/docker/test_env/test_env.go
+++ b/integration-tests/docker/test_env/test_env.go
@@ -1,9 +1,11 @@
 package test_env
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"math/big"
+	"os"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -197,8 +199,12 @@ func (te *CLClusterTestEnv) Terminate() error {
 	return nil
 }
 
+type CleanupOpts struct {
+	TestName string
+}
+
 // Cleanup cleans the environment up after it's done being used, mainly for returning funds when on live networks and logs.
-func (te *CLClusterTestEnv) Cleanup() error {
+func (te *CLClusterTestEnv) Cleanup(opts CleanupOpts) error {
 	te.l.Info().Msg("Cleaning up test environment")
 
 	runIdErr := runid.RemoveLocalRunId()
@@ -237,6 +243,28 @@ func (te *CLClusterTestEnv) Cleanup() error {
 		sethClient.Client.Close()
 	}
 
+	covSrcDir := os.Getenv("GO_COVERAGE_SRC_DIR")
+	covDestDir := os.Getenv("GO_COVERAGE_DEST_DIR")
+	shouldCheckCoverage := covSrcDir != "" && covDestDir != ""
+
+	if shouldCheckCoverage {
+		// Stop all nodes in the chainlink cluster.
+		// This is needed to get go coverage profile from the node containers https://go.dev/doc/build-cover#FAQ
+		err := te.ClCluster.Stop()
+		if err != nil {
+			return err
+		}
+
+		// Get go coverage profiles from node containers and save them to a local folder
+		finalCovDestDir := fmt.Sprintf("%s/%s", covDestDir, opts.TestName)
+		err = te.ClCluster.CopyFolderFromNodes(context.Background(), covSrcDir, finalCovDestDir)
+		if err != nil {
+			te.l.Error().Err(err).Str("srcDir", covSrcDir).Str("destDir", finalCovDestDir).Msg("Failed to copy test coverage files from nodes")
+		} else {
+			te.l.Info().Str("srcDir", covSrcDir).Str("destDir", finalCovDestDir).Msg("Chainlink node coverage files saved")
+		}
+	}
+
 	return nil
 }
 
diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
index 529da85e638..62611e79872 100644
--- a/integration-tests/docker/test_env/test_env_builder.go
+++ b/integration-tests/docker/test_env/test_env_builder.go
@@ -1,7 +1,6 @@
 package test_env
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"math/big"
@@ -257,24 +256,8 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 	switch b.cleanUpType {
 	case CleanUpTypeStandard:
 		b.t.Cleanup(func() {
-			// Save test coverage profile from node containers
-			baseCoverageDir := os.Getenv("GO_COVERAGE_DIR")
-			if baseCoverageDir != "" {
-				testCoverageDir := fmt.Sprintf("%s/%s", baseCoverageDir, b.t.Name())
-				b.l.Info().Str("testCoverageDir", testCoverageDir).Msg("Saving coverage files for chainlink nodes")
-				if err := os.MkdirAll(testCoverageDir, 0755); err != nil {
-					b.l.Error().Err(err).Str("coverageDir", testCoverageDir).Msg("Failed to create test coverage directory")
-				}
-				err = b.te.ClCluster.CopyFolderFromNodes(context.Background(), "/var/tmp/go-coverage", testCoverageDir)
-				if err != nil {
-					b.l.Error().Err(err).Msg("Failed to copy test coverage files from nodes")
-				} else {
-					b.l.Info().Str("coverageDir", testCoverageDir).Msg("Chainlink node coverage files saved")
-				}
-			}
-
 			// Cleanup test environment
-			if err := b.te.Cleanup(); err != nil {
+			if err := b.te.Cleanup(CleanupOpts{TestName: b.t.Name()}); err != nil {
 				b.l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		})
diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go
index 0edf35df8fa..f9413ced36b 100644
--- a/integration-tests/load/vrfv2/vrfv2_test.go
+++ b/integration-tests/load/vrfv2/vrfv2_test.go
@@ -88,7 +88,7 @@ func TestVRFV2Performance(t *testing.T) {
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -226,7 +226,7 @@ func TestVRFV2BHSPerformance(t *testing.T) {
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go
index 1589123c77e..794f4115bd9 100644
--- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go
+++ b/integration-tests/load/vrfv2plus/vrfv2plus_test.go
@@ -87,7 +87,7 @@ func TestVRFV2PlusPerformance(t *testing.T) {
 			}
 		}
 		if !*testConfig.VRFv2Plus.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -223,7 +223,7 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) {
 			}
 		}
 		if !*testConfig.VRFv2Plus.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go
index 9834cd77973..dd26073b7ba 100644
--- a/integration-tests/smoke/vrfv2_test.go
+++ b/integration-tests/smoke/vrfv2_test.go
@@ -61,7 +61,7 @@ func TestVRFv2Basic(t *testing.T) {
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -554,7 +554,7 @@ func TestVRFv2MultipleSendingKeys(t *testing.T) {
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -662,7 +662,7 @@ func TestVRFOwner(t *testing.T) {
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -805,7 +805,7 @@ func TestVRFV2WithBHS(t *testing.T) {
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
-			if err := testEnv.Cleanup(); err != nil {
+			if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go
index c5a35704b1a..d6191f0fce8 100644
--- a/integration-tests/smoke/vrfv2plus_test.go
+++ b/integration-tests/smoke/vrfv2plus_test.go
@@ -63,7 +63,7 @@ func TestVRFv2Plus(t *testing.T) {
 			}
 		}
 		if !*vrfv2PlusConfig.General.UseExistingEnv {
-			if err := env.Cleanup(); err != nil {
+			if err := env.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -693,7 +693,7 @@ func TestVRFv2PlusMultipleSendingKeys(t *testing.T) {
 			}
 		}
 		if !*vrfv2PlusConfig.General.UseExistingEnv {
-			if err := env.Cleanup(); err != nil {
+			if err := env.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -800,7 +800,7 @@ func TestVRFv2PlusMigration(t *testing.T) {
 			}
 		}
 		if !*vrfv2PlusConfig.General.UseExistingEnv {
-			if err := env.Cleanup(); err != nil {
+			if err := env.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -1223,7 +1223,7 @@ func TestVRFV2PlusWithBHS(t *testing.T) {
 			}
 		}
 		if !*vrfv2PlusConfig.General.UseExistingEnv {
-			if err := env.Cleanup(); err != nil {
+			if err := env.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -1456,7 +1456,7 @@ func TestVRFV2PlusWithBHF(t *testing.T) {
 			}
 		}
 		if !*vrfv2PlusConfig.General.UseExistingEnv {
-			if err := env.Cleanup(); err != nil {
+			if err := env.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -1610,7 +1610,7 @@ func TestVRFv2PlusReplayAfterTimeout(t *testing.T) {
 			}
 		}
 		if !*vrfv2PlusConfig.General.UseExistingEnv {
-			if err := env.Cleanup(); err != nil {
+			if err := env.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}
@@ -1807,7 +1807,7 @@ func TestVRFv2PlusPendingBlockSimulationAndZeroConfirmationDelays(t *testing.T)
 			}
 		}
 		if !*vrfv2PlusConfig.General.UseExistingEnv {
-			if err := env.Cleanup(); err != nil {
+			if err := env.Cleanup(test_env.CleanupOpts{}); err != nil {
 				l.Error().Err(err).Msg("Error cleaning up test environment")
 			}
 		}

From 932d149d6a544ae0cb5d1c12fe061ee3932e56e8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 14:13:57 +0200
Subject: [PATCH 12/32] Show coverage in github workflow console

---
 .github/workflows/integration-tests.yml   |  6 ++++
 integration-tests/scripts/showCoverage.go | 41 +++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 integration-tests/scripts/showCoverage.go

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 9bdc91cf699..3855306bb50 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -394,6 +394,9 @@ jobs:
           QA_KUBECONFIG: ""
           should_tidy: "false"
 
+      - name: Show Go Coverage
+        run: go run ./integration-tests/scripts/showCoverage.go ./integration-tests/go-coverage
+
       - name: Upload Go Coverage Directory
         uses: actions/upload-artifact@v3
         with:
@@ -696,6 +699,9 @@ jobs:
           QA_KUBECONFIG: ""
           should_tidy: "false"
 
+      - name: Show Go Coverage
+        run: go run ./integration-tests/scripts/showCoverage.go ./integration-tests/go-coverage
+
       - name: Upload Go Coverage Directory
         uses: actions/upload-artifact@v3
         with:
diff --git a/integration-tests/scripts/showCoverage.go b/integration-tests/scripts/showCoverage.go
new file mode 100644
index 00000000000..93fec47af15
--- /dev/null
+++ b/integration-tests/scripts/showCoverage.go
@@ -0,0 +1,41 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+func main() {
+	// Check if an argument is provided
+	if len(os.Args) < 2 {
+		fmt.Println("Usage: showCoverage [root-directory]")
+		os.Exit(1)
+	}
+	root := os.Args[1] // Use the first command-line argument as the root directory
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() && info.Name() == "go-coverage" {
+			fmt.Println("Found go-coverage folder:", path)
+			// Run the go tool covdata percent command
+			cmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
+			cmd.Dir = path // set working directory
+			output, err := cmd.CombinedOutput()
+			if err != nil {
+				fmt.Println("Error executing command:", err)
+				return err
+			}
+			fmt.Println("Command:", cmd.String())
+			fmt.Printf("Output:\n%s\n", string(output))
+		}
+		return nil
+	})
+
+	if err != nil {
+		fmt.Println("Error walking the path:", err)
+	}
+}

From ec1c8c41c97dbe9afcff2a0c3564b2d4ae0c805f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 14:15:03 +0200
Subject: [PATCH 13/32] Fix lint

---
 .../docker/test_env/cl_node_cluster.go        | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index abd949c596a..2f7883009e3 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -3,11 +3,9 @@ package test_env
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
@@ -15,8 +13,6 @@ import (
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 
-	tc "github.com/testcontainers/testcontainers-go"
-
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 )
 
@@ -133,26 +129,6 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 	return nil
 }
 
-func lsFilesInContainer(ctx context.Context, container tc.Container, srcPath string) ([]string, error) {
-	// List all files and directories recursively inside the container
-	lsCmd := []string{"find", srcPath, "-type", "f"} // Lists only files, omitting directories
-	outputCode, outputReader, err := container.Exec(ctx, lsCmd)
-	if err != nil {
-		return nil, fmt.Errorf("failed to list files in container: %w", err)
-	}
-	// Read the output into a slice of file paths
-	output, err := io.ReadAll(outputReader)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read command output: %w", err)
-	}
-	if outputCode != 0 {
-		return nil, fmt.Errorf("failed to list files in container. Command exited with code: %d. Output: %s", outputCode, output)
-	}
-	files := strings.Split(string(output), "\n")
-
-	return files, nil
-}
-
 func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error {
 	source := fmt.Sprintf("%s:%s", containerID, srcPath)
 

From b6464fbf062265d4da910c7dec1fea78ad5556eb Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 14:56:31 +0200
Subject: [PATCH 14/32] Cleanup

---
 .../actions/build-chainlink-image/action.yml  |  4 ++--
 .github/workflows/integration-tests.yml       | 20 ++++---------------
 core/chainlink.Dockerfile                     | 13 ++++++------
 integration-tests/scripts/showCoverage.go     | 12 ++++++++++-
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml
index 0c7deccd8d2..363d467ded1 100644
--- a/.github/actions/build-chainlink-image/action.yml
+++ b/.github/actions/build-chainlink-image/action.yml
@@ -33,7 +33,7 @@ runs:
         AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
     - name: Build Image
       if: steps.check-image.outputs.exists != 'true'
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@f9f5d5e01a6c15764206f732f34d6cc0a85ad964
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@8531737ebe6d7ab79b2ca705e822c9fe811bffb7
       with:
         cl_repo: smartcontractkit/chainlink
         cl_ref: ${{ inputs.git_commit_sha }}
@@ -41,7 +41,7 @@ runs:
         push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
         QA_AWS_REGION: ${{ inputs.AWS_REGION }}
         QA_AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
-        COVER_FLAG: true
+        GO_COVER_FLAG: true
         dep_evm_sha: ${{ inputs.dep_evm_sha }}
     - name: Print Chainlink Image Built
       shell: sh
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 3855306bb50..8901c0ac52e 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -386,7 +386,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: ${{ github.workspace }}/integration-tests/go-coverage
+          go_coverage_dest_dir: /var/tmp/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
@@ -395,14 +395,8 @@ jobs:
           should_tidy: "false"
 
       - name: Show Go Coverage
-        run: go run ./integration-tests/scripts/showCoverage.go ./integration-tests/go-coverage
+        run: go run ./integration-tests/scripts/showCoverage.go /var/tmp/go-coverage
 
-      - name: Upload Go Coverage Directory
-        uses: actions/upload-artifact@v3
-        with:
-          name: go-coverage-${{ matrix.product.name }}
-          path: ./integration-tests/go-coverage/
-          
       - name: Print failed test summary
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
@@ -691,7 +685,7 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: ${{ github.workspace }}/integration-tests/go-coverage
+          go_coverage_dest_dir: /var/tmp/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
@@ -700,13 +694,7 @@ jobs:
           should_tidy: "false"
 
       - name: Show Go Coverage
-        run: go run ./integration-tests/scripts/showCoverage.go ./integration-tests/go-coverage
-
-      - name: Upload Go Coverage Directory
-        uses: actions/upload-artifact@v3
-        with:
-          name: go-coverage-${{ matrix.product.name }}
-          path: ./integration-tests/go-coverage/
+        run: go run ./integration-tests/scripts/showCoverage.go /var/tmp/go-coverage
 
       # Run this step when changes that do not need the test to run are made
       - name: Run Setup
diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile
index 96ca2147754..5c000fb7354 100644
--- a/core/chainlink.Dockerfile
+++ b/core/chainlink.Dockerfile
@@ -12,15 +12,15 @@ RUN go mod download
 # Env vars needed for chainlink build
 ARG COMMIT_SHA
 
-# Build chainlink bin with cover flag
-ARG COVER_FLAG=false
+# Build chainlink bin with cover flag https://go.dev/doc/build-cover#FAQ
+ARG GO_COVER_FLAG=false
 
 COPY . .
 
 RUN apt-get update && apt-get install -y jq
 
 # Build the golang binary
-RUN if [ "$COVER_FLAG" = "true" ]; then \
+RUN if [ "$GO_COVER_FLAG" = "true" ]; then \
         make install-chainlink-cover; \
     else \
         make install-chainlink; \
@@ -75,9 +75,10 @@ WORKDIR /home/${CHAINLINK_USER}
 ENV XDG_CACHE_HOME /home/${CHAINLINK_USER}/.cache
 RUN mkdir -p ${XDG_CACHE_HOME}
 
-# Set up go coverage dir
-ENV GOCOVERDIR="/var/tmp/go-coverage"
-RUN mkdir -p "/var/tmp/go-coverage"
+# Set up env and dir for go coverage profiling https://go.dev/doc/build-cover#FAQ
+ARG GO_COVER_DIR="/var/tmp/go-coverage"
+ENV GOCOVERDIR=${GO_COVER_DIR}
+RUN mkdir -p $GO_COVER_DIR
 
 EXPOSE 6688
 ENTRYPOINT ["chainlink"]
diff --git a/integration-tests/scripts/showCoverage.go b/integration-tests/scripts/showCoverage.go
index 93fec47af15..02bfcccb75a 100644
--- a/integration-tests/scripts/showCoverage.go
+++ b/integration-tests/scripts/showCoverage.go
@@ -15,12 +15,22 @@ func main() {
 	}
 	root := os.Args[1] // Use the first command-line argument as the root directory
 
+	// Validate the root directory before proceeding
+	if _, err := os.Stat(root); os.IsNotExist(err) {
+		fmt.Printf("The specified root directory does not exist: %s\n", root)
+		os.Exit(1)
+	}
+
 	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
+		// Skip the root directory itself
+		if path == root {
+			return nil
+		}
 		if info.IsDir() && info.Name() == "go-coverage" {
-			fmt.Println("Found go-coverage folder:", path)
+			fmt.Println("Found coverage profile for:", path)
 			// Run the go tool covdata percent command
 			cmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
 			cmd.Dir = path // set working directory

From e3e8cede2f1185df2fa0bc3088e83ed0f7d6f477 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 15:06:41 +0200
Subject: [PATCH 15/32] Add docs

---
 integration-tests/docker/test_env/cl_node_cluster.go | 7 +------
 integration-tests/scripts/showCoverage.go            | 2 ++
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go
index 2f7883009e3..679e5393811 100644
--- a/integration-tests/docker/test_env/cl_node_cluster.go
+++ b/integration-tests/docker/test_env/cl_node_cluster.go
@@ -104,7 +104,7 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 		go func(n *ClNode, id int) {
 			defer wg.Done()
 			// Create a unique subdirectory for each node based on an identifier
-			finalDestPath := filepath.Join(destPath, fmt.Sprintf("node_%d", id)) // Use node ID or another unique identifier
+			finalDestPath := filepath.Join(destPath, fmt.Sprintf("node_%d", id))
 			if err := os.MkdirAll(finalDestPath, 0755); err != nil {
 				errors <- fmt.Errorf("failed to create directory for node %d: %w", id, err)
 				return
@@ -131,14 +131,9 @@ func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath s
 
 func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error {
 	source := fmt.Sprintf("%s:%s", containerID, srcPath)
-
-	// Prepare the docker cp command
 	cmd := exec.CommandContext(ctx, "docker", "cp", source, destPath)
-
-	// Execute the docker cp command
 	if output, err := cmd.CombinedOutput(); err != nil {
 		return errors.Wrapf(err, "docker cp command failed: %s, output: %s", cmd, string(output))
 	}
-
 	return nil
 }
diff --git a/integration-tests/scripts/showCoverage.go b/integration-tests/scripts/showCoverage.go
index 02bfcccb75a..e1f76024119 100644
--- a/integration-tests/scripts/showCoverage.go
+++ b/integration-tests/scripts/showCoverage.go
@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 )
 
+// This script searches for "go-coverage" directories within the root-directory. For each "go-coverage" directory found,
+// the program executes a Go command to display coverage data.
 func main() {
 	// Check if an argument is provided
 	if len(os.Args) < 2 {

From 168ff0fbb1648df4d0a6448ba77766bbbf4f0d8a Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 15:11:10 +0200
Subject: [PATCH 16/32] Do not fail if no coverage dir was found

---
 integration-tests/scripts/showCoverage.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration-tests/scripts/showCoverage.go b/integration-tests/scripts/showCoverage.go
index e1f76024119..e102954e3da 100644
--- a/integration-tests/scripts/showCoverage.go
+++ b/integration-tests/scripts/showCoverage.go
@@ -19,8 +19,8 @@ func main() {
 
 	// Validate the root directory before proceeding
 	if _, err := os.Stat(root); os.IsNotExist(err) {
-		fmt.Printf("The specified root directory does not exist: %s\n", root)
-		os.Exit(1)
+		fmt.Printf("No coverage dir found: %s\n", root)
+		os.Exit(0)
 	}
 
 	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {

From b194975659eed6d410a7a2f2a454373fcea853ad Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 21:05:31 +0200
Subject: [PATCH 17/32] Update show_coverage.go to merge nodes coverage for
 each test

---
 .github/workflows/integration-tests.yml       |  4 +-
 integration-tests/docker/test_env/test_env.go |  4 +-
 integration-tests/scripts/showCoverage.go     | 53 ------------
 integration-tests/scripts/show_coverage.go    | 82 +++++++++++++++++++
 4 files changed, 87 insertions(+), 56 deletions(-)
 delete mode 100644 integration-tests/scripts/showCoverage.go
 create mode 100644 integration-tests/scripts/show_coverage.go

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 8901c0ac52e..4f7d0c6e484 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -395,7 +395,7 @@ jobs:
           should_tidy: "false"
 
       - name: Show Go Coverage
-        run: go run ./integration-tests/scripts/showCoverage.go /var/tmp/go-coverage
+        run: go run ./integration-tests/scripts/show_coverage.go /var/tmp/go-coverage
 
       - name: Print failed test summary
         if: always()
@@ -694,7 +694,7 @@ jobs:
           should_tidy: "false"
 
       - name: Show Go Coverage
-        run: go run ./integration-tests/scripts/showCoverage.go /var/tmp/go-coverage
+        run: go run ./integration-tests/scripts/show_coverage.go /var/tmp/go-coverage
 
       # Run this step when changes that do not need the test to run are made
       - name: Run Setup
diff --git a/integration-tests/docker/test_env/test_env.go b/integration-tests/docker/test_env/test_env.go
index 85b784b4aab..73270956bf9 100644
--- a/integration-tests/docker/test_env/test_env.go
+++ b/integration-tests/docker/test_env/test_env.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -256,7 +257,8 @@ func (te *CLClusterTestEnv) Cleanup(opts CleanupOpts) error {
 		}
 
 		// Get go coverage profiles from node containers and save them to a local folder
-		finalCovDestDir := fmt.Sprintf("%s/%s", covDestDir, opts.TestName)
+		testName := strings.ReplaceAll(opts.TestName, "/", "_")
+		finalCovDestDir := fmt.Sprintf("%s/%s", covDestDir, testName)
 		err = te.ClCluster.CopyFolderFromNodes(context.Background(), covSrcDir, finalCovDestDir)
 		if err != nil {
 			te.l.Error().Err(err).Str("srcDir", covSrcDir).Str("destDir", finalCovDestDir).Msg("Failed to copy test coverage files from nodes")
diff --git a/integration-tests/scripts/showCoverage.go b/integration-tests/scripts/showCoverage.go
deleted file mode 100644
index e102954e3da..00000000000
--- a/integration-tests/scripts/showCoverage.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-)
-
-// This script searches for "go-coverage" directories within the root-directory. For each "go-coverage" directory found,
-// the program executes a Go command to display coverage data.
-func main() {
-	// Check if an argument is provided
-	if len(os.Args) < 2 {
-		fmt.Println("Usage: showCoverage [root-directory]")
-		os.Exit(1)
-	}
-	root := os.Args[1] // Use the first command-line argument as the root directory
-
-	// Validate the root directory before proceeding
-	if _, err := os.Stat(root); os.IsNotExist(err) {
-		fmt.Printf("No coverage dir found: %s\n", root)
-		os.Exit(0)
-	}
-
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		// Skip the root directory itself
-		if path == root {
-			return nil
-		}
-		if info.IsDir() && info.Name() == "go-coverage" {
-			fmt.Println("Found coverage profile for:", path)
-			// Run the go tool covdata percent command
-			cmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
-			cmd.Dir = path // set working directory
-			output, err := cmd.CombinedOutput()
-			if err != nil {
-				fmt.Println("Error executing command:", err)
-				return err
-			}
-			fmt.Println("Command:", cmd.String())
-			fmt.Printf("Output:\n%s\n", string(output))
-		}
-		return nil
-	})
-
-	if err != nil {
-		fmt.Println("Error walking the path:", err)
-	}
-}
diff --git a/integration-tests/scripts/show_coverage.go b/integration-tests/scripts/show_coverage.go
new file mode 100644
index 00000000000..28977b83a1f
--- /dev/null
+++ b/integration-tests/scripts/show_coverage.go
@@ -0,0 +1,82 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// main manages the process of combining coverage data for Chainlink nodes.
+// It identifies "go-coverage" directories within a given root directory,
+// merges their data into a "merged" directory for each test, and then
+// calculates the overall coverage percentage.
+func main() {
+	// Check if an argument is provided
+	if len(os.Args) < 2 {
+		fmt.Println("Usage: show_coverage [root-directory]")
+		os.Exit(1)
+	}
+	root := os.Args[1] // Use the first command-line argument as the root directory
+
+	// Validate the root directory before proceeding
+	if _, err := os.Stat(root); os.IsNotExist(err) {
+		fmt.Printf("No coverage dir found: %s\n", root)
+		os.Exit(0)
+	}
+
+	testDirs := make(map[string][]string)
+
+	// Walk the file system from the root
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() && filepath.Base(path) == "go-coverage" && path != root {
+			// Assuming path structure /var/tmp/go-coverage/TestName/node_X/go-coverage
+			testName := filepath.Dir(filepath.Dir(path)) // This should get the test name directory
+			testDirs[testName] = append(testDirs[testName], path)
+		}
+		return nil
+	})
+
+	if err != nil {
+		fmt.Println("Error walking the path:", err)
+		os.Exit(1)
+	}
+
+	// Iterate over the map and run the merge command for each test
+	for test, dirs := range testDirs {
+		fmt.Printf("Merging coverage for test: %s\n", test)
+
+		// Ensure the merged directory exists
+		mergedDir := filepath.Join(test, "merged")
+		if err := os.MkdirAll(mergedDir, 0755); err != nil {
+			fmt.Printf("Failed to create merged directory %s: %v\n", mergedDir, err)
+			continue
+		}
+
+		// Merge the coverage data from all chainlink nodes
+		dirInput := strings.Join(dirs, ",")
+		mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", mergedDir, "-i="+dirInput)
+		mergeCmd.Dir = test // Set working directory
+		// fmt.Printf("Running merge command: %s\n", mergeCmd.String())
+		output, err := mergeCmd.CombinedOutput()
+		if err != nil {
+			fmt.Printf("Error executing merge command for %s: %v, output: %s\n", test, err, output)
+			os.Exit(1)
+		}
+
+		// Calculate coverage percentage in the merged directory
+		percentCmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
+		percentCmd.Dir = filepath.Join(test, "merged") // Set working directory to the merged folder
+		percentOutput, err := percentCmd.CombinedOutput()
+		if err != nil {
+			fmt.Printf("Error calculating coverage percentage for %s: %v\n", test, err)
+			continue
+		}
+		fmt.Printf("Coverage command: %s\n", percentCmd.String())
+		fmt.Printf("Coverage output:\n%s", string(percentOutput))
+	}
+}

From ccd37e3c6b56f49f75ccaf4204df2631a514f1f8 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Wed, 17 Apr 2024 21:38:32 +0200
Subject: [PATCH 18/32] Cleanup

---
 integration-tests/scripts/show_coverage.go | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/integration-tests/scripts/show_coverage.go b/integration-tests/scripts/show_coverage.go
index 28977b83a1f..2c033d665b4 100644
--- a/integration-tests/scripts/show_coverage.go
+++ b/integration-tests/scripts/show_coverage.go
@@ -48,7 +48,7 @@ func main() {
 
 	// Iterate over the map and run the merge command for each test
 	for test, dirs := range testDirs {
-		fmt.Printf("Merging coverage for test: %s\n", test)
+		testName := filepath.Base(test)
 
 		// Ensure the merged directory exists
 		mergedDir := filepath.Join(test, "merged")
@@ -60,8 +60,8 @@ func main() {
 		// Merge the coverage data from all chainlink nodes
 		dirInput := strings.Join(dirs, ",")
 		mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", mergedDir, "-i="+dirInput)
-		mergeCmd.Dir = test // Set working directory
-		// fmt.Printf("Running merge command: %s\n", mergeCmd.String())
+		mergeCmd.Dir = test
+		fmt.Printf("Merging coverage for %s:\n%s\n", testName, mergeCmd.String())
 		output, err := mergeCmd.CombinedOutput()
 		if err != nil {
 			fmt.Printf("Error executing merge command for %s: %v, output: %s\n", test, err, output)
@@ -69,14 +69,13 @@ func main() {
 		}
 
 		// Calculate coverage percentage in the merged directory
-		percentCmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
-		percentCmd.Dir = filepath.Join(test, "merged") // Set working directory to the merged folder
-		percentOutput, err := percentCmd.CombinedOutput()
+		coverageCmd := exec.Command("go", "tool", "covdata", "percent", "-i=merged")
+		coverageCmd.Dir = test
+		coverageOutput, err := coverageCmd.CombinedOutput()
 		if err != nil {
-			fmt.Printf("Error calculating coverage percentage for %s: %v\n", test, err)
+			fmt.Printf("Error calculating coverage for %s: %v, %s\n", test, err, string(coverageOutput))
 			continue
 		}
-		fmt.Printf("Coverage command: %s\n", percentCmd.String())
-		fmt.Printf("Coverage output:\n%s", string(percentOutput))
+		fmt.Printf("Total coverage for %s:\n%s\n%s\n\n", testName, coverageCmd.String(), string(coverageOutput))
 	}
 }

From 706167ecfed1bf25fbc36087d905860e072bc40b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 18 Apr 2024 10:56:56 +0200
Subject: [PATCH 19/32] Update gha

---
 .github/actions/build-chainlink-image/action.yml | 2 +-
 .github/workflows/integration-tests.yml          | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml
index 363d467ded1..cb2f03f054a 100644
--- a/.github/actions/build-chainlink-image/action.yml
+++ b/.github/actions/build-chainlink-image/action.yml
@@ -33,7 +33,7 @@ runs:
         AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
     - name: Build Image
       if: steps.check-image.outputs.exists != 'true'
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@8531737ebe6d7ab79b2ca705e822c9fe811bffb7
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@519851800779323566b7b7c22cc21bff95dbb639
       with:
         cl_repo: smartcontractkit/chainlink
         cl_ref: ${{ inputs.git_commit_sha }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 0ee8f974f0d..aee8e4d4614 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -470,7 +470,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -862,7 +862,7 @@ jobs:
           upgradeImage: ${{ env.UPGRADE_IMAGE }}
           upgradeVersion: ${{ env.UPGRADE_VERSION }}
       - name: Run Migration Tests
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -1162,7 +1162,7 @@ jobs:
           echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11
         with:
           test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke
           cl_repo: ${{ env.CHAINLINK_IMAGE }}

From fe2dfb3834f464d64235ffda4b689006eab3c2fb Mon Sep 17 00:00:00 2001
From: Lukasz <120112546+lukaszcl@users.noreply.github.com>
Date: Thu, 18 Apr 2024 22:14:50 +0200
Subject: [PATCH 20/32] Create NodeCoverageHelper and generate total coverage
 for all tests

---
 .github/workflows/integration-tests.yml       |  68 ++++++--
 .gitignore                                    |   1 +
 .../docker/node_coverage_helper.go            | 162 ++++++++++++++++++
 integration-tests/docker/test_env/test_env.go |  84 +++++++--
 integration-tests/go.mod                      |   4 +-
 integration-tests/go.sum                      |   4 +-
 integration-tests/load/go.mod                 |   2 +-
 integration-tests/load/go.sum                 |   4 +-
 integration-tests/scripts/show_coverage.go    | 107 ++++++------
 9 files changed, 354 insertions(+), 82 deletions(-)
 create mode 100644 integration-tests/docker/node_coverage_helper.go

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index aee8e4d4614..25321ed3a35 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -372,7 +372,7 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@be4b0f4b2e3bb5f9e56255f19e5a4809c515f954
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639
         with:
           test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
@@ -386,17 +386,20 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
-          go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: /var/tmp/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: .covdata
 
-      - name: Show Go Coverage
-        run: go run ./integration-tests/scripts/show_coverage.go /var/tmp/go-coverage
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@v3
+        with:
+          path: .covdata
+          retention-days: 1
 
       - name: Print failed test summary
         if: always()
@@ -490,6 +493,14 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: .covdata
+
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@v3
+        with:
+          path: .covdata
+          retention-days: 1          
 
   eth-smoke-tests-matrix:
     if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
@@ -672,9 +683,9 @@ jobs:
       ## Run this step when changes that require tests to be run are made
       - name: Run Tests
         if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
-        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@be4b0f4b2e3bb5f9e56255f19e5a4809c515f954
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639
         with:
-          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -onlyerrors=false
+          test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }}
@@ -686,17 +697,20 @@ jobs:
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
-          go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: /var/tmp/go-coverage
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
           cache_restore_only: "true"
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           should_tidy: "false"
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: .covdata
 
-      - name: Show Go Coverage
-        run: go run ./integration-tests/scripts/show_coverage.go /var/tmp/go-coverage
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@v3
+        with:
+          path: .covdata
+          retention-days: 1
 
       # Run this step when changes that do not need the test to run are made
       - name: Run Setup
@@ -790,6 +804,22 @@ jobs:
           this-job-name: Clean up integration environment deployments
         continue-on-error: true
 
+  show-coverage:
+    name: Show Chainlink Node Go Coverage
+    if: always()
+    needs: [cleanup]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        with:
+          repository: smartcontractkit/chainlink
+          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
+      - name: Download All Artifacts
+        uses: actions/download-artifact@v3
+      - name: Show Coverage
+        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/artifact*/*/merged"
+
   # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
   # this will also only run if both of the matrix jobs pass
   eth-smoke-go-mod-cache:
@@ -880,6 +910,14 @@ jobs:
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: .covdata
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@v3
+        with:
+          path: .covdata
+          retention-days: 1          
+
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics
@@ -1180,4 +1218,12 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           run_setup: false
+          go_coverage_src_dir: /var/tmp/go-coverage
+          go_coverage_dest_dir: .covdata          
+
+      - name: Upload Coverage Data
+        uses: actions/upload-artifact@v3
+        with:
+          path: .covdata
+          retention-days: 1          
 
diff --git a/.gitignore b/.gitignore
index 7d07300311f..5ea248c085d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,7 @@ output.txt
 race.*
 golangci-lint-output.txt
 /golangci-lint/
+.covdata
 
 # DB state
 ./db/
diff --git a/integration-tests/docker/node_coverage_helper.go b/integration-tests/docker/node_coverage_helper.go
new file mode 100644
index 00000000000..52531bd35cf
--- /dev/null
+++ b/integration-tests/docker/node_coverage_helper.go
@@ -0,0 +1,162 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+	tc "github.com/testcontainers/testcontainers-go"
+)
+
+type NodeCoverageHelper struct {
+	Nodes            []tc.Container
+	GoCoverSrcDir    string   // Path to the source directory on the chainlink image with go coverage data
+	NodeCoverageDirs []string // Paths to individual node coverage directories
+	CoverageDir      string   // Path to the base directory with all coverage
+	MergedDir        string   // Path to the directory where all coverage will be merged
+	ChainlinkDir     string   // Path to the root chainlink directory
+}
+
+func NewNodeCoverageHelper(ctx context.Context, nodes []tc.Container, chainlinkDir, coverageDir string) (*NodeCoverageHelper, error) {
+	coverSrcDir := os.Getenv("GO_COVERAGE_SRC_DIR")
+	if coverSrcDir == "" {
+		coverSrcDir = "/var/tmp/go-coverage" // Default path
+	}
+
+	helper := &NodeCoverageHelper{
+		Nodes:         nodes,
+		GoCoverSrcDir: coverSrcDir,
+		CoverageDir:   coverageDir,
+		MergedDir:     filepath.Join(coverageDir, "merged"),
+		ChainlinkDir:  chainlinkDir,
+	}
+
+	if err := os.MkdirAll(coverageDir, 0755); err != nil {
+		return nil, errors.Wrap(err, "failed to create base directory for node coverage")
+	}
+
+	// Copy coverage data from nodes
+	if err := helper.copyCoverageFromNodes(ctx); err != nil {
+		return nil, errors.Wrap(err, "failed to copy coverage from nodes during initialization")
+	}
+
+	// Merge the coverage data
+	if err := helper.mergeCoverage(); err != nil {
+		return nil, errors.Wrap(err, "failed to merge coverage data")
+	}
+
+	return helper, nil
+}
+
+func (c *NodeCoverageHelper) SaveMergedHTMLReport() (string, error) {
+	// Generate the textual coverage report
+	txtCommand := exec.Command("go", "tool", "covdata", "textfmt", "-i=.", "-o=cov.txt")
+	txtCommand.Dir = c.MergedDir
+	if txtOutput, err := txtCommand.CombinedOutput(); err != nil {
+		return "", errors.Wrapf(err, "failed to generate textual coverage report: %s", string(txtOutput))
+	}
+
+	// Generate the HTML coverage report
+	htmlFilePath := filepath.Join(c.CoverageDir, "coverage.html")
+	// #nosec G204
+	htmlCommand := exec.Command("go", "tool", "cover", "-html="+filepath.Join(c.MergedDir, "cov.txt"), "-o="+htmlFilePath)
+	htmlCommand.Dir = c.ChainlinkDir
+	if htmlOutput, err := htmlCommand.CombinedOutput(); err != nil {
+		return "", errors.Wrapf(err, "failed to generate HTML coverage report: %s", string(htmlOutput))
+	}
+
+	return htmlFilePath, nil
+}
+
+func (c *NodeCoverageHelper) SaveMergedCoveragePercentage() (string, error) {
+	filePath := filepath.Join(c.CoverageDir, "percentage.txt")
+
+	// Calculate coverage percentage from the merged data
+	percentCmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
+	percentCmd.Dir = c.MergedDir // Ensure the command runs in the directory with the merged data
+	output, err := percentCmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("failed to get merged coverage percentage report: %w, output: %s", err, string(output))
+	}
+
+	// Save the cmd output to a file
+	if err := os.WriteFile(filePath, output, 0600); err != nil {
+		return "", errors.Wrap(err, "failed to write coverage percentage to file")
+	}
+
+	return filePath, nil
+}
+
+func (c *NodeCoverageHelper) mergeCoverage() error {
+	if err := os.MkdirAll(c.MergedDir, 0755); err != nil {
+		return fmt.Errorf("failed to create merged directory: %w", err)
+	}
+
+	// Merge the coverage data from all chainlink nodes
+	dirInput := strings.Join(c.NodeCoverageDirs, ",")
+	// #nosec G204
+	mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", c.MergedDir, "-i="+dirInput)
+	mergeCmd.Dir = filepath.Dir(c.MergedDir)
+	output, err := mergeCmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error executing merge command: %w, output: %s", err, string(output))
+	}
+
+	// Remove the coverage dirs after merging
+	for _, dir := range c.NodeCoverageDirs {
+		if err := os.RemoveAll(dir); err != nil {
+			return fmt.Errorf("failed to remove directory %s: %w", dir, err)
+		}
+	}
+	c.NodeCoverageDirs = []string{} // Reset the coverage paths after merging
+
+	return nil
+}
+
+func (c *NodeCoverageHelper) copyCoverageFromNodes(ctx context.Context) error {
+	var wg sync.WaitGroup
+	errorsChan := make(chan error, len(c.Nodes))
+
+	for i, node := range c.Nodes {
+		wg.Add(1)
+		go func(n tc.Container, id int) {
+			defer wg.Done()
+			finalDestPath := filepath.Join(c.CoverageDir, fmt.Sprintf("node_%d", id))
+			if err := os.MkdirAll(finalDestPath, 0755); err != nil {
+				errorsChan <- fmt.Errorf("failed to create directory for node %d: %w", id, err)
+				return
+			}
+			err := copyFolderFromContainerUsingDockerCP(ctx, n.GetContainerID(), c.GoCoverSrcDir, finalDestPath)
+			if err != nil {
+				errorsChan <- fmt.Errorf("failed to copy folder from container for node %d: %w", id, err)
+				return
+			}
+			finalDestPath = filepath.Join(finalDestPath, "go-coverage") // Assuming path structure /var/tmp/go-coverage/TestName/node_X/go-coverage
+			c.NodeCoverageDirs = append(c.NodeCoverageDirs, finalDestPath)
+		}(node, i)
+	}
+
+	wg.Wait()
+	close(errorsChan)
+
+	for err := range errorsChan {
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error {
+	source := fmt.Sprintf("%s:%s", containerID, srcPath)
+	cmd := exec.CommandContext(ctx, "docker", "cp", source, destPath)
+	if output, err := cmd.CombinedOutput(); err != nil {
+		return errors.Wrapf(err, "docker cp command failed: %s, output: %s", cmd, string(output))
+	}
+	return nil
+}
diff --git a/integration-tests/docker/test_env/test_env.go b/integration-tests/docker/test_env/test_env.go
index f88e0905368..dbe5cb6f202 100644
--- a/integration-tests/docker/test_env/test_env.go
+++ b/integration-tests/docker/test_env/test_env.go
@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -26,6 +28,7 @@ import (
 	actions_seth "github.com/smartcontractkit/chainlink/integration-tests/actions/seth"
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
+	d "github.com/smartcontractkit/chainlink/integration-tests/docker"
 	core_testconfig "github.com/smartcontractkit/chainlink/integration-tests/testconfig"
 )
 
@@ -235,6 +238,11 @@ func (te *CLClusterTestEnv) Cleanup(opts CleanupOpts) error {
 		}
 	}
 
+	err := te.handleNodeCoverageReports(opts.TestName)
+	if err != nil {
+		te.l.Error().Err(err).Msg("Error handling node coverage reports")
+	}
+
 	// close EVMClient connections
 	for _, evmClient := range te.evmClients {
 		err := evmClient.Close()
@@ -245,11 +253,24 @@ func (te *CLClusterTestEnv) Cleanup(opts CleanupOpts) error {
 		sethClient.Client.Close()
 	}
 
-	covSrcDir := os.Getenv("GO_COVERAGE_SRC_DIR")
-	covDestDir := os.Getenv("GO_COVERAGE_DEST_DIR")
-	shouldCheckCoverage := covSrcDir != "" && covDestDir != ""
+	return nil
+}
+
+// handleNodeCoverageReports handles the coverage reports for the chainlink nodes
+func (te *CLClusterTestEnv) handleNodeCoverageReports(testName string) error {
+	testName = strings.ReplaceAll(testName, "/", "_")
+	showHTMLCoverageReport := te.TestConfig.GetLoggingConfig().ShowHTMLCoverageReport != nil && *te.TestConfig.GetLoggingConfig().ShowHTMLCoverageReport
+	isCI := os.Getenv("CI") != ""
+
+	te.l.Info().
+		Bool("showCoverageReportFlag", showHTMLCoverageReport).
+		Bool("isCI", isCI).
+		Bool("show", showHTMLCoverageReport || isCI).
+		Msg("Checking if coverage report should be shown")
+
+	var covHelper *d.NodeCoverageHelper
 
-	if shouldCheckCoverage {
+	if showHTMLCoverageReport || isCI {
 		// Stop all nodes in the chainlink cluster.
 		// This is needed to get go coverage profile from the node containers https://go.dev/doc/build-cover#FAQ
 		err := te.ClCluster.Stop()
@@ -257,20 +278,63 @@ func (te *CLClusterTestEnv) Cleanup(opts CleanupOpts) error {
 			return err
 		}
 
-		// Get go coverage profiles from node containers and save them to a local folder
-		testName := strings.ReplaceAll(opts.TestName, "/", "_")
-		finalCovDestDir := fmt.Sprintf("%s/%s", covDestDir, testName)
-		err = te.ClCluster.CopyFolderFromNodes(context.Background(), covSrcDir, finalCovDestDir)
+		clDir, err := getChainlinkDir()
 		if err != nil {
-			te.l.Error().Err(err).Str("srcDir", covSrcDir).Str("destDir", finalCovDestDir).Msg("Failed to copy test coverage files from nodes")
+			return err
+		}
+
+		var coverageRootDir string
+		if os.Getenv("GO_COVERAGE_DEST_DIR") != "" {
+			coverageRootDir = filepath.Join(clDir, os.Getenv("GO_COVERAGE_DEST_DIR"), testName)
 		} else {
-			te.l.Info().Str("srcDir", covSrcDir).Str("destDir", finalCovDestDir).Msg("Chainlink node coverage files saved")
+			coverageRootDir = filepath.Join(clDir, ".covdata", testName)
+		}
+
+		var containers []tc.Container
+		for _, node := range te.ClCluster.Nodes {
+			containers = append(containers, node.Container)
+		}
+
+		covHelper, err = d.NewNodeCoverageHelper(context.Background(), containers, clDir, coverageRootDir)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Show html coverage report when flag is set (local runs)
+	if showHTMLCoverageReport {
+		path, err := covHelper.SaveMergedHTMLReport()
+		if err != nil {
+			return err
+		}
+		te.l.Info().Str("testName", testName).Str("filePath", path).Msg("Chainlink node coverage html report saved")
+	}
+
+	// Save percentage coverage report when running in CI
+	if isCI {
+		// Save coverage percentage to a file to show in the CI
+		path, err := covHelper.SaveMergedCoveragePercentage()
+		if err != nil {
+			te.l.Error().Err(err).Str("testName", testName).Msg("Failed to save coverage percentage for test")
+		} else {
+			te.l.Info().Str("testName", testName).Str("filePath", path).Msg("Chainlink node coverage percentage report saved")
 		}
 	}
 
 	return nil
 }
 
+// getChainlinkDir returns the path to the chainlink directory
+func getChainlinkDir() (string, error) {
+	_, filename, _, ok := runtime.Caller(1)
+	if !ok {
+		return "", fmt.Errorf("cannot determine the path of the calling file")
+	}
+	dir := filepath.Dir(filename)
+	chainlinkDir := filepath.Clean(filepath.Join(dir, "../../.."))
+	return chainlinkDir, nil
+}
+
 func (te *CLClusterTestEnv) logWhetherAllContainersAreRunning() {
 	for _, node := range te.ClCluster.Nodes {
 		if node.Container == nil {
diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index 2042495382a..06470cf6808 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240415164156-8872a8f311cb
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.3
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052
@@ -502,4 +502,6 @@ replace (
 
 	// type func(a Label, b Label) bool of func(a, b Label) bool {…} does not match inferred type func(a Label, b Label) int for func(a E, b E) int
 	github.com/prometheus/prometheus => github.com/prometheus/prometheus v0.47.2-0.20231010075449-4b9c19fe5510
+
+// github.com/smartcontractkit/chainlink-testing-framework => /Users/lukasz/Documents/smartcontractkit/chainlink-testing-framework
 )
diff --git a/integration-tests/go.sum b/integration-tests/go.sum
index c054bcc21b0..7701c779be8 100644
--- a/integration-tests/go.sum
+++ b/integration-tests/go.sum
@@ -1533,8 +1533,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.3 h1:rZ622PUSE9jJvI2g1SNNcMJedXyMzq9XJ8SbV2j9TvA=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859 h1:NQolsMjelZC4qpaQElLhWMPcKuUxVwEezwUdGih66o0=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index 882a998feec..1a3e2ef0636 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240415164156-8872a8f311cb
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.3
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052
diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum
index d36ce67be0c..88a37e04b83 100644
--- a/integration-tests/load/go.sum
+++ b/integration-tests/load/go.sum
@@ -1516,8 +1516,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.3 h1:rZ622PUSE9jJvI2g1SNNcMJedXyMzq9XJ8SbV2j9TvA=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859 h1:NQolsMjelZC4qpaQElLhWMPcKuUxVwEezwUdGih66o0=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea h1:ZdLmNAfKRjH8AYUvjiiDGUgiWQfq/7iNpxyTkvjx/ko=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea/go.mod h1:gCKC9w6XpNk6jm+XIk2psrkkfxhi421N9NSiFceXW88=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo=
diff --git a/integration-tests/scripts/show_coverage.go b/integration-tests/scripts/show_coverage.go
index 2c033d665b4..0491ec744a2 100644
--- a/integration-tests/scripts/show_coverage.go
+++ b/integration-tests/scripts/show_coverage.go
@@ -8,74 +8,71 @@ import (
 	"strings"
 )
 
-// main manages the process of combining coverage data for Chainlink nodes.
-// It identifies "go-coverage" directories within a given root directory,
-// merges their data into a "merged" directory for each test, and then
-// calculates the overall coverage percentage.
+// main manages the process of combining coverage data for all tests
 func main() {
-	// Check if an argument is provided
+	// Check if the user has provided an argument
 	if len(os.Args) < 2 {
-		fmt.Println("Usage: show_coverage [root-directory]")
+		fmt.Println("Usage: go run script.go <searchPattern>")
 		os.Exit(1)
 	}
-	root := os.Args[1] // Use the first command-line argument as the root directory
 
-	// Validate the root directory before proceeding
-	if _, err := os.Stat(root); os.IsNotExist(err) {
-		fmt.Printf("No coverage dir found: %s\n", root)
-		os.Exit(0)
+	// First argument after the program name is the search pattern
+	searchPattern := os.Args[1]
+
+	// Glob pattern to find all 'merged' directories in artifact folders
+	dirs, err := filepath.Glob(searchPattern)
+	if err != nil {
+		fmt.Printf("Failed to find directories: %v\n", err)
+		os.Exit(1)
 	}
 
-	testDirs := make(map[string][]string)
+	if len(dirs) == 0 {
+		fmt.Println("No directories found.")
+		return
+	}
 
-	// Walk the file system from the root
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() && filepath.Base(path) == "go-coverage" && path != root {
-			// Assuming path structure /var/tmp/go-coverage/TestName/node_X/go-coverage
-			testName := filepath.Dir(filepath.Dir(path)) // This should get the test name directory
-			testDirs[testName] = append(testDirs[testName], path)
-		}
-		return nil
-	})
+	fmt.Printf("Found directories with test coverage data: %v\n", dirs)
 
-	if err != nil {
-		fmt.Println("Error walking the path:", err)
+	// Join the directory paths for input
+	dirInput := strings.Join(dirs, ",")
+
+	// Ensure the merged directory exists
+	mergedDir := filepath.Join(".covdata", "merged")
+	if err := os.MkdirAll(mergedDir, 0755); err != nil {
+		fmt.Printf("Failed to create merged directory %s: %v\n", mergedDir, err)
 		os.Exit(1)
 	}
 
-	// Iterate over the map and run the merge command for each test
-	for test, dirs := range testDirs {
-		testName := filepath.Base(test)
-
-		// Ensure the merged directory exists
-		mergedDir := filepath.Join(test, "merged")
-		if err := os.MkdirAll(mergedDir, 0755); err != nil {
-			fmt.Printf("Failed to create merged directory %s: %v\n", mergedDir, err)
-			continue
-		}
+	// Merge the coverage data from all chainlink nodes
+	mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", mergedDir, "-i="+dirInput)
+	fmt.Printf("Merging coverage data for all tests:\n%s\n", mergeCmd.String())
+	output, err := mergeCmd.CombinedOutput()
+	if err != nil {
+		fmt.Printf("Error executing merge command: %v, output: %s\n", err, output)
+		os.Exit(1)
+	}
 
-		// Merge the coverage data from all chainlink nodes
-		dirInput := strings.Join(dirs, ",")
-		mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", mergedDir, "-i="+dirInput)
-		mergeCmd.Dir = test
-		fmt.Printf("Merging coverage for %s:\n%s\n", testName, mergeCmd.String())
-		output, err := mergeCmd.CombinedOutput()
-		if err != nil {
-			fmt.Printf("Error executing merge command for %s: %v, output: %s\n", test, err, output)
-			os.Exit(1)
-		}
+	// Calculate coverage percentage in the merged directory
+	coverageCmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
+	coverageCmd.Dir = mergedDir
+	fmt.Printf("Calculate total coverage for on all tests: %s\n", coverageCmd.String())
+	coverageOutput, err := coverageCmd.CombinedOutput()
+	if err != nil {
+		fmt.Printf("Error calculating coverage percentage: %v\n", err)
+		os.Exit(1)
+	}
 
-		// Calculate coverage percentage in the merged directory
-		coverageCmd := exec.Command("go", "tool", "covdata", "percent", "-i=merged")
-		coverageCmd.Dir = test
-		coverageOutput, err := coverageCmd.CombinedOutput()
-		if err != nil {
-			fmt.Printf("Error calculating coverage for %s: %v, %s\n", test, err, string(coverageOutput))
-			continue
-		}
-		fmt.Printf("Total coverage for %s:\n%s\n%s\n\n", testName, coverageCmd.String(), string(coverageOutput))
+	// Save the coverage percentage to a file
+	filePath, err := filepath.Abs(filepath.Join(mergedDir, "percentage.txt"))
+	if err != nil {
+		fmt.Printf("Error obtaining absolute path: %s\n", err)
+		os.Exit(1)
+	}
+	if err := os.WriteFile(filePath, coverageOutput, 0600); err != nil {
+		fmt.Printf("Failed to write coverage percentage to file: %v\n", err)
+		os.Exit(1)
 	}
+	fmt.Printf("Total coverage for all tests saved to %s\n", filePath)
+
+	fmt.Printf("Total coverage for all tests:\n%s\n", string(coverageOutput))
 }

From 4ca1e85604a409a2be1a62712ff82086d3c6493f Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 19 Apr 2024 12:24:17 +0200
Subject: [PATCH 21/32] Update covdata artifact name

---
 .github/workflows/integration-tests.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 25321ed3a35..911d3b60e5f 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -398,6 +398,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
+          name: covdata
           path: .covdata
           retention-days: 1
 
@@ -499,6 +500,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
+          name: covdata
           path: .covdata
           retention-days: 1          
 
@@ -709,6 +711,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
+          name: covdata
           path: .covdata
           retention-days: 1
 
@@ -817,8 +820,10 @@ jobs:
           ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
       - name: Download All Artifacts
         uses: actions/download-artifact@v3
+        with:
+          name: covdata
       - name: Show Coverage
-        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/artifact*/*/merged"
+        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/covdata*/*/merged"
 
   # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
   # this will also only run if both of the matrix jobs pass
@@ -915,6 +920,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
+          name: covdata
           path: .covdata
           retention-days: 1          
 
@@ -1224,6 +1230,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
+          name: covdata
           path: .covdata
           retention-days: 1          
 

From 14f42fdb1b2404702a80be6b71ade9cb4b395741 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 19 Apr 2024 13:04:03 +0200
Subject: [PATCH 22/32] Fix

---
 .github/workflows/integration-tests.yml | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 911d3b60e5f..6ee8eef7151 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -398,7 +398,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
-          name: covdata
+          name: cl-node-coverage-data
           path: .covdata
           retention-days: 1
 
@@ -500,7 +500,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
-          name: covdata
+          name: cl-node-coverage-data
           path: .covdata
           retention-days: 1          
 
@@ -711,7 +711,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
-          name: covdata
+          name: cl-node-coverage-data
           path: .covdata
           retention-days: 1
 
@@ -821,9 +821,12 @@ jobs:
       - name: Download All Artifacts
         uses: actions/download-artifact@v3
         with:
-          name: covdata
+          name: cl-node-coverage-data
+          path: cl-node-coverage-data
+      - name: List files in the downloaded folder
+        run: ls -R          
       - name: Show Coverage
-        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/covdata*/*/merged"
+        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged"
 
   # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
   # this will also only run if both of the matrix jobs pass
@@ -920,7 +923,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
-          name: covdata
+          name: cl-node-coverage-data
           path: .covdata
           retention-days: 1          
 
@@ -1230,7 +1233,7 @@ jobs:
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
-          name: covdata
+          name: cl-node-coverage-data
           path: .covdata
           retention-days: 1          
 

From 7cda8a03ecf32b13bb4ce144eb0d6caa56fe6c53 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 19 Apr 2024 14:36:45 +0200
Subject: [PATCH 23/32] Remove unnecessary build step

---
 .github/workflows/integration-tests.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 6ee8eef7151..5d84a0d73af 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -823,8 +823,6 @@ jobs:
         with:
           name: cl-node-coverage-data
           path: cl-node-coverage-data
-      - name: List files in the downloaded folder
-        run: ls -R          
       - name: Show Coverage
         run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged"
 

From f212743a3b25f29b026343279676a99d9491284b Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 19 Apr 2024 16:04:15 +0200
Subject: [PATCH 24/32] Update README

---
 integration-tests/docker/README.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/integration-tests/docker/README.md b/integration-tests/docker/README.md
index 7df481832e6..4615a9ec9a6 100644
--- a/integration-tests/docker/README.md
+++ b/integration-tests/docker/README.md
@@ -20,3 +20,27 @@ cd ./integration-tests/docker/cmd
 
 go run test_env.go start-env cl-cluster
 ```
+
+## Obtaining Test Coverage for Chainlink Node
+
+To acquire test coverage data for end-to-end (E2E) tests on the Chainlink Node, follow these steps:
+
+1. Build Chainlink Node docker image with the cover flag.
+
+    First, build the Chainlink Node Docker image with the `GO_COVER_FLAG` argument set to `true`. This enables the coverage flag in the build process. Here’s how you can do it:
+    ```
+    docker buildx build --platform linux/arm64 . -t localhost/chainlink-local:develop -f ./core/chainlink.Dockerfile --build-arg GO_COVER_FLAG=true
+    ```
+    Make sure to replace localhost/chainlink-local:develop with the appropriate repository and tag.
+
+2. Configure and Run E2E Tests
+    Next, configure the E2E tests to generate an HTML coverage report. You need to modify the `overrides.toml` file as shown below to include the show_html_coverage_report setting under the `[Logging]` section:
+
+    ```
+    [Logging]
+    show_html_coverage_report=true
+    ```
+
+After the tests are complete, the coverage report will be generated in HTML format. 
+
+

From 7df33ad5bb3134fdd1e7320213499bb68a5a0db9 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 19 Apr 2024 16:28:53 +0200
Subject: [PATCH 25/32] Fix

---
 integration-tests/docker/test_env/test_env.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/docker/test_env/test_env.go b/integration-tests/docker/test_env/test_env.go
index dbe5cb6f202..fc0ba355556 100644
--- a/integration-tests/docker/test_env/test_env.go
+++ b/integration-tests/docker/test_env/test_env.go
@@ -285,7 +285,7 @@ func (te *CLClusterTestEnv) handleNodeCoverageReports(testName string) error {
 
 		var coverageRootDir string
 		if os.Getenv("GO_COVERAGE_DEST_DIR") != "" {
-			coverageRootDir = filepath.Join(clDir, os.Getenv("GO_COVERAGE_DEST_DIR"), testName)
+			coverageRootDir = filepath.Join(os.Getenv("GO_COVERAGE_DEST_DIR"), testName)
 		} else {
 			coverageRootDir = filepath.Join(clDir, ".covdata", testName)
 		}

From 0cb0fe6aff0aa588b27d684dc376ccc192e35f26 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 19 Apr 2024 16:31:28 +0200
Subject: [PATCH 26/32] Update readme

---
 integration-tests/docker/README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/integration-tests/docker/README.md b/integration-tests/docker/README.md
index 4615a9ec9a6..c558f3bca8a 100644
--- a/integration-tests/docker/README.md
+++ b/integration-tests/docker/README.md
@@ -41,6 +41,8 @@ To acquire test coverage data for end-to-end (E2E) tests on the Chainlink Node,
     show_html_coverage_report=true
     ```
 
-After the tests are complete, the coverage report will be generated in HTML format. 
-
+After the tests are complete, the coverage report will be generated in HTML format. Example: `~/Downloads/go-coverage/TestOCRv2Basic_plugins-chain-reader/coverage.html`
+```
+    log.go:43: 16:29:46.73 INF Chainlink node coverage html report saved filePath=~/Downloads/go-coverage/TestOCRv2Basic_plugins-chain-reader/coverage.html
+```
 

From 17b6c56cc8f86d22e19614c4c5000c27c2ec5641 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Fri, 19 Apr 2024 17:27:50 +0200
Subject: [PATCH 27/32] Fix

---
 .github/workflows/integration-tests.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 5d84a0d73af..42e8c430f89 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -393,7 +393,7 @@ jobs:
           QA_KUBECONFIG: ""
           should_tidy: "false"
           go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: .covdata
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
@@ -495,7 +495,7 @@ jobs:
           QA_KUBECONFIG: ""
           should_tidy: "false"
           go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: .covdata
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
@@ -706,7 +706,7 @@ jobs:
           QA_KUBECONFIG: ""
           should_tidy: "false"
           go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: .covdata
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
@@ -917,7 +917,7 @@ jobs:
           QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
           go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: .covdata
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3
         with:
@@ -1226,7 +1226,7 @@ jobs:
           QA_KUBECONFIG: ""
           run_setup: false
           go_coverage_src_dir: /var/tmp/go-coverage
-          go_coverage_dest_dir: .covdata          
+          go_coverage_dest_dir: ${{ github.workspace }}/.covdata          
 
       - name: Upload Coverage Data
         uses: actions/upload-artifact@v3

From e238b296750b82808524c2dedbedc42e8bd62056 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 22 Apr 2024 13:05:57 +0200
Subject: [PATCH 28/32] Fixes

---
 .../actions/build-chainlink-image/action.yml  |  2 +-
 .github/workflows/integration-tests.yml       | 27 ++++++++++---------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml
index cb2f03f054a..bd5ffe6403d 100644
--- a/.github/actions/build-chainlink-image/action.yml
+++ b/.github/actions/build-chainlink-image/action.yml
@@ -33,7 +33,7 @@ runs:
         AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
     - name: Build Image
       if: steps.check-image.outputs.exists != 'true'
-      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@519851800779323566b7b7c22cc21bff95dbb639
+      uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.14
       with:
         cl_repo: smartcontractkit/chainlink
         cl_ref: ${{ inputs.git_commit_sha }}
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 42e8c430f89..7ec4b00471e 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -396,9 +396,9 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: cl-node-coverage-data
+          name: cl-node-coverage-data-${{ matrix.product.name }}
           path: .covdata
           retention-days: 1
 
@@ -498,9 +498,9 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: cl-node-coverage-data
+          name: cl-node-coverage-data-${{ matrix.product.name }}
           path: .covdata
           retention-days: 1          
 
@@ -709,9 +709,9 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: cl-node-coverage-data
+          name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }}
           path: .covdata
           retention-days: 1
 
@@ -819,10 +819,13 @@ jobs:
           repository: smartcontractkit/chainlink
           ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
       - name: Download All Artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: cl-node-coverage-data
           path: cl-node-coverage-data
+          pattern: cl-node-coverage-data-*
+          merge-multiple: true
+      - name: List all files and directories
+        run: ls -R
       - name: Show Coverage
         run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged"
 
@@ -919,9 +922,9 @@ jobs:
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: cl-node-coverage-data
+          name: cl-node-coverage-data-migration-tests
           path: .covdata
           retention-days: 1          
 
@@ -1229,9 +1232,9 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata          
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: cl-node-coverage-data
+          name: cl-node-coverage-data-solana-tests
           path: .covdata
           retention-days: 1          
 

From dde4c499fee24af4ef4659b1f6be0c5173858813 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 22 Apr 2024 15:06:28 +0200
Subject: [PATCH 29/32] Remove ls step from workflow

---
 .github/workflows/integration-tests.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 7ec4b00471e..2d5b692663a 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -824,8 +824,6 @@ jobs:
           path: cl-node-coverage-data
           pattern: cl-node-coverage-data-*
           merge-multiple: true
-      - name: List all files and directories
-        run: ls -R
       - name: Show Coverage
         run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged"
 

From cd3b8f5a8fcf0099353dbd0ee4a5b38f2d9d31aa Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Mon, 22 Apr 2024 18:41:22 +0200
Subject: [PATCH 30/32] Fix action version and tag

---
 .github/workflows/integration-tests.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 2d5b692663a..3ba0f2976ec 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -396,7 +396,7 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: cl-node-coverage-data-${{ matrix.product.name }}
           path: .covdata
@@ -498,7 +498,7 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: cl-node-coverage-data-${{ matrix.product.name }}
           path: .covdata
@@ -709,7 +709,7 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }}
           path: .covdata
@@ -819,7 +819,7 @@ jobs:
           repository: smartcontractkit/chainlink
           ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
       - name: Download All Artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
         with:
           path: cl-node-coverage-data
           pattern: cl-node-coverage-data-*
@@ -920,7 +920,7 @@ jobs:
           go_coverage_src_dir: /var/tmp/go-coverage
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: cl-node-coverage-data-migration-tests
           path: .covdata
@@ -1230,7 +1230,7 @@ jobs:
           go_coverage_dest_dir: ${{ github.workspace }}/.covdata          
 
       - name: Upload Coverage Data
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: cl-node-coverage-data-solana-tests
           path: .covdata

From 712e4c36272ae046d87e7618263bc1b2e52831c6 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 23 Apr 2024 16:10:19 +0200
Subject: [PATCH 31/32] Bump CTF

---
 integration-tests/go.mod      | 2 +-
 integration-tests/go.sum      | 4 ++--
 integration-tests/load/go.mod | 2 +-
 integration-tests/load/go.sum | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index 67c7f07b445..80531dc97b0 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240419105123-fc5d616c7d2e
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
diff --git a/integration-tests/go.sum b/integration-tests/go.sum
index 8b09bb55ba7..20d2d6d7a23 100644
--- a/integration-tests/go.sum
+++ b/integration-tests/go.sum
@@ -1531,8 +1531,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859 h1:NQolsMjelZC4qpaQElLhWMPcKuUxVwEezwUdGih66o0=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7 h1:QaWiyEOZxkrzAmw5I0kjMFifhk73tlV9qbW5aYNh86w=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index a307bbb2685..768a767e62a 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240419105123-fc5d616c7d2e
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum
index 7ac169bd53e..12a39f4dbd7 100644
--- a/integration-tests/load/go.sum
+++ b/integration-tests/load/go.sum
@@ -1514,8 +1514,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859 h1:NQolsMjelZC4qpaQElLhWMPcKuUxVwEezwUdGih66o0=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240418154117-56aa01f31859/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7 h1:QaWiyEOZxkrzAmw5I0kjMFifhk73tlV9qbW5aYNh86w=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea h1:ZdLmNAfKRjH8AYUvjiiDGUgiWQfq/7iNpxyTkvjx/ko=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea/go.mod h1:gCKC9w6XpNk6jm+XIk2psrkkfxhi421N9NSiFceXW88=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo=

From fb07194f33f02e4bb96f23b62fc36457ec4c3dd9 Mon Sep 17 00:00:00 2001
From: lukaszcl <120112546+lukaszcl@users.noreply.github.com>
Date: Tue, 23 Apr 2024 16:53:37 +0200
Subject: [PATCH 32/32] Bump CTF

---
 integration-tests/go.mod      | 2 +-
 integration-tests/go.sum      | 4 ++--
 integration-tests/load/go.mod | 2 +-
 integration-tests/load/go.sum | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/integration-tests/go.mod b/integration-tests/go.mod
index 421e122eac9..017d3de4bae 100644
--- a/integration-tests/go.mod
+++ b/integration-tests/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240419205832-845fa69af8d9
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.4
 	github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772
 	github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
diff --git a/integration-tests/go.sum b/integration-tests/go.sum
index b7dbccce076..b3a7d5d4d5c 100644
--- a/integration-tests/go.sum
+++ b/integration-tests/go.sum
@@ -1529,8 +1529,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba5
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba58/go.mod h1:oV5gIuSKrPEcjQ6uB6smBsm5kXHxyydVLNyAs4V9CoQ=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7 h1:QaWiyEOZxkrzAmw5I0kjMFifhk73tlV9qbW5aYNh86w=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4 h1:/OOPH76VFQlG5HEXrXgBVDv1fjuasQzMV1EyeaaXWzM=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs=
 github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod
index 8740b65b78b..c43760fcd76 100644
--- a/integration-tests/load/go.mod
+++ b/integration-tests/load/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/slack-go/slack v0.12.2
 	github.com/smartcontractkit/chainlink-automation v1.0.3
 	github.com/smartcontractkit/chainlink-common v0.1.7-0.20240419205832-845fa69af8d9
-	github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7
+	github.com/smartcontractkit/chainlink-testing-framework v1.28.4
 	github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
 	github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
 	github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c
diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum
index 93ba9d84a62..3b572b59a40 100644
--- a/integration-tests/load/go.sum
+++ b/integration-tests/load/go.sum
@@ -1512,8 +1512,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba5
 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba58/go.mod h1:oV5gIuSKrPEcjQ6uB6smBsm5kXHxyydVLNyAs4V9CoQ=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw=
 github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7 h1:QaWiyEOZxkrzAmw5I0kjMFifhk73tlV9qbW5aYNh86w=
-github.com/smartcontractkit/chainlink-testing-framework v1.28.4-0.20240423140805-12d0c1613ff7/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4 h1:/OOPH76VFQlG5HEXrXgBVDv1fjuasQzMV1EyeaaXWzM=
+github.com/smartcontractkit/chainlink-testing-framework v1.28.4/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea h1:ZdLmNAfKRjH8AYUvjiiDGUgiWQfq/7iNpxyTkvjx/ko=
 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea/go.mod h1:gCKC9w6XpNk6jm+XIk2psrkkfxhi421N9NSiFceXW88=
 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo=