diff --git a/.github/actions/build-chainlink-image/action.yml b/.github/actions/build-chainlink-image/action.yml index f75bd68a87e..bd5ffe6403d 100644 --- a/.github/actions/build-chainlink-image/action.yml +++ b/.github/actions/build-chainlink-image/action.yml @@ -33,7 +33,7 @@ runs: AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }} - name: Build Image if: steps.check-image.outputs.exists != 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.14 with: cl_repo: smartcontractkit/chainlink cl_ref: ${{ inputs.git_commit_sha }} @@ -41,6 +41,7 @@ runs: push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }} QA_AWS_REGION: ${{ inputs.AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }} + GO_COVER_FLAG: true dep_evm_sha: ${{ inputs.dep_evm_sha }} - name: Print Chainlink Image Built shell: sh diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 98d67a8b2d3..3ba0f2976ec 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -372,7 +372,7 @@ jobs: ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -392,6 +392,16 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" should_tidy: "false" + go_coverage_src_dir: /var/tmp/go-coverage + go_coverage_dest_dir: ${{ github.workspace }}/.covdata + + - name: Upload Coverage Data + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: cl-node-coverage-data-${{ matrix.product.name }} + path: .covdata + retention-days: 1 + - name: Print failed test summary if: always() uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 @@ -464,7 +474,7 @@ jobs: ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -484,6 +494,15 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" should_tidy: "false" + go_coverage_src_dir: /var/tmp/go-coverage + go_coverage_dest_dir: ${{ github.workspace }}/.covdata + + - name: Upload Coverage Data + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: cl-node-coverage-data-${{ matrix.product.name }} + path: .covdata + retention-days: 1 eth-smoke-tests-matrix: if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} @@ -666,7 +685,7 @@ jobs: ## Run this step when changes that require tests to be run are made - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -686,6 +705,16 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" should_tidy: "false" + go_coverage_src_dir: /var/tmp/go-coverage + go_coverage_dest_dir: ${{ github.workspace }}/.covdata + + - name: Upload Coverage Data + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: cl-node-coverage-data-${{ matrix.product.name }}-${{ matrix.product.tag_suffix }} + path: .covdata + retention-days: 1 + # Run this step when changes that do not need the test to run are made - name: Run Setup if: needs.changes.outputs.src == 'false' @@ -778,6 +807,26 @@ jobs: this-job-name: Clean up integration environment deployments continue-on-error: true + show-coverage: + name: Show Chainlink Node Go Coverage + if: always() + needs: [cleanup] + runs-on: ubuntu-latest + steps: + - name: Checkout the repo + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + repository: smartcontractkit/chainlink + ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Download All Artifacts + uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6 + with: + path: cl-node-coverage-data + pattern: cl-node-coverage-data-* + merge-multiple: true + - name: Show Coverage + run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl-node-coverage-data/*/merged" + # Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss # this will also only run if both of the matrix jobs pass eth-smoke-go-mod-cache: @@ -850,7 +899,7 @@ jobs: upgradeImage: ${{ env.UPGRADE_IMAGE }} upgradeVersion: ${{ env.UPGRADE_VERSION }} - name: Run Migration Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11 with: test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json ./migration 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage test_download_vendor_packages_command: cd ./integration-tests && go mod download @@ -868,6 +917,15 @@ jobs: QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + go_coverage_src_dir: /var/tmp/go-coverage + go_coverage_dest_dir: ${{ github.workspace }}/.covdata + - name: Upload Coverage Data + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: cl-node-coverage-data-migration-tests + path: .covdata + retention-days: 1 + - name: Collect Metrics if: always() id: collect-gha-metrics @@ -1150,7 +1208,7 @@ jobs: echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV - name: Run Tests if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639 # v2.3.11 with: test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke cl_repo: ${{ env.CHAINLINK_IMAGE }} @@ -1168,4 +1226,13 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" run_setup: false + go_coverage_src_dir: /var/tmp/go-coverage + go_coverage_dest_dir: ${{ github.workspace }}/.covdata + + - name: Upload Coverage Data + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: cl-node-coverage-data-solana-tests + path: .covdata + retention-days: 1 diff --git a/.gitignore b/.gitignore index 0a938ff9123..b286c329292 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ output.txt race.* golangci-lint-output.txt /golangci-lint/ +.covdata # DB state ./db/ diff --git a/GNUmakefile b/GNUmakefile index c5a0dfc6c21..524786585e9 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -45,6 +45,10 @@ docs: ## Install and run pkgsite to view Go docs install-chainlink: operator-ui ## Install the chainlink binary. go install $(GOFLAGS) . +.PHONY: install-chainlink-cover +install-chainlink-cover: operator-ui ## Install the chainlink binary with cover flag. + go install -cover $(GOFLAGS) . + .PHONY: chainlink chainlink: ## Build the chainlink binary. go build $(GOFLAGS) . diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index 4aa447b5ddd..48f6da73612 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -12,12 +12,19 @@ RUN go mod download # Env vars needed for chainlink build ARG COMMIT_SHA +# Build chainlink bin with cover flag https://go.dev/doc/build-cover#FAQ +ARG GO_COVER_FLAG=false + COPY . . RUN apt-get update && apt-get install -y jq # Build the golang binary -RUN make install-chainlink +RUN if [ "$GO_COVER_FLAG" = "true" ]; then \ + make install-chainlink-cover; \ + else \ + make install-chainlink; \ + fi # Link LOOP Plugin source dirs with simple names RUN go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-feeds | xargs -I % ln -s % /chainlink-feeds @@ -68,6 +75,11 @@ WORKDIR /home/${CHAINLINK_USER} ENV XDG_CACHE_HOME /home/${CHAINLINK_USER}/.cache RUN mkdir -p ${XDG_CACHE_HOME} +# Set up env and dir for go coverage profiling https://go.dev/doc/build-cover#FAQ +ARG GO_COVER_DIR="/var/tmp/go-coverage" +ENV GOCOVERDIR=${GO_COVER_DIR} +RUN mkdir -p $GO_COVER_DIR + EXPOSE 6688 ENTRYPOINT ["chainlink"] diff --git a/integration-tests/docker/README.md b/integration-tests/docker/README.md index 7df481832e6..c558f3bca8a 100644 --- a/integration-tests/docker/README.md +++ b/integration-tests/docker/README.md @@ -20,3 +20,29 @@ cd ./integration-tests/docker/cmd go run test_env.go start-env cl-cluster ``` + +## Obtaining Test Coverage for Chainlink Node + +To acquire test coverage data for end-to-end (E2E) tests on the Chainlink Node, follow these steps: + +1. Build Chainlink Node docker image with the cover flag. + + First, build the Chainlink Node Docker image with the `GO_COVER_FLAG` argument set to `true`. This enables the coverage flag in the build process. Here’s how you can do it: + ``` + docker buildx build --platform linux/arm64 . -t localhost/chainlink-local:develop -f ./core/chainlink.Dockerfile --build-arg GO_COVER_FLAG=true + ``` + Make sure to replace localhost/chainlink-local:develop with the appropriate repository and tag. + +2. Configure and Run E2E Tests + Next, configure the E2E tests to generate an HTML coverage report. You need to modify the `overrides.toml` file as shown below to include the show_html_coverage_report setting under the `[Logging]` section: + + ``` + [Logging] + show_html_coverage_report=true + ``` + +After the tests are complete, the coverage report will be generated in HTML format. Example: `~/Downloads/go-coverage/TestOCRv2Basic_plugins-chain-reader/coverage.html` +``` + log.go:43: 16:29:46.73 INF Chainlink node coverage html report saved filePath=~/Downloads/go-coverage/TestOCRv2Basic_plugins-chain-reader/coverage.html +``` + diff --git a/integration-tests/docker/node_coverage_helper.go b/integration-tests/docker/node_coverage_helper.go new file mode 100644 index 00000000000..52531bd35cf --- /dev/null +++ b/integration-tests/docker/node_coverage_helper.go @@ -0,0 +1,162 @@ +package docker + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "github.com/pkg/errors" + tc "github.com/testcontainers/testcontainers-go" +) + +type NodeCoverageHelper struct { + Nodes []tc.Container + GoCoverSrcDir string // Path to the source directory on the chainlink image with go coverage data + NodeCoverageDirs []string // Paths to individual node coverage directories + CoverageDir string // Path to the base directory with all coverage + MergedDir string // Path to the directory where all coverage will be merged + ChainlinkDir string // Path to the root chainlink directory +} + +func NewNodeCoverageHelper(ctx context.Context, nodes []tc.Container, chainlinkDir, coverageDir string) (*NodeCoverageHelper, error) { + coverSrcDir := os.Getenv("GO_COVERAGE_SRC_DIR") + if coverSrcDir == "" { + coverSrcDir = "/var/tmp/go-coverage" // Default path + } + + helper := &NodeCoverageHelper{ + Nodes: nodes, + GoCoverSrcDir: coverSrcDir, + CoverageDir: coverageDir, + MergedDir: filepath.Join(coverageDir, "merged"), + ChainlinkDir: chainlinkDir, + } + + if err := os.MkdirAll(coverageDir, 0755); err != nil { + return nil, errors.Wrap(err, "failed to create base directory for node coverage") + } + + // Copy coverage data from nodes + if err := helper.copyCoverageFromNodes(ctx); err != nil { + return nil, errors.Wrap(err, "failed to copy coverage from nodes during initialization") + } + + // Merge the coverage data + if err := helper.mergeCoverage(); err != nil { + return nil, errors.Wrap(err, "failed to merge coverage data") + } + + return helper, nil +} + +func (c *NodeCoverageHelper) SaveMergedHTMLReport() (string, error) { + // Generate the textual coverage report + txtCommand := exec.Command("go", "tool", "covdata", "textfmt", "-i=.", "-o=cov.txt") + txtCommand.Dir = c.MergedDir + if txtOutput, err := txtCommand.CombinedOutput(); err != nil { + return "", errors.Wrapf(err, "failed to generate textual coverage report: %s", string(txtOutput)) + } + + // Generate the HTML coverage report + htmlFilePath := filepath.Join(c.CoverageDir, "coverage.html") + // #nosec G204 + htmlCommand := exec.Command("go", "tool", "cover", "-html="+filepath.Join(c.MergedDir, "cov.txt"), "-o="+htmlFilePath) + htmlCommand.Dir = c.ChainlinkDir + if htmlOutput, err := htmlCommand.CombinedOutput(); err != nil { + return "", errors.Wrapf(err, "failed to generate HTML coverage report: %s", string(htmlOutput)) + } + + return htmlFilePath, nil +} + +func (c *NodeCoverageHelper) SaveMergedCoveragePercentage() (string, error) { + filePath := filepath.Join(c.CoverageDir, "percentage.txt") + + // Calculate coverage percentage from the merged data + percentCmd := exec.Command("go", "tool", "covdata", "percent", "-i=.") + percentCmd.Dir = c.MergedDir // Ensure the command runs in the directory with the merged data + output, err := percentCmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to get merged coverage percentage report: %w, output: %s", err, string(output)) + } + + // Save the cmd output to a file + if err := os.WriteFile(filePath, output, 0600); err != nil { + return "", errors.Wrap(err, "failed to write coverage percentage to file") + } + + return filePath, nil +} + +func (c *NodeCoverageHelper) mergeCoverage() error { + if err := os.MkdirAll(c.MergedDir, 0755); err != nil { + return fmt.Errorf("failed to create merged directory: %w", err) + } + + // Merge the coverage data from all chainlink nodes + dirInput := strings.Join(c.NodeCoverageDirs, ",") + // #nosec G204 + mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", c.MergedDir, "-i="+dirInput) + mergeCmd.Dir = filepath.Dir(c.MergedDir) + output, err := mergeCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("error executing merge command: %w, output: %s", err, string(output)) + } + + // Remove the coverage dirs after merging + for _, dir := range c.NodeCoverageDirs { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("failed to remove directory %s: %w", dir, err) + } + } + c.NodeCoverageDirs = []string{} // Reset the coverage paths after merging + + return nil +} + +func (c *NodeCoverageHelper) copyCoverageFromNodes(ctx context.Context) error { + var wg sync.WaitGroup + errorsChan := make(chan error, len(c.Nodes)) + + for i, node := range c.Nodes { + wg.Add(1) + go func(n tc.Container, id int) { + defer wg.Done() + finalDestPath := filepath.Join(c.CoverageDir, fmt.Sprintf("node_%d", id)) + if err := os.MkdirAll(finalDestPath, 0755); err != nil { + errorsChan <- fmt.Errorf("failed to create directory for node %d: %w", id, err) + return + } + err := copyFolderFromContainerUsingDockerCP(ctx, n.GetContainerID(), c.GoCoverSrcDir, finalDestPath) + if err != nil { + errorsChan <- fmt.Errorf("failed to copy folder from container for node %d: %w", id, err) + return + } + finalDestPath = filepath.Join(finalDestPath, "go-coverage") // Assuming path structure /var/tmp/go-coverage/TestName/node_X/go-coverage + c.NodeCoverageDirs = append(c.NodeCoverageDirs, finalDestPath) + }(node, i) + } + + wg.Wait() + close(errorsChan) + + for err := range errorsChan { + if err != nil { + return err + } + } + + return nil +} +func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error { + source := fmt.Sprintf("%s:%s", containerID, srcPath) + cmd := exec.CommandContext(ctx, "docker", "cp", source, destPath) + if output, err := cmd.CombinedOutput(); err != nil { + return errors.Wrapf(err, "docker cp command failed: %s, output: %s", cmd, string(output)) + } + return nil +} diff --git a/integration-tests/docker/test_env/cl_node_cluster.go b/integration-tests/docker/test_env/cl_node_cluster.go index 08122b5744d..679e5393811 100644 --- a/integration-tests/docker/test_env/cl_node_cluster.go +++ b/integration-tests/docker/test_env/cl_node_cluster.go @@ -1,9 +1,16 @@ package test_env import ( + "context" "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + "time" "github.com/ethereum/go-ethereum/common" + "github.com/pkg/errors" "golang.org/x/sync/errgroup" "github.com/smartcontractkit/chainlink/integration-tests/client" @@ -36,6 +43,25 @@ func (c *ClCluster) Start() error { return eg.Wait() } +func (c *ClCluster) Stop() error { + eg := &errgroup.Group{} + nodes := c.Nodes + timeout := time.Minute * 1 + + for i := 0; i < len(nodes); i++ { + nodeIndex := i + eg.Go(func() error { + err := nodes[nodeIndex].Container.Stop(context.Background(), &timeout) + if err != nil { + return err + } + return nil + }) + } + + return eg.Wait() +} + func (c *ClCluster) NodeAPIs() []*client.ChainlinkClient { clients := make([]*client.ChainlinkClient, 0) for _, c := range c.Nodes { @@ -68,3 +94,46 @@ func (c *ClCluster) NodeCSAKeys() ([]string, error) { } return keys, nil } + +func (c *ClCluster) CopyFolderFromNodes(ctx context.Context, srcPath, destPath string) error { + var wg sync.WaitGroup + errors := make(chan error, len(c.Nodes)) + + for i, node := range c.Nodes { + wg.Add(1) + go func(n *ClNode, id int) { + defer wg.Done() + // Create a unique subdirectory for each node based on an identifier + finalDestPath := filepath.Join(destPath, fmt.Sprintf("node_%d", id)) + if err := os.MkdirAll(finalDestPath, 0755); err != nil { + errors <- fmt.Errorf("failed to create directory for node %d: %w", id, err) + return + } + err := copyFolderFromContainerUsingDockerCP(ctx, n.Container.GetContainerID(), srcPath, finalDestPath) + if err != nil { + errors <- fmt.Errorf("failed to copy folder for node %d: %w", id, err) + return + } + }(node, i) + } + + wg.Wait() + close(errors) + + for err := range errors { + if err != nil { + return err + } + } + + return nil +} + +func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error { + source := fmt.Sprintf("%s:%s", containerID, srcPath) + cmd := exec.CommandContext(ctx, "docker", "cp", source, destPath) + if output, err := cmd.CombinedOutput(); err != nil { + return errors.Wrapf(err, "docker cp command failed: %s, output: %s", cmd, string(output)) + } + return nil +} diff --git a/integration-tests/docker/test_env/test_env.go b/integration-tests/docker/test_env/test_env.go index d54f51eeea1..fc0ba355556 100644 --- a/integration-tests/docker/test_env/test_env.go +++ b/integration-tests/docker/test_env/test_env.go @@ -1,9 +1,14 @@ package test_env import ( + "context" "encoding/json" "fmt" "math/big" + "os" + "path/filepath" + "runtime" + "strings" "testing" "github.com/ethereum/go-ethereum/accounts/keystore" @@ -23,6 +28,7 @@ import ( actions_seth "github.com/smartcontractkit/chainlink/integration-tests/actions/seth" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" + d "github.com/smartcontractkit/chainlink/integration-tests/docker" core_testconfig "github.com/smartcontractkit/chainlink/integration-tests/testconfig" ) @@ -198,8 +204,12 @@ func (te *CLClusterTestEnv) Terminate() error { return nil } +type CleanupOpts struct { + TestName string +} + // Cleanup cleans the environment up after it's done being used, mainly for returning funds when on live networks and logs. -func (te *CLClusterTestEnv) Cleanup() error { +func (te *CLClusterTestEnv) Cleanup(opts CleanupOpts) error { te.l.Info().Msg("Cleaning up test environment") runIdErr := runid.RemoveLocalRunId(te.TestConfig.GetLoggingConfig().RunId) @@ -228,6 +238,11 @@ func (te *CLClusterTestEnv) Cleanup() error { } } + err := te.handleNodeCoverageReports(opts.TestName) + if err != nil { + te.l.Error().Err(err).Msg("Error handling node coverage reports") + } + // close EVMClient connections for _, evmClient := range te.evmClients { err := evmClient.Close() @@ -241,6 +256,85 @@ func (te *CLClusterTestEnv) Cleanup() error { return nil } +// handleNodeCoverageReports handles the coverage reports for the chainlink nodes +func (te *CLClusterTestEnv) handleNodeCoverageReports(testName string) error { + testName = strings.ReplaceAll(testName, "/", "_") + showHTMLCoverageReport := te.TestConfig.GetLoggingConfig().ShowHTMLCoverageReport != nil && *te.TestConfig.GetLoggingConfig().ShowHTMLCoverageReport + isCI := os.Getenv("CI") != "" + + te.l.Info(). + Bool("showCoverageReportFlag", showHTMLCoverageReport). + Bool("isCI", isCI). + Bool("show", showHTMLCoverageReport || isCI). + Msg("Checking if coverage report should be shown") + + var covHelper *d.NodeCoverageHelper + + if showHTMLCoverageReport || isCI { + // Stop all nodes in the chainlink cluster. + // This is needed to get go coverage profile from the node containers https://go.dev/doc/build-cover#FAQ + err := te.ClCluster.Stop() + if err != nil { + return err + } + + clDir, err := getChainlinkDir() + if err != nil { + return err + } + + var coverageRootDir string + if os.Getenv("GO_COVERAGE_DEST_DIR") != "" { + coverageRootDir = filepath.Join(os.Getenv("GO_COVERAGE_DEST_DIR"), testName) + } else { + coverageRootDir = filepath.Join(clDir, ".covdata", testName) + } + + var containers []tc.Container + for _, node := range te.ClCluster.Nodes { + containers = append(containers, node.Container) + } + + covHelper, err = d.NewNodeCoverageHelper(context.Background(), containers, clDir, coverageRootDir) + if err != nil { + return err + } + } + + // Show html coverage report when flag is set (local runs) + if showHTMLCoverageReport { + path, err := covHelper.SaveMergedHTMLReport() + if err != nil { + return err + } + te.l.Info().Str("testName", testName).Str("filePath", path).Msg("Chainlink node coverage html report saved") + } + + // Save percentage coverage report when running in CI + if isCI { + // Save coverage percentage to a file to show in the CI + path, err := covHelper.SaveMergedCoveragePercentage() + if err != nil { + te.l.Error().Err(err).Str("testName", testName).Msg("Failed to save coverage percentage for test") + } else { + te.l.Info().Str("testName", testName).Str("filePath", path).Msg("Chainlink node coverage percentage report saved") + } + } + + return nil +} + +// getChainlinkDir returns the path to the chainlink directory +func getChainlinkDir() (string, error) { + _, filename, _, ok := runtime.Caller(1) + if !ok { + return "", fmt.Errorf("cannot determine the path of the calling file") + } + dir := filepath.Dir(filename) + chainlinkDir := filepath.Clean(filepath.Join(dir, "../../..")) + return chainlinkDir, nil +} + func (te *CLClusterTestEnv) logWhetherAllContainersAreRunning() { for _, node := range te.ClCluster.Nodes { if node.Container == nil { diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go index b9406bf16aa..c2aa07c8fa9 100644 --- a/integration-tests/docker/test_env/test_env_builder.go +++ b/integration-tests/docker/test_env/test_env_builder.go @@ -258,7 +258,8 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { switch b.cleanUpType { case CleanUpTypeStandard: b.t.Cleanup(func() { - if err := b.te.Cleanup(); err != nil { + // Cleanup test environment + if err := b.te.Cleanup(CleanupOpts{TestName: b.t.Name()}); err != nil { b.l.Error().Err(err).Msg("Error cleaning up test environment") } }) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 4f166385c69..327e5e69d94 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -26,7 +26,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240419205832-845fa69af8d9 - github.com/smartcontractkit/chainlink-testing-framework v1.28.3 + github.com/smartcontractkit/chainlink-testing-framework v1.28.4 github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 65b67170dea..577ae3f2a0d 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1529,8 +1529,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba5 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba58/go.mod h1:oV5gIuSKrPEcjQ6uB6smBsm5kXHxyydVLNyAs4V9CoQ= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg= -github.com/smartcontractkit/chainlink-testing-framework v1.28.3 h1:rZ622PUSE9jJvI2g1SNNcMJedXyMzq9XJ8SbV2j9TvA= -github.com/smartcontractkit/chainlink-testing-framework v1.28.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= +github.com/smartcontractkit/chainlink-testing-framework v1.28.4 h1:/OOPH76VFQlG5HEXrXgBVDv1fjuasQzMV1EyeaaXWzM= +github.com/smartcontractkit/chainlink-testing-framework v1.28.4/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo= github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 42e97969260..eaf0332af63 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.3 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240419205832-845fa69af8d9 - github.com/smartcontractkit/chainlink-testing-framework v1.28.3 + github.com/smartcontractkit/chainlink-testing-framework v1.28.4 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240419185742-fd3cab206b2c diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 5572271a58f..631336409b3 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1512,8 +1512,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba5 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240422172640-59d47c73ba58/go.mod h1:oV5gIuSKrPEcjQ6uB6smBsm5kXHxyydVLNyAs4V9CoQ= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595 h1:y6ks0HsSOhPUueOmTcoxDQ50RCS1XINlRDTemZyHjFw= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240325075535-0f7eb05ee595/go.mod h1:vV6WfnVIbK5Q1JsIru4YcTG0T1uRpLJm6t2BgCnCSsg= -github.com/smartcontractkit/chainlink-testing-framework v1.28.3 h1:rZ622PUSE9jJvI2g1SNNcMJedXyMzq9XJ8SbV2j9TvA= -github.com/smartcontractkit/chainlink-testing-framework v1.28.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= +github.com/smartcontractkit/chainlink-testing-framework v1.28.4 h1:/OOPH76VFQlG5HEXrXgBVDv1fjuasQzMV1EyeaaXWzM= +github.com/smartcontractkit/chainlink-testing-framework v1.28.4/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea h1:ZdLmNAfKRjH8AYUvjiiDGUgiWQfq/7iNpxyTkvjx/ko= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea/go.mod h1:gCKC9w6XpNk6jm+XIk2psrkkfxhi421N9NSiFceXW88= github.com/smartcontractkit/chainlink-vrf v0.0.0-20240222010609-cd67d123c772 h1:LQmRsrzzaYYN3wEU1l5tWiccznhvbyGnu2N+wHSXZAo= diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go index e420d66ec8c..ca41dd40ea9 100644 --- a/integration-tests/load/vrfv2/vrfv2_test.go +++ b/integration-tests/load/vrfv2/vrfv2_test.go @@ -86,7 +86,7 @@ func TestVRFV2Performance(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -230,7 +230,7 @@ func TestVRFV2BHSPerformance(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go index d461b4cde20..e24b2ff17d8 100644 --- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go +++ b/integration-tests/load/vrfv2plus/vrfv2plus_test.go @@ -86,7 +86,7 @@ func TestVRFV2PlusPerformance(t *testing.T) { } } if !*testConfig.VRFv2Plus.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -229,7 +229,7 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) { } } if !*testConfig.VRFv2Plus.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } diff --git a/integration-tests/scripts/show_coverage.go b/integration-tests/scripts/show_coverage.go new file mode 100644 index 00000000000..0491ec744a2 --- /dev/null +++ b/integration-tests/scripts/show_coverage.go @@ -0,0 +1,78 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// main manages the process of combining coverage data for all tests +func main() { + // Check if the user has provided an argument + if len(os.Args) < 2 { + fmt.Println("Usage: go run script.go ") + os.Exit(1) + } + + // First argument after the program name is the search pattern + searchPattern := os.Args[1] + + // Glob pattern to find all 'merged' directories in artifact folders + dirs, err := filepath.Glob(searchPattern) + if err != nil { + fmt.Printf("Failed to find directories: %v\n", err) + os.Exit(1) + } + + if len(dirs) == 0 { + fmt.Println("No directories found.") + return + } + + fmt.Printf("Found directories with test coverage data: %v\n", dirs) + + // Join the directory paths for input + dirInput := strings.Join(dirs, ",") + + // Ensure the merged directory exists + mergedDir := filepath.Join(".covdata", "merged") + if err := os.MkdirAll(mergedDir, 0755); err != nil { + fmt.Printf("Failed to create merged directory %s: %v\n", mergedDir, err) + os.Exit(1) + } + + // Merge the coverage data from all chainlink nodes + mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", mergedDir, "-i="+dirInput) + fmt.Printf("Merging coverage data for all tests:\n%s\n", mergeCmd.String()) + output, err := mergeCmd.CombinedOutput() + if err != nil { + fmt.Printf("Error executing merge command: %v, output: %s\n", err, output) + os.Exit(1) + } + + // Calculate coverage percentage in the merged directory + coverageCmd := exec.Command("go", "tool", "covdata", "percent", "-i=.") + coverageCmd.Dir = mergedDir + fmt.Printf("Calculate total coverage for on all tests: %s\n", coverageCmd.String()) + coverageOutput, err := coverageCmd.CombinedOutput() + if err != nil { + fmt.Printf("Error calculating coverage percentage: %v\n", err) + os.Exit(1) + } + + // Save the coverage percentage to a file + filePath, err := filepath.Abs(filepath.Join(mergedDir, "percentage.txt")) + if err != nil { + fmt.Printf("Error obtaining absolute path: %s\n", err) + os.Exit(1) + } + if err := os.WriteFile(filePath, coverageOutput, 0600); err != nil { + fmt.Printf("Failed to write coverage percentage to file: %v\n", err) + os.Exit(1) + } + fmt.Printf("Total coverage for all tests saved to %s\n", filePath) + + fmt.Printf("Total coverage for all tests:\n%s\n", string(coverageOutput)) +} diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go index 77fe49f422d..62c5dc84fe6 100644 --- a/integration-tests/smoke/vrfv2_test.go +++ b/integration-tests/smoke/vrfv2_test.go @@ -63,7 +63,7 @@ func TestVRFv2Basic(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -556,7 +556,7 @@ func TestVRFv2MultipleSendingKeys(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -664,7 +664,7 @@ func TestVRFOwner(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -807,7 +807,7 @@ func TestVRFV2WithBHS(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index 36fc9d5268d..c535e77ef62 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -64,7 +64,7 @@ func TestVRFv2Plus(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -719,7 +719,7 @@ func TestVRFv2PlusMultipleSendingKeys(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -826,7 +826,7 @@ func TestVRFv2PlusMigration(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1249,7 +1249,7 @@ func TestVRFV2PlusWithBHS(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1470,7 +1470,7 @@ func TestVRFV2PlusWithBHF(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1618,7 +1618,7 @@ func TestVRFv2PlusReplayAfterTimeout(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1817,7 +1817,7 @@ func TestVRFv2PlusPendingBlockSimulationAndZeroConfirmationDelays(t *testing.T) } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } }