Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Show E2E test coverage of chainlink node binary #12835

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
26d43f4
Get E2E test coverage of chainlink node binary
lukaszcl Apr 15, 2024
b01b883
Fix
lukaszcl Apr 16, 2024
57a1d88
Fix lint
lukaszcl Apr 16, 2024
9f6f748
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 16, 2024
69debaa
Add step to upload go coverage folder
lukaszcl Apr 16, 2024
c04cbff
Fix
lukaszcl Apr 16, 2024
3f0c6f1
debug
lukaszcl Apr 16, 2024
a3315f7
Fix
lukaszcl Apr 16, 2024
27c5176
Fix lint
lukaszcl Apr 16, 2024
c4617a4
Debug
lukaszcl Apr 16, 2024
7d8aed8
Fix
lukaszcl Apr 16, 2024
958916c
Fix
lukaszcl Apr 17, 2024
932d149
Show coverage in github workflow console
lukaszcl Apr 17, 2024
ec1c8c4
Fix lint
lukaszcl Apr 17, 2024
b6464fb
Cleanup
lukaszcl Apr 17, 2024
e3e8ced
Add docs
lukaszcl Apr 17, 2024
168ff0f
Do not fail if no coverage dir was found
lukaszcl Apr 17, 2024
b194975
Update show_coverage.go to merge nodes coverage for each test
lukaszcl Apr 17, 2024
ccd37e3
Cleanup
lukaszcl Apr 17, 2024
a8d82a7
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 18, 2024
706167e
Update gha
lukaszcl Apr 18, 2024
fe2dfb3
Create NodeCoverageHelper and generate total coverage for all tests
lukaszcl Apr 18, 2024
4ca1e85
Update covdata artifact name
lukaszcl Apr 19, 2024
14f42fd
Fix
lukaszcl Apr 19, 2024
f4991b5
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 19, 2024
7cda8a0
Remove unnecessary build step
lukaszcl Apr 19, 2024
f212743
Update README
lukaszcl Apr 19, 2024
7df33ad
Fix
lukaszcl Apr 19, 2024
0cb0fe6
Update readme
lukaszcl Apr 19, 2024
42df261
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 19, 2024
fd0cbb1
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 19, 2024
17b6c56
Fix
lukaszcl Apr 19, 2024
e238b29
Fixes
lukaszcl Apr 22, 2024
81791eb
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 22, 2024
dde4c49
Remove ls step from workflow
lukaszcl Apr 22, 2024
9cc7af8
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 22, 2024
cd3b8f5
Fix action version and tag
lukaszcl Apr 22, 2024
9001a2c
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 22, 2024
712e4c3
Bump CTF
lukaszcl Apr 23, 2024
3859115
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 23, 2024
fb07194
Bump CTF
lukaszcl Apr 23, 2024
64b4f1c
Merge branch 'develop' into TT-1049-Provide-e2e-coverage-reports-for-…
lukaszcl Apr 24, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Create NodeCoverageHelper and generate total coverage for all tests
  • Loading branch information
lukaszcl authored Apr 18, 2024
commit fe2dfb3834f464d64235ffda4b689006eab3c2fb
68 changes: 57 additions & 11 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ jobs:
## Run this step when changes that require tests to be run are made
- name: Run Tests
if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@be4b0f4b2e3bb5f9e56255f19e5a4809c515f954
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639
with:
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
test_download_vendor_packages_command: cd ./integration-tests && go mod download
Expand All @@ -386,17 +386,20 @@ jobs:
publish_check_name: ${{ matrix.product.name }}
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
go_coverage_src_dir: /var/tmp/go-coverage
go_coverage_dest_dir: /var/tmp/go-coverage
cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
cache_restore_only: "true"
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ""
should_tidy: "false"
go_coverage_src_dir: /var/tmp/go-coverage
go_coverage_dest_dir: .covdata

- name: Show Go Coverage
run: go run ./integration-tests/scripts/show_coverage.go /var/tmp/go-coverage
- name: Upload Coverage Data
uses: actions/upload-artifact@v3
lukaszcl marked this conversation as resolved.
Show resolved Hide resolved
with:
path: .covdata
retention-days: 1

- name: Print failed test summary
if: always()
Expand Down Expand Up @@ -490,6 +493,14 @@ jobs:
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ""
should_tidy: "false"
go_coverage_src_dir: /var/tmp/go-coverage
go_coverage_dest_dir: .covdata

- name: Upload Coverage Data
uses: actions/upload-artifact@v3
with:
path: .covdata
retention-days: 1

eth-smoke-tests-matrix:
if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
Expand Down Expand Up @@ -672,9 +683,9 @@ jobs:
## Run this step when changes that require tests to be run are made
- name: Run Tests
if: needs.changes.outputs.src == 'true' || github.event_name == 'workflow_dispatch'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@be4b0f4b2e3bb5f9e56255f19e5a4809c515f954
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@519851800779323566b7b7c22cc21bff95dbb639
with:
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -onlyerrors=false
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=${{ matrix.product.nodes }} ${{ steps.build-go-test-command.outputs.run_command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage
test_download_vendor_packages_command: cd ./integration-tests && go mod download
cl_repo: ${{ env.CHAINLINK_IMAGE }}
cl_image_tag: ${{ inputs.evm-ref || github.sha }}${{ matrix.product.tag_suffix }}
Expand All @@ -686,17 +697,20 @@ jobs:
publish_check_name: ${{ matrix.product.name }}
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
go_coverage_src_dir: /var/tmp/go-coverage
go_coverage_dest_dir: /var/tmp/go-coverage
cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
cache_restore_only: "true"
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ""
should_tidy: "false"
go_coverage_src_dir: /var/tmp/go-coverage
go_coverage_dest_dir: .covdata

- name: Show Go Coverage
run: go run ./integration-tests/scripts/show_coverage.go /var/tmp/go-coverage
- name: Upload Coverage Data
uses: actions/upload-artifact@v3
with:
path: .covdata
retention-days: 1

# Run this step when changes that do not need the test to run are made
- name: Run Setup
Expand Down Expand Up @@ -790,6 +804,22 @@ jobs:
this-job-name: Clean up integration environment deployments
continue-on-error: true

show-coverage:
name: Show Chainlink Node Go Coverage
if: always()
needs: [cleanup]
runs-on: ubuntu-latest
steps:
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
repository: smartcontractkit/chainlink
ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
- name: Download All Artifacts
uses: actions/download-artifact@v3
lukaszcl marked this conversation as resolved.
Show resolved Hide resolved
- name: Show Coverage
run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/artifact*/*/merged"

# Run the setup if the matrix finishes but this time save the cache if we have a cache hit miss
# this will also only run if both of the matrix jobs pass
eth-smoke-go-mod-cache:
Expand Down Expand Up @@ -880,6 +910,14 @@ jobs:
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
go_coverage_src_dir: /var/tmp/go-coverage
go_coverage_dest_dir: .covdata
- name: Upload Coverage Data
uses: actions/upload-artifact@v3
with:
path: .covdata
retention-days: 1

- name: Collect Metrics
if: always()
id: collect-gha-metrics
Expand Down Expand Up @@ -1180,4 +1218,12 @@ jobs:
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ""
run_setup: false
go_coverage_src_dir: /var/tmp/go-coverage
go_coverage_dest_dir: .covdata

- name: Upload Coverage Data
uses: actions/upload-artifact@v3
with:
path: .covdata
retention-days: 1

1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ output.txt
race.*
golangci-lint-output.txt
/golangci-lint/
.covdata

# DB state
./db/
Expand Down
162 changes: 162 additions & 0 deletions integration-tests/docker/node_coverage_helper.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package docker

import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"

"github.com/pkg/errors"
tc "github.com/testcontainers/testcontainers-go"
)

type NodeCoverageHelper struct {
Nodes []tc.Container
GoCoverSrcDir string // Path to the source directory on the chainlink image with go coverage data
NodeCoverageDirs []string // Paths to individual node coverage directories
CoverageDir string // Path to the base directory with all coverage
MergedDir string // Path to the directory where all coverage will be merged
ChainlinkDir string // Path to the root chainlink directory
}

func NewNodeCoverageHelper(ctx context.Context, nodes []tc.Container, chainlinkDir, coverageDir string) (*NodeCoverageHelper, error) {
coverSrcDir := os.Getenv("GO_COVERAGE_SRC_DIR")
if coverSrcDir == "" {
coverSrcDir = "/var/tmp/go-coverage" // Default path
}

helper := &NodeCoverageHelper{
Nodes: nodes,
GoCoverSrcDir: coverSrcDir,
CoverageDir: coverageDir,
MergedDir: filepath.Join(coverageDir, "merged"),
ChainlinkDir: chainlinkDir,
}

if err := os.MkdirAll(coverageDir, 0755); err != nil {
return nil, errors.Wrap(err, "failed to create base directory for node coverage")
}

// Copy coverage data from nodes
if err := helper.copyCoverageFromNodes(ctx); err != nil {
return nil, errors.Wrap(err, "failed to copy coverage from nodes during initialization")
}

// Merge the coverage data
if err := helper.mergeCoverage(); err != nil {
return nil, errors.Wrap(err, "failed to merge coverage data")
}

return helper, nil
}

func (c *NodeCoverageHelper) SaveMergedHTMLReport() (string, error) {
// Generate the textual coverage report
txtCommand := exec.Command("go", "tool", "covdata", "textfmt", "-i=.", "-o=cov.txt")
txtCommand.Dir = c.MergedDir
if txtOutput, err := txtCommand.CombinedOutput(); err != nil {
return "", errors.Wrapf(err, "failed to generate textual coverage report: %s", string(txtOutput))
}

// Generate the HTML coverage report
htmlFilePath := filepath.Join(c.CoverageDir, "coverage.html")
// #nosec G204
htmlCommand := exec.Command("go", "tool", "cover", "-html="+filepath.Join(c.MergedDir, "cov.txt"), "-o="+htmlFilePath)
htmlCommand.Dir = c.ChainlinkDir
if htmlOutput, err := htmlCommand.CombinedOutput(); err != nil {
return "", errors.Wrapf(err, "failed to generate HTML coverage report: %s", string(htmlOutput))
}

return htmlFilePath, nil
}

func (c *NodeCoverageHelper) SaveMergedCoveragePercentage() (string, error) {
filePath := filepath.Join(c.CoverageDir, "percentage.txt")

// Calculate coverage percentage from the merged data
percentCmd := exec.Command("go", "tool", "covdata", "percent", "-i=.")
percentCmd.Dir = c.MergedDir // Ensure the command runs in the directory with the merged data
output, err := percentCmd.CombinedOutput()
if err != nil {
return "", fmt.Errorf("failed to get merged coverage percentage report: %w, output: %s", err, string(output))
}

// Save the cmd output to a file
if err := os.WriteFile(filePath, output, 0600); err != nil {
return "", errors.Wrap(err, "failed to write coverage percentage to file")
}

return filePath, nil
}

func (c *NodeCoverageHelper) mergeCoverage() error {
if err := os.MkdirAll(c.MergedDir, 0755); err != nil {
return fmt.Errorf("failed to create merged directory: %w", err)
}

// Merge the coverage data from all chainlink nodes
dirInput := strings.Join(c.NodeCoverageDirs, ",")
// #nosec G204
mergeCmd := exec.Command("go", "tool", "covdata", "merge", "-o", c.MergedDir, "-i="+dirInput)
mergeCmd.Dir = filepath.Dir(c.MergedDir)
output, err := mergeCmd.CombinedOutput()
if err != nil {
return fmt.Errorf("error executing merge command: %w, output: %s", err, string(output))
}

// Remove the coverage dirs after merging
for _, dir := range c.NodeCoverageDirs {
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("failed to remove directory %s: %w", dir, err)
}
}
c.NodeCoverageDirs = []string{} // Reset the coverage paths after merging

return nil
}

func (c *NodeCoverageHelper) copyCoverageFromNodes(ctx context.Context) error {
var wg sync.WaitGroup
errorsChan := make(chan error, len(c.Nodes))

for i, node := range c.Nodes {
wg.Add(1)
go func(n tc.Container, id int) {
defer wg.Done()
finalDestPath := filepath.Join(c.CoverageDir, fmt.Sprintf("node_%d", id))
if err := os.MkdirAll(finalDestPath, 0755); err != nil {
errorsChan <- fmt.Errorf("failed to create directory for node %d: %w", id, err)
return
}
err := copyFolderFromContainerUsingDockerCP(ctx, n.GetContainerID(), c.GoCoverSrcDir, finalDestPath)
if err != nil {
errorsChan <- fmt.Errorf("failed to copy folder from container for node %d: %w", id, err)
return
}
finalDestPath = filepath.Join(finalDestPath, "go-coverage") // Assuming path structure /var/tmp/go-coverage/TestName/node_X/go-coverage
c.NodeCoverageDirs = append(c.NodeCoverageDirs, finalDestPath)
}(node, i)
}

wg.Wait()
close(errorsChan)

for err := range errorsChan {
if err != nil {
return err
}
}

return nil
}
func copyFolderFromContainerUsingDockerCP(ctx context.Context, containerID, srcPath, destPath string) error {
source := fmt.Sprintf("%s:%s", containerID, srcPath)
cmd := exec.CommandContext(ctx, "docker", "cp", source, destPath)
if output, err := cmd.CombinedOutput(); err != nil {
return errors.Wrapf(err, "docker cp command failed: %s, output: %s", cmd, string(output))
}
return nil
}
Loading
Loading