Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry pick flaky test failure from develop to release/2.11.0 #13056

Closed
wants to merge 1 commit into from
Closed

Cherry pick flaky test failure from develop to release/2.11.0 #13056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tools/bin/go_core_tests
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,5 @@ else
# uploading coverage.txt to CodeCov
$(dirname "$0")/codecov -f coverage.txt
fi
echo "go_core_tests exiting with code $EXITCODE"
exit $EXITCODE
4 changes: 2 additions & 2 deletions tools/flakeytests/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,8 @@ func (r *Runner) runTests(rep *Report) (*Report, error) {
ts = append(ts, test)
}

log.Printf("[FLAKEY_TEST] Executing test command with parameters: pkg=%s, tests=%+v, numReruns=%d\n", pkg, ts, r.numReruns)
for i := 0; i < r.numReruns; i++ {
log.Printf("[FLAKEY_TEST] Executing test command with parameters: pkg=%s, tests=%+v, numReruns=%d currentRun=%d\n", pkg, ts, r.numReruns, i)
pr, err := r.runTest(pkg, ts)
if err != nil {
return report, err
Expand All @@ -237,8 +237,8 @@ func (r *Runner) runTests(rep *Report) (*Report, error) {
}

for pkg := range rep.packagePanics {
log.Printf("[PACKAGE_PANIC]: Executing test command with parameters: pkg=%s, numReruns=%d\n", pkg, r.numReruns)
for i := 0; i < r.numReruns; i++ {
log.Printf("[PACKAGE_PANIC]: Executing test command with parameters: pkg=%s, numReruns=%d currentRun=%d\n", pkg, r.numReruns, i)
pr, err := r.runTest(pkg, []string{})
if err != nil {
return report, err
Expand Down
23 changes: 17 additions & 6 deletions tools/flakeytests/runner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,17 @@ import (
"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
)

// This test suite has two sets of tests that are run in two different modes:
// 1. Normal mode: The tests are run as usual. All tests that start with TestSkippedForTests will be skipped.
// There are a set of integration tests that will run this same test suite in integration mode.
// 2. Integration mode: The tests are run with an environment variable set to run the tests that were skipped in normal mode.
// Their output is captured and parsed to check if the flake runner is behaving as expected.
//
// In summary, the first set of tests are executed by the user, and these tests will trigger the second set of tests to run.
var runInIntegrationTestModeEnvKey = "FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST"
var skipIntegrationTestMode = os.Getenv(runInIntegrationTestModeEnvKey) != "1"
var runInIntegrationTestModeEnv = runInIntegrationTestModeEnvKey + "=1"

type mockReporter struct {
report *Report
}
Expand Down Expand Up @@ -307,7 +318,7 @@ func TestRunner_RerunWithNonZeroExitCodeDoesntStopCommand(t *testing.T) {

// Used for integration tests
func TestSkippedForTests_Subtests(t *testing.T) {
if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" {
if skipIntegrationTestMode {
t.Skip()
}

Expand All @@ -322,7 +333,7 @@ func TestSkippedForTests_Subtests(t *testing.T) {

// Used for integration tests
func TestSkippedForTests(t *testing.T) {
if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" {
if skipIntegrationTestMode {
t.Skip()
}

Expand All @@ -333,7 +344,7 @@ func TestSkippedForTests(t *testing.T) {

// Used for integration tests
func TestSkippedForTests_Success(t *testing.T) {
if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" {
if skipIntegrationTestMode {
t.Skip()
}

Expand All @@ -356,7 +367,7 @@ func TestIntegration_DealsWithSubtests(t *testing.T) {
repo: "github.com/smartcontractkit/chainlink/v2/tools/flakeytests",
command: "../bin/go_core_tests",
overrides: func(cmd *exec.Cmd) {
cmd.Env = append(cmd.Env, "FLAKEY_TESTRUNNER_RUN_FIXTURE_TEST=1")
cmd.Env = append(cmd.Env, runInIntegrationTestModeEnv)
cmd.Stdout = io.Discard
cmd.Stderr = io.Discard
},
Expand Down Expand Up @@ -392,7 +403,7 @@ func TestIntegration_ParsesPanics(t *testing.T) {
repo: "github.com/smartcontractkit/chainlink/v2/tools/flakeytests",
command: "../bin/go_core_tests",
overrides: func(cmd *exec.Cmd) {
cmd.Env = append(cmd.Env, "FLAKEY_TESTRUNNER_RUN_FIXTURE_TEST=1")
cmd.Env = append(cmd.Env, runInIntegrationTestModeEnv)
cmd.Stdout = io.Discard
cmd.Stderr = io.Discard
},
Expand Down Expand Up @@ -423,7 +434,7 @@ func TestIntegration(t *testing.T) {
repo: "github.com/smartcontractkit/chainlink/v2/tools/flakeytests",
command: "../bin/go_core_tests",
overrides: func(cmd *exec.Cmd) {
cmd.Env = append(cmd.Env, "FLAKEY_TESTRUNNER_RUN_FIXTURE_TEST=1")
cmd.Env = append(cmd.Env, runInIntegrationTestModeEnv)
cmd.Stdout = io.Discard
cmd.Stderr = io.Discard
},
Expand Down
Loading