From 262d492ae06f18195ba7694abc85958ba7265cc9 Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Fri, 26 Apr 2024 20:35:33 -0700 Subject: [PATCH 1/4] Add exit logs for go_core_tests --- tools/bin/go_core_tests | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests index c0b0f43aed4..0a93022099d 100755 --- a/tools/bin/go_core_tests +++ b/tools/bin/go_core_tests @@ -33,4 +33,5 @@ if [[ $EXITCODE != 0 ]]; then else echo "All tests passed!" fi +echo "go_core_tests.sh exiting with code $EXITCODE" exit $EXITCODE From c70a6e42ca71591158e2afa929c7d6d7a766d7d9 Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Fri, 26 Apr 2024 20:38:02 -0700 Subject: [PATCH 2/4] Print each iteration of a flakey test execution --- tools/flakeytests/runner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/flakeytests/runner.go b/tools/flakeytests/runner.go index 88ab647e7c1..f5a779f8b08 100644 --- a/tools/flakeytests/runner.go +++ b/tools/flakeytests/runner.go @@ -219,8 +219,8 @@ func (r *Runner) runTests(rep *Report) (*Report, error) { ts = append(ts, test) } - log.Printf("[FLAKEY_TEST] Executing test command with parameters: pkg=%s, tests=%+v, numReruns=%d\n", pkg, ts, r.numReruns) for i := 0; i < r.numReruns; i++ { + log.Printf("[FLAKEY_TEST] Executing test command with parameters: pkg=%s, tests=%+v, numReruns=%d currentRun=%d\n", pkg, ts, r.numReruns, i) pr, err := r.runTest(pkg, ts) if err != nil { return report, err @@ -237,8 +237,8 @@ func (r *Runner) runTests(rep *Report) (*Report, error) { } for pkg := range rep.packagePanics { - log.Printf("[PACKAGE_PANIC]: Executing test command with parameters: pkg=%s, numReruns=%d\n", pkg, r.numReruns) for i := 0; i < r.numReruns; i++ { + log.Printf("[PACKAGE_PANIC]: Executing test command with parameters: pkg=%s, numReruns=%d currentRun=%d\n", pkg, r.numReruns, i) pr, err := r.runTest(pkg, []string{}) if err != nil { return report, err From fafd6e92be202ff0f0fc2ff88a8c011d51a92b57 Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Fri, 26 Apr 2024 20:56:01 -0700 Subject: [PATCH 3/4] DRY up env var usage to prevent typos --- tools/flakeytests/runner.go | 2 +- tools/flakeytests/runner_test.go | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/tools/flakeytests/runner.go b/tools/flakeytests/runner.go index f5a779f8b08..a37b123d5cf 100644 --- a/tools/flakeytests/runner.go +++ b/tools/flakeytests/runner.go @@ -238,7 +238,7 @@ func (r *Runner) runTests(rep *Report) (*Report, error) { for pkg := range rep.packagePanics { for i := 0; i < r.numReruns; i++ { - log.Printf("[PACKAGE_PANIC]: Executing test command with parameters: pkg=%s, numReruns=%d currentRun=%d\n", pkg, r.numReruns, i) + log.Printf("[PACKAGE_PANIC]: Executing test command with parameters: pkg=%s, numReruns=%d currentRun=%d\n", pkg, r.numReruns, i) pr, err := r.runTest(pkg, []string{}) if err != nil { return report, err diff --git a/tools/flakeytests/runner_test.go b/tools/flakeytests/runner_test.go index ee069a1655d..8f9433a427b 100644 --- a/tools/flakeytests/runner_test.go +++ b/tools/flakeytests/runner_test.go @@ -14,6 +14,17 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" ) +// This test suite has two sets of tests that are run in two different modes: +// 1. Normal mode: The tests are run as usual. All tests that start with TestSkippedForTests will be skipped. +// There are a set of integration tests that will run this same test suite in integration mode. +// 2. Integration mode: The tests are run with an environment variable set to run the tests that were skipped in normal mode. +// Their output is captured and parsed to check if the flake runner is behaving as expected. +// +// In summary, the first set of tests are executed by the user, and these tests will trigger the second set of tests to run. +var runInIntegrationTestModeEnvKey = "FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST" +var skipIntegrationTestMode = os.Getenv(runInIntegrationTestModeEnvKey) != "1" +var runInIntegrationTestModeEnv = runInIntegrationTestModeEnvKey + "=1" + type mockReporter struct { report *Report } @@ -307,7 +318,7 @@ func TestRunner_RerunWithNonZeroExitCodeDoesntStopCommand(t *testing.T) { // Used for integration tests func TestSkippedForTests_Subtests(t *testing.T) { - if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" { + if skipIntegrationTestMode { t.Skip() } @@ -322,7 +333,7 @@ func TestSkippedForTests_Subtests(t *testing.T) { // Used for integration tests func TestSkippedForTests(t *testing.T) { - if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" { + if skipIntegrationTestMode { t.Skip() } @@ -333,7 +344,7 @@ func TestSkippedForTests(t *testing.T) { // Used for integration tests func TestSkippedForTests_Success(t *testing.T) { - if os.Getenv("FLAKEY_TEST_RUNNER_RUN_FIXTURE_TEST") != "1" { + if skipIntegrationTestMode { t.Skip() } @@ -356,7 +367,7 @@ func TestIntegration_DealsWithSubtests(t *testing.T) { repo: "github.com/smartcontractkit/chainlink/v2/tools/flakeytests", command: "../bin/go_core_tests", overrides: func(cmd *exec.Cmd) { - cmd.Env = append(cmd.Env, "FLAKEY_TESTRUNNER_RUN_FIXTURE_TEST=1") + cmd.Env = append(cmd.Env, runInIntegrationTestModeEnv) cmd.Stdout = io.Discard cmd.Stderr = io.Discard }, @@ -392,7 +403,7 @@ func TestIntegration_ParsesPanics(t *testing.T) { repo: "github.com/smartcontractkit/chainlink/v2/tools/flakeytests", command: "../bin/go_core_tests", overrides: func(cmd *exec.Cmd) { - cmd.Env = append(cmd.Env, "FLAKEY_TESTRUNNER_RUN_FIXTURE_TEST=1") + cmd.Env = append(cmd.Env, runInIntegrationTestModeEnv) cmd.Stdout = io.Discard cmd.Stderr = io.Discard }, @@ -423,7 +434,7 @@ func TestIntegration(t *testing.T) { repo: "github.com/smartcontractkit/chainlink/v2/tools/flakeytests", command: "../bin/go_core_tests", overrides: func(cmd *exec.Cmd) { - cmd.Env = append(cmd.Env, "FLAKEY_TESTRUNNER_RUN_FIXTURE_TEST=1") + cmd.Env = append(cmd.Env, runInIntegrationTestModeEnv) cmd.Stdout = io.Discard cmd.Stderr = io.Discard }, From 22253c45bb4885e7feeef76872ff6b829eae237a Mon Sep 17 00:00:00 2001 From: HenryNguyen5 <6404866+HenryNguyen5@users.noreply.github.com> Date: Mon, 29 Apr 2024 19:47:34 -0400 Subject: [PATCH 4/4] Remove .sh prefix on go_core_tests log Co-authored-by: Jordan Krage --- tools/bin/go_core_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/go_core_tests b/tools/bin/go_core_tests index 0a93022099d..074527698b3 100755 --- a/tools/bin/go_core_tests +++ b/tools/bin/go_core_tests @@ -33,5 +33,5 @@ if [[ $EXITCODE != 0 ]]; then else echo "All tests passed!" fi -echo "go_core_tests.sh exiting with code $EXITCODE" +echo "go_core_tests exiting with code $EXITCODE" exit $EXITCODE