From e6b1813b2f7040e702b7c02d6089f287778ea5bc Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 22 Jan 2024 12:38:59 -0500 Subject: [PATCH 1/6] TT-756 --- .../remote-runner/remote_runner_envs_test.go | 17 +++++++++++++++++ k8s/environment/environment.go | 7 +++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/k8s/e2e/remote-runner/remote_runner_envs_test.go b/k8s/e2e/remote-runner/remote_runner_envs_test.go index e90b41bfe..f5badd468 100644 --- a/k8s/e2e/remote-runner/remote_runner_envs_test.go +++ b/k8s/e2e/remote-runner/remote_runner_envs_test.go @@ -81,6 +81,23 @@ func TestFundReturnShutdownLogic(t *testing.T) { fmt.Println(environment.FAILED_FUND_RETURN) } +func TestFailedTestLogic(t *testing.T) { + t.Parallel() + testEnvConfig := common.GetTestEnvConfig(t) + e, err := presets.EVMOneNode(testEnvConfig) + require.NoError(t, err, "Error building initial environment") + err = e.Run() + if e.WillUseRemoteRunner() { + require.True(t, e.Cfg.Test.Failed(), "Expecting test status to be failed") + return + } + t.Cleanup(func() { + assert.NoError(t, e.Shutdown()) + }) + require.NoError(t, err) + fmt.Println(environment.TEST_FAILED) +} + func TestRemoteRunnerOneSetupWithMultipeTests(t *testing.T) { t.Parallel() testEnvConfig := common.GetTestEnvConfig(t) diff --git a/k8s/environment/environment.go b/k8s/environment/environment.go index 4e5d36a27..21b56c4c4 100644 --- a/k8s/environment/environment.go +++ b/k8s/environment/environment.go @@ -32,6 +32,7 @@ import ( const ( COVERAGE_DIR string = "cover" FAILED_FUND_RETURN string = "FAILED_FUND_RETURN" + TEST_FAILED string = "TEST_FAILED" ) const ( @@ -1021,10 +1022,12 @@ func DefaultJobLogFunction(e *Environment, message string) { for _, chunk := range logChunks { e.Cfg.Test.Log(chunk) } - found := strings.Contains(message, FAILED_FUND_RETURN) - if found { + if strings.Contains(message, FAILED_FUND_RETURN) { e.Cfg.fundReturnFailed = true } + if strings.Contains(message, TEST_FAILED) { + e.Cfg.Test.Fail() + } } // markNotSafeToEvict adds the safe to evict annotation to the provided map if needed From bb935435599b1db7fe8dbcae7b7f64e0368417b0 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 22 Jan 2024 13:02:31 -0500 Subject: [PATCH 2/6] Easier test debug --- Makefile | 4 +++- k8s/e2e/remote-runner/remote_runner_envs_test.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 776365ac0..aef903392 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,9 @@ k8s_test_e2e_ci: go test ./k8s/e2e/local-runner -count 1 -v -test.parallel=14 -test.timeout=1h -json 2>&1 | tee /tmp/gotest.log | gotestfmt k8s_test_e2e_ci_remote_runner: - go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=16 -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt + # go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=20 -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt + ## DEBUG: Remove before merge + go test ./k8s/e2e/remote-runner -count 1 -v -run TestFailedTestLogic -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt .PHONY: examples examples: diff --git a/k8s/e2e/remote-runner/remote_runner_envs_test.go b/k8s/e2e/remote-runner/remote_runner_envs_test.go index f5badd468..ad305fd51 100644 --- a/k8s/e2e/remote-runner/remote_runner_envs_test.go +++ b/k8s/e2e/remote-runner/remote_runner_envs_test.go @@ -88,7 +88,7 @@ func TestFailedTestLogic(t *testing.T) { require.NoError(t, err, "Error building initial environment") err = e.Run() if e.WillUseRemoteRunner() { - require.True(t, e.Cfg.Test.Failed(), "Expecting test status to be failed") + fmt.Println("Test Failed?", e.Cfg.Test.Failed()) return } t.Cleanup(func() { From 6de84081f192a0f388568d0ff3c957ca20680638 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 22 Jan 2024 13:06:17 -0500 Subject: [PATCH 3/6] Prints and only use remote runner --- k8s/e2e/remote-runner/remote_runner_envs_test.go | 7 ++++--- k8s/presets/presets.go | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/k8s/e2e/remote-runner/remote_runner_envs_test.go b/k8s/e2e/remote-runner/remote_runner_envs_test.go index ad305fd51..baf24012c 100644 --- a/k8s/e2e/remote-runner/remote_runner_envs_test.go +++ b/k8s/e2e/remote-runner/remote_runner_envs_test.go @@ -84,10 +84,10 @@ func TestFundReturnShutdownLogic(t *testing.T) { func TestFailedTestLogic(t *testing.T) { t.Parallel() testEnvConfig := common.GetTestEnvConfig(t) - e, err := presets.EVMOneNode(testEnvConfig) - require.NoError(t, err, "Error building initial environment") - err = e.Run() + e := presets.OnlyRemoteRunner(testEnvConfig) + err := e.Run() if e.WillUseRemoteRunner() { + fmt.Println("Inside K8s?", e.Cfg.InsideK8s) fmt.Println("Test Failed?", e.Cfg.Test.Failed()) return } @@ -95,6 +95,7 @@ func TestFailedTestLogic(t *testing.T) { assert.NoError(t, e.Shutdown()) }) require.NoError(t, err) + fmt.Println("Inside K8s?", e.Cfg.InsideK8s) fmt.Println(environment.TEST_FAILED) } diff --git a/k8s/presets/presets.go b/k8s/presets/presets.go index dc49d7a6c..3c5ca6d9b 100644 --- a/k8s/presets/presets.go +++ b/k8s/presets/presets.go @@ -35,6 +35,10 @@ Enabled = true [P2P.V2] ListenAddresses = ["0.0.0.0:6690"]` +func OnlyRemoteRunner(config *environment.Config) *environment.Environment { + return environment.New(config) +} + // EVMOneNode local development Chainlink deployment func EVMOneNode(config *environment.Config) (*environment.Environment, error) { From 88bdf4a1e5633e5f525ddbe2f23c8098194b3573 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 22 Jan 2024 13:21:55 -0500 Subject: [PATCH 4/6] Try a skip? --- k8s/e2e/remote-runner/remote_runner_envs_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/e2e/remote-runner/remote_runner_envs_test.go b/k8s/e2e/remote-runner/remote_runner_envs_test.go index baf24012c..704c502df 100644 --- a/k8s/e2e/remote-runner/remote_runner_envs_test.go +++ b/k8s/e2e/remote-runner/remote_runner_envs_test.go @@ -89,6 +89,8 @@ func TestFailedTestLogic(t *testing.T) { if e.WillUseRemoteRunner() { fmt.Println("Inside K8s?", e.Cfg.InsideK8s) fmt.Println("Test Failed?", e.Cfg.Test.Failed()) + require.True(t, e.Cfg.Test.Failed(), "Test should have failed") + e.Cfg.Test.SkipNow() return } t.Cleanup(func() { From 4becd191e8b57873969bf675d3fcdc1e175ebf66 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 22 Jan 2024 13:39:51 -0500 Subject: [PATCH 5/6] Skips intentionally failing test --- k8s/e2e/remote-runner/remote_runner_envs_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/e2e/remote-runner/remote_runner_envs_test.go b/k8s/e2e/remote-runner/remote_runner_envs_test.go index 704c502df..1f315c49d 100644 --- a/k8s/e2e/remote-runner/remote_runner_envs_test.go +++ b/k8s/e2e/remote-runner/remote_runner_envs_test.go @@ -82,6 +82,7 @@ func TestFundReturnShutdownLogic(t *testing.T) { } func TestFailedTestLogic(t *testing.T) { + t.Skip("This test is meant to fail, and can only be evaluated by looking at the logs. Only turn on if checking this specific logic.") t.Parallel() testEnvConfig := common.GetTestEnvConfig(t) e := presets.OnlyRemoteRunner(testEnvConfig) @@ -90,7 +91,7 @@ func TestFailedTestLogic(t *testing.T) { fmt.Println("Inside K8s?", e.Cfg.InsideK8s) fmt.Println("Test Failed?", e.Cfg.Test.Failed()) require.True(t, e.Cfg.Test.Failed(), "Test should have failed") - e.Cfg.Test.SkipNow() + fmt.Println("This is a test-of-a-test and is confusing. The test that this tests should fail. But that also means this tests fails. If you're reading this, the test has actually passed.") return } t.Cleanup(func() { From 459effc2873ffa599de847596fea8d4a60cf76ac Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 22 Jan 2024 13:47:50 -0500 Subject: [PATCH 6/6] Remove debug --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index aef903392..00d8b0ff5 100644 --- a/Makefile +++ b/Makefile @@ -128,9 +128,7 @@ k8s_test_e2e_ci: go test ./k8s/e2e/local-runner -count 1 -v -test.parallel=14 -test.timeout=1h -json 2>&1 | tee /tmp/gotest.log | gotestfmt k8s_test_e2e_ci_remote_runner: - # go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=20 -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt - ## DEBUG: Remove before merge - go test ./k8s/e2e/remote-runner -count 1 -v -run TestFailedTestLogic -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt + go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=20 -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt .PHONY: examples examples: