[TT-1198] Fix missing logs (#13300)

* dont fail with Failf(), use Errorf() instead; break loop when first concerning log is found * print test summary in automation nightly tests * remove unneeded file * do not add log stream clean up only when cleanup is set to none * fix VRFv2 smoke test * Fix log artifacts in github * TT-1198:fix TestVRFv2BatchFulfillmentEnabledDisabled test --------- Co-authored-by: lukaszcl <[email protected]> Co-authored-by: Ilja Pavlovs <[email protected]>
smartcontractkit · May 23, 2024 · 203a95e · 203a95e
1 parent d90cd65
commit 203a95e
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 14 deletions.
diff --git a/.github/workflows/automation-nightly-tests.yml b/.github/workflows/automation-nightly-tests.yml
@@ -129,6 +129,7 @@ jobs:
           cl_image_tag: 'latest'
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs
+          artifacts_name: testcontainers-logs-${{ matrix.tests.name }}
           publish_check_name: Automation Results ${{ matrix.tests.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
@@ -139,7 +140,7 @@ jobs:
         uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()
         with:
-          name: test-log-${{ matrix.tests.name }}
+          name: gotest-logs-${{ matrix.tests.name }}
           path: /tmp/gotest.log
           retention-days: 7
         continue-on-error: true
@@ -155,6 +156,9 @@ jobs:
           this-job-name: Automation ${{ matrix.tests.name }} Test
           test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
         continue-on-error: true
+      - name: Print failed test summary
+        if: always()
+        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17
 
   test-notify:
     name: Start Slack Thread

diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go
@@ -279,7 +279,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 		}
 
 		// this clean up has to be added as the FIRST one, because cleanup functions are executed in reverse order (LIFO)
-		if b.t != nil && b.cleanUpType == CleanUpTypeStandard {
+		if b.t != nil && b.cleanUpType != CleanUpTypeNone {
 			b.t.Cleanup(func() {
 				b.l.Info().Msg("Shutting down LogStream")
 				logPath, err := osutil.GetAbsoluteFolderPath("logs")
@@ -306,31 +306,35 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 
 					// we cannot do parallel processing here, because ProcessContainerLogs() locks a mutex that controls whether
 					// new logs can be added to the log stream, so parallel processing would get stuck on waiting for it to be unlocked
+				LogScanningLoop:
 					for i := 0; i < b.clNodesCount; i++ {
 						// ignore count return, because we are only interested in the error
 						_, err := logProcessor.ProcessContainerLogs(b.te.ClCluster.Nodes[i].ContainerName, processFn)
 						if err != nil && !strings.Contains(err.Error(), testreporters.MultipleLogsAtLogLevelErr) && !strings.Contains(err.Error(), testreporters.OneLogAtLogLevelErr) {
-							b.l.Error().Err(err).Msg("Error processing logs")
-							return
+							b.l.Error().Err(err).Msg("Error processing CL node logs")
+							continue
 						} else if err != nil && (strings.Contains(err.Error(), testreporters.MultipleLogsAtLogLevelErr) || strings.Contains(err.Error(), testreporters.OneLogAtLogLevelErr)) {
 							flushLogStream = true
-							b.t.Fatalf("Found a concerning log in Chainklink Node logs: %v", err)
+							b.t.Errorf("Found a concerning log in Chainklink Node logs: %v", err)
+							break LogScanningLoop
 						}
 					}
 					b.l.Info().Msg("Finished scanning Chainlink Node logs for concerning errors")
 				}
 
 				if flushLogStream {
+					b.l.Info().Msg("Flushing LogStream logs")
 					// we can't do much if this fails, so we just log the error in LogStream
 					if err := b.te.LogStream.FlushAndShutdown(); err != nil {
 						b.l.Error().Err(err).Msg("Error flushing and shutting down LogStream")
 					}
 					b.te.LogStream.PrintLogTargetsLocations()
 					b.te.LogStream.SaveLogLocationInTestSummary()
 				}
+				b.l.Info().Msg("Finished shutting down LogStream")
 			})
 		} else {
-			b.l.Warn().Msg("LogStream won't be cleaned up, because test instance is not set or cleanup type is not standard")
+			b.l.Warn().Msg("LogStream won't be cleaned up, because either test instance is not set or cleanup type is set to none")
 		}
 	}
 

diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go
@@ -861,10 +861,11 @@ func TestVRFV2WithBHS(t *testing.T) {
 		CleanupFn:  cleanupFn,
 	}
 	newEnvConfig := vrfcommon.NewEnvConfig{
-		NodesToCreate:          []vrfcommon.VRFNodeType{vrfcommon.VRF, vrfcommon.BHS},
-		NumberOfTxKeysToCreate: 0,
-		UseVRFOwner:            false,
-		UseTestCoordinator:     false,
+		NodesToCreate:                   []vrfcommon.VRFNodeType{vrfcommon.VRF, vrfcommon.BHS},
+		NumberOfTxKeysToCreate:          0,
+		UseVRFOwner:                     false,
+		UseTestCoordinator:              false,
+		ChainlinkNodeLogScannerSettings: test_env.DefaultChainlinkNodeLogScannerSettings,
 	}
 	testEnv, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, vrfEnvConfig, newEnvConfig, l)
 	require.NoError(t, err, "Error setting up VRFV2 universe")
@@ -1266,10 +1267,11 @@ func TestVRFv2BatchFulfillmentEnabledDisabled(t *testing.T) {
 		CleanupFn:  cleanupFn,
 	}
 	newEnvConfig := vrfcommon.NewEnvConfig{
-		NodesToCreate:          []vrfcommon.VRFNodeType{vrfcommon.VRF},
-		NumberOfTxKeysToCreate: 0,
-		UseVRFOwner:            false,
-		UseTestCoordinator:     false,
+		NodesToCreate:                   []vrfcommon.VRFNodeType{vrfcommon.VRF},
+		NumberOfTxKeysToCreate:          0,
+		UseVRFOwner:                     false,
+		UseTestCoordinator:              false,
+		ChainlinkNodeLogScannerSettings: test_env.DefaultChainlinkNodeLogScannerSettings,
 	}
 	env, vrfContracts, vrfKey, nodeTypeToNodeMap, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, vrfEnvConfig, newEnvConfig, l)
 	require.NoError(t, err, "Error setting up VRFv2 universe")