Triage really long logs issues

.github/workflows/k8s-e2e.yaml

@@ -1,6 +1,7 @@
 name: (k8s package) E2E tests
 on:
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: e2e-tests-k8s-${{ github.ref }}
@@ -88,6 +89,7 @@ jobs:
           path: /tmp/gotest.log
 
   e2e_remote_runner_tests:
+    timeout-minutes: 12
     runs-on: ubuntu-latest
     environment: integration
     needs: [build_tests]
@@ -115,7 +117,7 @@ jobs:
           QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
       - name: Upload test log
         uses: actions/upload-artifact@v2
-        if: failure()
+        # if: failure()
         with:
           name: remote-runner-test-log
           path: /tmp/remoterunnergotest.log

Makefile

@@ -125,10 +125,10 @@ k8s_test_e2e:
 	go test ./k8s/e2e/local-runner -count 1 -test.parallel=12 -v $(args)
 
 k8s_test_e2e_ci:
-	go test ./k8s/e2e/local-runner -count 1 -v -test.parallel=14 -test.timeout=1h -json 2>&1 | tee /tmp/gotest.log | gotestfmt
+	go test ./k8s/e2e/local-runner -count 1 -v -test.parallel=14 -test.timeout=1h -json 2>&1 | tee /tmp/gotest.log
 
 k8s_test_e2e_ci_remote_runner:
-	go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=16 -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt
+	go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=16 -test.timeout=7m 2>&1 | tee /tmp/remoterunnergotest.log
 
 .PHONY: examples
 examples:

k8s/client/cmd.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"io"
 	"os/exec"
@@ -20,14 +21,40 @@ func ExecCmdWithContext(ctx context.Context, command string) error {
 	})
 }
 
-// readStdPipe continuously read a pipe from the command
+// readStdPipe continuously reads from a given pipe (either stdout or stderr)
+// and processes the output line by line using the provided outputFunction.
+// It handles lines of any length dynamically without the need for a large predefined buffer.
 func readStdPipe(pipe io.ReadCloser, outputFunction func(string)) {
-	scanner := bufio.NewScanner(pipe)
-	scanner.Split(bufio.ScanLines)
-	for scanner.Scan() {
-		m := scanner.Text()
-		if outputFunction != nil {
-			outputFunction(m)
+	reader := bufio.NewReader(pipe)
+	var output []rune
+
+	for {
+		// ReadLine tries to return a single line, not including the end-of-line bytes.
+		// The returned line may be incomplete if the line's too long for the buffer.
+		// isPrefix will be true if the line is longer than the buffer.
+		chunk, isPrefix, err := reader.ReadLine()
+
+		// Handle any errors that occurred during the read.
+		if err != nil {
+			// Log any error that's not an EOF (end of file).
+			if err != io.EOF {
+				log.Error().Err(err).Msg("Error while reading standard pipe")
+			}
+			break
+		}
+
+		// Append the chunk to the output buffer.
+		// bytes.Runes converts the byte slice to a slice of runes, handling multi-byte characters.
+		output = append(output, bytes.Runes(chunk)...)
+
+		// If isPrefix is false, we've reached the end of the line and can process it.
+		if !isPrefix {
+			// Call the output function with the complete line if it's defined.
+			if outputFunction != nil {
+				outputFunction(string(output))
+			}
+			// Reset output to an empty slice for reading the next line.
+			output = output[:0]
 		}
 	}
 }

k8s/client/cmd_test.go

@@ -0,0 +1,45 @@
+package client
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadStdPipeWithLongString(t *testing.T) {
+	// Create a string with a million characters ('a').
+	longString := strings.Repeat("a", 1000000)
+
+	// Use an io.Pipe to simulate the stdout or stderr pipe.
+	reader, writer := io.Pipe()
+
+	// Channel to communicate errors from the writing goroutine.
+	errChan := make(chan error, 1)
+
+	// Write the long string to the pipe in a goroutine.
+	go func() {
+		_, err := writer.Write([]byte(longString))
+		if err != nil {
+			// Send any errors to the main test goroutine via the channel.
+			errChan <- err
+		}
+		writer.Close()
+		errChan <- nil // Send nil to indicate successful write.
+	}()
+
+	// Variable to store the output from the readStdPipe function.
+	var output string
+	outputFunction := func(s string) {
+		output = s
+	}
+
+	// Call the readStdPipe function with the reader part of the pipe.
+	readStdPipe(reader, outputFunction)
+
+	// Check for errors from the write goroutine.
+	err := <-errChan
+	require.NoError(t, err, "Failed to write to pipe")
+	require.Equal(t, longString, output, "Output did not match the input long string")
+}

k8s/e2e/common/test_common.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -446,3 +447,18 @@ func TestRunTimeout(t *testing.T) {
 	err = e.Run()
 	require.Error(t, err)
 }
+
+func TestReallyLongLogs(t *testing.T) {
+	t.Parallel()
+	l := logging.GetTestLogger(t)
+	testEnvConfig := GetTestEnvConfig(t)
+	val, _ := os.LookupEnv(config.EnvVarJobImage)
+	if val != "" {
+		env := environment.New(testEnvConfig)
+		err := env.Run()
+		require.NoError(t, err)
+	}
+	s := strings.Repeat("a", 500000)
+	// this shouldn't hang
+	l.Info().Int("len", len(s)).Str("string", s).Msg("string")
+}

k8s/e2e/local-runner/envs_test.go

@@ -3,71 +3,74 @@ package env_test
 import (
 	"testing"
 
-	"github.com/smartcontractkit/chainlink-testing-framework/k8s/config"
 	"github.com/smartcontractkit/chainlink-testing-framework/k8s/e2e/common"
 )
 
-func TestMultiStageMultiManifestConnection(t *testing.T) {
-	common.TestMultiStageMultiManifestConnection(t)
-}
+// func TestMultiStageMultiManifestConnection(t *testing.T) {
+// 	common.TestMultiStageMultiManifestConnection(t)
+// }
 
-func TestConnectWithoutManifest(t *testing.T) {
-	common.TestConnectWithoutManifest(t)
-}
+// func TestConnectWithoutManifest(t *testing.T) {
+// 	common.TestConnectWithoutManifest(t)
+// }
 
-func Test5NodesSoakEnvironmentWithPVCs(t *testing.T) {
-	common.Test5NodesSoakEnvironmentWithPVCs(t)
-}
+// func Test5NodesSoakEnvironmentWithPVCs(t *testing.T) {
+// 	common.Test5NodesSoakEnvironmentWithPVCs(t)
+// }
 
-func TestWithSingleNodeEnv(t *testing.T) {
-	common.TestWithSingleNodeEnvParallel(t)
-}
+// func TestWithSingleNodeEnv(t *testing.T) {
+// 	common.TestWithSingleNodeEnvParallel(t)
+// }
 
-func TestWithSingleNodeEnvLocalCharts(t *testing.T) {
-	t.Setenv(config.EnvVarLocalCharts, "true")
-	common.TestWithSingleNodeEnv(t)
-}
+// func TestWithSingleNodeEnvLocalCharts(t *testing.T) {
+// 	t.Setenv(config.EnvVarLocalCharts, "true")
+// 	common.TestWithSingleNodeEnv(t)
+// }
 
-func TestMultipleNodeWithDiffDBVersionEnv(t *testing.T) {
-	common.TestMultipleNodeWithDiffDBVersionEnv(t)
-}
+// func TestMultipleNodeWithDiffDBVersionEnv(t *testing.T) {
+// 	common.TestMultipleNodeWithDiffDBVersionEnv(t)
+// }
 
-func TestMinResources5NodesEnv(t *testing.T) {
-	common.TestMinResources5NodesEnv(t)
-}
+// func TestMinResources5NodesEnv(t *testing.T) {
+// 	common.TestMinResources5NodesEnv(t)
+// }
 
-func TestMinResources5NodesEnvWithBlockscout(t *testing.T) {
-	common.TestMinResources5NodesEnvWithBlockscout(t)
-}
+// func TestMinResources5NodesEnvWithBlockscout(t *testing.T) {
+// 	common.TestMinResources5NodesEnvWithBlockscout(t)
+// }
 
-func TestMultipleInstancesOfTheSameType(t *testing.T) {
-	common.TestMultipleInstancesOfTheSameType(t)
-}
+// func TestMultipleInstancesOfTheSameType(t *testing.T) {
+// 	common.TestMultipleInstancesOfTheSameType(t)
+// }
 
-func Test5NodesPlus2MiningGethsReorgEnv(t *testing.T) {
-	common.Test5NodesPlus2MiningGethsReorgEnv(t)
-}
+// func Test5NodesPlus2MiningGethsReorgEnv(t *testing.T) {
+// 	common.Test5NodesPlus2MiningGethsReorgEnv(t)
+// }
 
-func TestWithChaos(t *testing.T) {
-	common.TestWithChaos(t)
-}
+// func TestWithChaos(t *testing.T) {
+// 	common.TestWithChaos(t)
+// }
 
-func TestEmptyEnvironmentStartup(t *testing.T) {
-	common.TestEmptyEnvironmentStartup(t)
-}
+// func TestEmptyEnvironmentStartup(t *testing.T) {
+// 	common.TestEmptyEnvironmentStartup(t)
+// }
 
-func TestRolloutRestartUpdate(t *testing.T) {
-	common.TestRolloutRestart(t, true)
-}
+// func TestRolloutRestartUpdate(t *testing.T) {
+// 	common.TestRolloutRestart(t, true)
+// }
 
-func TestRolloutRestartBySelector(t *testing.T) {
-	common.TestRolloutRestart(t, false)
-}
+// func TestRolloutRestartBySelector(t *testing.T) {
+// 	common.TestRolloutRestart(t, false)
+// }
 
-func TestReplaceHelm(t *testing.T) {
-	common.TestReplaceHelm(t)
-}
+// func TestReplaceHelm(t *testing.T) {
+// 	common.TestReplaceHelm(t)
+// }
+
+// func TestRunTimeout(t *testing.T) {
+// 	common.TestRunTimeout(t)
+// }
 
-func TestRunTimeout(t *testing.T) {
-	common.TestRunTimeout(t)
+func TestReallyLongLogs(t *testing.T) {
+	common.TestReallyLongLogs(t)
 }