Skip to content

Commit

Permalink
Triage really long logs issues
Browse files Browse the repository at this point in the history
  • Loading branch information
tateexon committed Dec 13, 2023
1 parent e9f7824 commit 3f22399
Show file tree
Hide file tree
Showing 6 changed files with 273 additions and 215 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/k8s-e2e.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
name: (k8s package) E2E tests
on:
pull_request:
workflow_dispatch:

concurrency:
group: e2e-tests-k8s-${{ github.ref }}
Expand Down Expand Up @@ -88,6 +89,7 @@ jobs:
path: /tmp/gotest.log

e2e_remote_runner_tests:
timeout-minutes: 12
runs-on: ubuntu-latest
environment: integration
needs: [build_tests]
Expand Down Expand Up @@ -115,7 +117,7 @@ jobs:
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
- name: Upload test log
uses: actions/upload-artifact@v2
if: failure()
# if: failure()
with:
name: remote-runner-test-log
path: /tmp/remoterunnergotest.log
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,10 @@ k8s_test_e2e:
go test ./k8s/e2e/local-runner -count 1 -test.parallel=12 -v $(args)

k8s_test_e2e_ci:
go test ./k8s/e2e/local-runner -count 1 -v -test.parallel=14 -test.timeout=1h -json 2>&1 | tee /tmp/gotest.log | gotestfmt
go test ./k8s/e2e/local-runner -count 1 -v -test.parallel=14 -test.timeout=1h -json 2>&1 | tee /tmp/gotest.log

k8s_test_e2e_ci_remote_runner:
go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=16 -test.timeout=1h -json 2>&1 | tee /tmp/remoterunnergotest.log | gotestfmt
go test ./k8s/e2e/remote-runner -count 1 -v -test.parallel=16 -test.timeout=7m 2>&1 | tee /tmp/remoterunnergotest.log

.PHONY: examples
examples:
Expand Down
59 changes: 52 additions & 7 deletions k8s/client/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package client

import (
"bufio"
"bytes"
"context"
"io"
"os/exec"
Expand All @@ -20,14 +21,58 @@ func ExecCmdWithContext(ctx context.Context, command string) error {
})
}

// readStdPipe continuously read a pipe from the command
// // readStdPipe continuously read a pipe from the command
// func readStdPipe(pipe io.ReadCloser, outputFunction func(string)) {
// scanner := bufio.NewScanner(pipe)

// // Increase the buffer size for the scanner to handle long lines
// const maxCapacity = 1000000 // Adjust this value as needed
// buf := make([]byte, maxCapacity)
// scanner.Buffer(buf, maxCapacity)

// scanner.Split(bufio.ScanLines)
// for scanner.Scan() {
// m := scanner.Text()
// if outputFunction != nil {
// outputFunction(m)
// }
// }
// }

// readStdPipe continuously reads from a given pipe (either stdout or stderr)
// and processes the output line by line using the provided outputFunction.
// It handles lines of any length dynamically without the need for a large predefined buffer.
func readStdPipe(pipe io.ReadCloser, outputFunction func(string)) {
scanner := bufio.NewScanner(pipe)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
m := scanner.Text()
if outputFunction != nil {
outputFunction(m)
reader := bufio.NewReader(pipe)
var output []rune

for {
// ReadLine tries to return a single line, not including the end-of-line bytes.
// The returned line may be incomplete if the line's too long for the buffer.
// isPrefix will be true if the line is longer than the buffer.
chunk, isPrefix, err := reader.ReadLine()

// Handle any errors that occurred during the read.
if err != nil {
// Log any error that's not an EOF (end of file).
if err != io.EOF {
log.Error().Err(err).Msg("Error while reading standard pipe")
}
break
}

// Append the chunk to the output buffer.
// bytes.Runes converts the byte slice to a slice of runes, handling multi-byte characters.
output = append(output, bytes.Runes(chunk)...)

// If isPrefix is false, we've reached the end of the line and can process it.
if !isPrefix {
// Call the output function with the complete line if it's defined.
if outputFunction != nil {
outputFunction(string(output))
}
// Reset output to an empty slice for reading the next line.
output = output[:0]
}
}
}
Expand Down
18 changes: 18 additions & 0 deletions k8s/e2e/common/test_common.go
Original file line number Diff line number Diff line change
Expand Up @@ -446,3 +446,21 @@ func TestRunTimeout(t *testing.T) {
err = e.Run()
require.Error(t, err)
}

func TestReallyLongLogs(t *testing.T) {
t.Parallel()
l := logging.GetTestLogger(t)
testEnvConfig := GetTestEnvConfig(t)
val, _ := os.LookupEnv(config.EnvVarJobImage)
if val != "" && testEnvConfig.Test.Name() != "" {
env := environment.New(testEnvConfig)
err := env.Run()
require.NoError(t, err)
}
// create a 1,000,000 character string
s := ""
for i := 0; i < 1000; i++ {
s = fmt.Sprintf("%s%s", s, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
}
l.Info().Int("len", len(s)).Str("string", s).Msg("string")
}
101 changes: 52 additions & 49 deletions k8s/e2e/local-runner/envs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,71 +3,74 @@ package env_test
import (
"testing"

"github.com/smartcontractkit/chainlink-testing-framework/k8s/config"
"github.com/smartcontractkit/chainlink-testing-framework/k8s/e2e/common"
)

func TestMultiStageMultiManifestConnection(t *testing.T) {
common.TestMultiStageMultiManifestConnection(t)
}
// func TestMultiStageMultiManifestConnection(t *testing.T) {
// common.TestMultiStageMultiManifestConnection(t)
// }

func TestConnectWithoutManifest(t *testing.T) {
common.TestConnectWithoutManifest(t)
}
// func TestConnectWithoutManifest(t *testing.T) {
// common.TestConnectWithoutManifest(t)
// }

func Test5NodesSoakEnvironmentWithPVCs(t *testing.T) {
common.Test5NodesSoakEnvironmentWithPVCs(t)
}
// func Test5NodesSoakEnvironmentWithPVCs(t *testing.T) {
// common.Test5NodesSoakEnvironmentWithPVCs(t)
// }

func TestWithSingleNodeEnv(t *testing.T) {
common.TestWithSingleNodeEnvParallel(t)
}
// func TestWithSingleNodeEnv(t *testing.T) {
// common.TestWithSingleNodeEnvParallel(t)
// }

func TestWithSingleNodeEnvLocalCharts(t *testing.T) {
t.Setenv(config.EnvVarLocalCharts, "true")
common.TestWithSingleNodeEnv(t)
}
// func TestWithSingleNodeEnvLocalCharts(t *testing.T) {
// t.Setenv(config.EnvVarLocalCharts, "true")
// common.TestWithSingleNodeEnv(t)
// }

func TestMultipleNodeWithDiffDBVersionEnv(t *testing.T) {
common.TestMultipleNodeWithDiffDBVersionEnv(t)
}
// func TestMultipleNodeWithDiffDBVersionEnv(t *testing.T) {
// common.TestMultipleNodeWithDiffDBVersionEnv(t)
// }

func TestMinResources5NodesEnv(t *testing.T) {
common.TestMinResources5NodesEnv(t)
}
// func TestMinResources5NodesEnv(t *testing.T) {
// common.TestMinResources5NodesEnv(t)
// }

func TestMinResources5NodesEnvWithBlockscout(t *testing.T) {
common.TestMinResources5NodesEnvWithBlockscout(t)
}
// func TestMinResources5NodesEnvWithBlockscout(t *testing.T) {
// common.TestMinResources5NodesEnvWithBlockscout(t)
// }

func TestMultipleInstancesOfTheSameType(t *testing.T) {
common.TestMultipleInstancesOfTheSameType(t)
}
// func TestMultipleInstancesOfTheSameType(t *testing.T) {
// common.TestMultipleInstancesOfTheSameType(t)
// }

func Test5NodesPlus2MiningGethsReorgEnv(t *testing.T) {
common.Test5NodesPlus2MiningGethsReorgEnv(t)
}
// func Test5NodesPlus2MiningGethsReorgEnv(t *testing.T) {
// common.Test5NodesPlus2MiningGethsReorgEnv(t)
// }

func TestWithChaos(t *testing.T) {
common.TestWithChaos(t)
}
// func TestWithChaos(t *testing.T) {
// common.TestWithChaos(t)
// }

func TestEmptyEnvironmentStartup(t *testing.T) {
common.TestEmptyEnvironmentStartup(t)
}
// func TestEmptyEnvironmentStartup(t *testing.T) {
// common.TestEmptyEnvironmentStartup(t)
// }

func TestRolloutRestartUpdate(t *testing.T) {
common.TestRolloutRestart(t, true)
}
// func TestRolloutRestartUpdate(t *testing.T) {
// common.TestRolloutRestart(t, true)
// }

func TestRolloutRestartBySelector(t *testing.T) {
common.TestRolloutRestart(t, false)
}
// func TestRolloutRestartBySelector(t *testing.T) {
// common.TestRolloutRestart(t, false)
// }

func TestReplaceHelm(t *testing.T) {
common.TestReplaceHelm(t)
}
// func TestReplaceHelm(t *testing.T) {
// common.TestReplaceHelm(t)
// }

// func TestRunTimeout(t *testing.T) {
// common.TestRunTimeout(t)
// }

func TestRunTimeout(t *testing.T) {
common.TestRunTimeout(t)
func TestReallyLongLogs(t *testing.T) {
common.TestReallyLongLogs(t)
}
Loading

0 comments on commit 3f22399

Please sign in to comment.