Skip to content

Commit

Permalink
WIP (#10542)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cedric-cordenier
cedric-cordenier authored Sep 8, 2023
1 parent 7d44d7b commit 1347424
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 195 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci-core.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ jobs:
-gh_sha=$GITHUB_SHA \
-gh_event_path=$GITHUB_EVENT_PATH \
-command=./tools/bin/go_core_tests \
`ls -R ./artifacts/go_core_tests*/output-short.txt`
`ls -R ./artifacts/go_core_tests*/output.txt`
- name: Store logs artifacts
if: always()
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
Expand Down
65 changes: 45 additions & 20 deletions tools/flakeytests/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,19 @@ package flakeytests
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"os"
"os/exec"
"regexp"
"strings"
"time"
)

var (
failedTestRe = regexp.MustCompile(`^--- FAIL: (Test\w+)`)
logPanicRe = regexp.MustCompile(`^panic: Log in goroutine after (Test\w+)`)

failedPkgRe = regexp.MustCompile(`^FAIL\s+github\.com\/smartcontractkit\/chainlink\/v2\/(\S+)`)
panicRe = regexp.MustCompile(`^panic:`)
)

type Runner struct {
Expand Down Expand Up @@ -53,29 +52,55 @@ func runGoTest(pkg string, tests []string, numReruns int, w io.Writer) error {
return cmd.Run()
}

type TestEvent struct {
Time time.Time
Action string
Package string
Test string
Elapsed float64 // seconds
Output string
}

func newEvent(b []byte) (*TestEvent, error) {
e := &TestEvent{}
err := json.Unmarshal(b, e)
if err != nil {
return nil, err
}

e.Package = strings.Replace(e.Package, "github.com/smartcontractkit/chainlink/v2/", "", -1)
return e, nil

}

func parseOutput(readers ...io.Reader) (map[string]map[string]int, error) {
testsWithoutPackage := []string{}
tests := map[string]map[string]int{}
for _, r := range readers {
s := bufio.NewScanner(r)
for s.Scan() {
t := s.Text()
switch {
case failedTestRe.MatchString(t):
m := failedTestRe.FindStringSubmatch(t)
testsWithoutPackage = append(testsWithoutPackage, m[1])
case logPanicRe.MatchString(t):
m := logPanicRe.FindStringSubmatch(t)
testsWithoutPackage = append(testsWithoutPackage, m[1])
case failedPkgRe.MatchString(t):
p := failedPkgRe.FindStringSubmatch(t)
for _, t := range testsWithoutPackage {
if tests[p[1]] == nil {
tests[p[1]] = map[string]int{}
t := s.Bytes()
if len(t) == 0 {
continue
}

e, err := newEvent(t)
if err != nil {
return nil, err
}

switch e.Action {
case "fail":
if tests[e.Package] == nil {
tests[e.Package] = map[string]int{}
}
tests[e.Package][e.Test]++
case "output":
if panicRe.MatchString(e.Output) {
if tests[e.Package] == nil {
tests[e.Package] = map[string]int{}
}
tests[p[1]][t]++
tests[e.Package][e.Test]++
}
testsWithoutPackage = []string{}
}
}

Expand Down
197 changes: 23 additions & 174 deletions tools/flakeytests/runner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,7 @@ func newMockReporter() *mockReporter {
}

func TestParser(t *testing.T) {
output := `
--- FAIL: TestLink (0.00s)
--- FAIL: TestLink/1.1_link#01 (0.00s)
currencies_test.go:325:
Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
Error: Not equal:
expected: "1.2 link"
actual : "1.1 link"
Diff:
--- Expected
+++ Actual
@@ -1 +1 @@
-1.2 link
+1.1 link
Test: TestLink/1.1_link#01
FAIL
FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s
FAIL
output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}
`

r := strings.NewReader(output)
Expand All @@ -53,80 +35,34 @@ FAIL
assert.Equal(t, ts["core/assets"]["TestLink"], 1)
}

func TestParser_PanicInTest(t *testing.T) {
output := `
? github.com/smartcontractkit/chainlink/v2/tools/flakeytests/cmd/runner [no test files]
--- FAIL: TestParser (0.00s)
panic: foo [recovered]
panic: foo
goroutine 21 [running]:
testing.tRunner.func1.2({0x1009953c0, 0x1009d1e40})
/opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1526 +0x1c8
testing.tRunner.func1()
/opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1529 +0x384
panic({0x1009953c0, 0x1009d1e40})
/opt/homebrew/Cellar/go/1.20.3/libexec/src/runtime/panic.go:884 +0x204
github.com/smartcontractkit/chainlink/v2/tools/flakeytests.TestParser(0x0?)
/Users/ccordenier/Development/chainlink/tools/flakeytests/runner_test.go:50 +0xa4
testing.tRunner(0x14000083520, 0x1009d1588)
/opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1576 +0x10c
created by testing.(*T).Run
/opt/homebrew/Cellar/go/1.20.3/libexec/src/testing/testing.go:1629 +0x368
FAIL github.com/smartcontractkit/chainlink/v2/tools/flakeytests 0.197s
FAIL`

r := strings.NewReader(output)
ts, err := parseOutput(r)
require.NoError(t, err)

assert.Len(t, ts, 1)
assert.Len(t, ts["tools/flakeytests"], 1)
assert.Equal(t, ts["tools/flakeytests"]["TestParser"], 1)
}

func TestParser_PanicDueToLogging(t *testing.T) {
output := `
panic: Log in goroutine after TestIntegration_LogEventProvider_Backfill has completed: 2023-07-19T10:10:45.925Z WARN KeepersRegistry.LogEventProvider logprovider/provider.go:218 failed to read logs {"version": "2.3.0@d898528", "where": "reader", "err": "fetched logs with errors: context canceled"}
goroutine 4999 [running]:
testing.(*common).logDepth(0xc0051f6000, {0xc003011960, 0xd3}, 0x3)
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1003 +0x4e7
testing.(*common).log(...)
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:985
testing.(*common).Logf(0xc0051f6000, {0x21ba777?, 0x41ac8a?}, {0xc00217c330?, 0x1e530c0?, 0x1?})
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1036 +0x5a
go.uber.org/zap/zaptest.testingWriter.Write({{0x7f4c5c94f018?, 0xc0051f6000?}, 0xa8?}, {0xc003017000?, 0xd4, 0xc00217c320?})
/home/runner/go/pkg/mod/go.uber.org/[email protected]/zaptest/logger.go:130 +0xe6
go.uber.org/zap/zapcore.(*ioCore).Write(0xc0022f60f0, {0x1, {0xc1260b897723e1ca, 0x4bdc75e54, 0x3d56e40}, {0x22e96a1, 0x20}, {0x22c3204, 0x13}, {0x1, ...}, ...}, ...)
/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/core.go:99 +0xb5
go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc001265ba0, {0xc0023ca100, 0x1, 0x2})
/home/runner/go/pkg/mod/go.uber.org/[email protected]/zapcore/entry.go:255 +0x1d9
go.uber.org/zap.(*SugaredLogger).log(0xc00363a008, 0x1, {0x22c3204?, 0x13?}, {0x0?, 0x0?, 0x0?}, {0xc004ea3f80, 0x2, 0x2})
/home/runner/go/pkg/mod/go.uber.org/[email protected]/sugar.go:295 +0xee
go.uber.org/zap.(*SugaredLogger).Warnw(...)
/home/runner/go/pkg/mod/go.uber.org/[email protected]/sugar.go:216
github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider.(*logEventProvider).startReader(0xc00076d730, {0x2917018?, 0xc0043c4000?}, 0xc003b2a000)
/home/runner/work/chainlink/chainlink/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider/provider.go:218 +0x29f
created by github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider.(*logEventProvider).Start
/home/runner/work/chainlink/chainlink/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider/provider.go:108 +0x133
FAIL github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/logprovider 20.380s
FAIL
{"Time":"2023-09-07T16:01:40.649849+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_LinkScanValue","Output":"panic: foo\n"}
`

r := strings.NewReader(output)
ts, err := parseOutput(r)
require.NoError(t, err)

assert.Len(t, ts, 1)
assert.Len(t, ts["core/services/ocr2/plugins/ocr2keeper/evm21/logprovider"], 1)
assert.Equal(t, ts["core/services/ocr2/plugins/ocr2keeper/evm21/logprovider"]["TestIntegration_LogEventProvider_Backfill"], 1)
assert.Len(t, ts["core/assets"], 1)
assert.Equal(t, ts["core/assets"]["TestAssets_LinkScanValue"], 1)
}

func TestParser_SuccessfulOutput(t *testing.T) {
output := `
? github.com/smartcontractkit/chainlink/v2/tools/flakeytests/cmd/runner [no test files]
ok github.com/smartcontractkit/chainlink/v2/tools/flakeytests 0.320s
{"Time":"2023-09-07T16:22:52.556853+01:00","Action":"start","Package":"github.com/smartcontractkit/chainlink/v2/core/assets"}
{"Time":"2023-09-07T16:22:52.762353+01:00","Action":"run","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString"}
{"Time":"2023-09-07T16:22:52.762456+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"=== RUN TestAssets_NewLinkAndString\n"}
{"Time":"2023-09-07T16:22:52.76249+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"=== PAUSE TestAssets_NewLinkAndString\n"}
{"Time":"2023-09-07T16:22:52.7625+01:00","Action":"pause","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString"}
{"Time":"2023-09-07T16:22:52.762511+01:00","Action":"cont","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString"}
{"Time":"2023-09-07T16:22:52.762528+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"=== CONT TestAssets_NewLinkAndString\n"}
{"Time":"2023-09-07T16:22:52.762546+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Output":"--- PASS: TestAssets_NewLinkAndString (0.00s)\n"}
{"Time":"2023-09-07T16:22:52.762557+01:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestAssets_NewLinkAndString","Elapsed":0}
{"Time":"2023-09-07T16:22:52.762566+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Output":"PASS\n"}
{"Time":"2023-09-07T16:22:52.762955+01:00","Action":"output","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Output":"ok \tgithub.com/smartcontractkit/chainlink/v2/core/assets\t0.206s\n"}
{"Time":"2023-09-07T16:22:52.765598+01:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Elapsed":0.209}
`

r := strings.NewReader(output)
Expand All @@ -136,26 +72,7 @@ ok github.com/smartcontractkit/chainlink/v2/tools/flakeytests 0.320s
}

func TestRunner_WithFlake(t *testing.T) {
output := `
--- FAIL: TestLink (0.00s)
--- FAIL: TestLink/1.1_link#01 (0.00s)
currencies_test.go:325:
Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
Error: Not equal:
expected: "1.2 link"
actual : "1.1 link"
Diff:
--- Expected
+++ Actual
@@ -1 +1 @@
-1.2 link
+1.1 link
Test: TestLink/1.1_link#01
FAIL
FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s
FAIL
`
output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}`
m := newMockReporter()
r := &Runner{
numReruns: 2,
Expand All @@ -177,61 +94,11 @@ FAIL
}

func TestRunner_AllFailures(t *testing.T) {
output := `
--- FAIL: TestLink (0.00s)
--- FAIL: TestLink/1.1_link#01 (0.00s)
currencies_test.go:325:
Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
Error: Not equal:
expected: "1.2 link"
actual : "1.1 link"
Diff:
--- Expected
+++ Actual
@@ -1 +1 @@
-1.2 link
+1.1 link
Test: TestLink/1.1_link#01
FAIL
FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s
FAIL
`
output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}`

rerunOutput := `
--- FAIL: TestLink (0.00s)
--- FAIL: TestLink/1.1_link#01 (0.00s)
currencies_test.go:325:
Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
Error: Not equal:
expected: "1.2 link"
actual : "1.1 link"
Diff:
--- Expected
+++ Actual
@@ -1 +1 @@
-1.2 link
+1.1 link
Test: TestLink/1.1_link#01
--- FAIL: TestLink (0.00s)
--- FAIL: TestLink/1.1_link#01 (0.00s)
currencies_test.go:325:
Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
Error: Not equal:
expected: "1.2 link"
actual : "1.1 link"
Diff:
--- Expected
+++ Actual
@@ -1 +1 @@
-1.2 link
+1.1 link
Test: TestLink/1.1_link#01
FAIL
FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.315s
FAIL
{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}
{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}
`
m := newMockReporter()
r := &Runner{
Expand All @@ -251,29 +118,11 @@ FAIL
}

func TestRunner_RerunSuccessful(t *testing.T) {
output := `
--- FAIL: TestLink (0.00s)
--- FAIL: TestLink/1.1_link#01 (0.00s)
currencies_test.go:325:
Error Trace: /Users/ccordenier/Development/chainlink/core/assets/currencies_test.go:325
Error: Not equal:
expected: "1.2 link"
actual : "1.1 link"
Diff:
--- Expected
+++ Actual
@@ -1 +1 @@
-1.2 link
+1.1 link
Test: TestLink/1.1_link#01
FAIL
FAIL github.com/smartcontractkit/chainlink/v2/core/assets 0.338s
FAIL
`
output := `{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}`

rerunOutput := `
ok github.com/smartcontractkit/chainlink/v2/core/assets 0.320s
{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"fail","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}
{"Time":"2023-09-07T15:39:46.378315+01:00","Action":"pass","Package":"github.com/smartcontractkit/chainlink/v2/core/assets","Test":"TestLink","Elapsed":0}
`
m := newMockReporter()
r := &Runner{
Expand Down

0 comments on commit 1347424

Please sign in to comment.