Skip to content

Commit

Permalink
[TT-1376] more stable compatibility pipeline (#13720)
Browse files Browse the repository at this point in the history
* increase cl node startup timeout

* use newer print summary action

* use /readyz endpoint not /health, when waiting for CL node to be ready

* slightly increase min time between emitting events to avoid 'alreadyKnown' tx error, when processing is slow

* add step that gathers resource metric

* move where telemetry action runs

* use latest CTF

* put collection in correct place, really

* WIP#1 - doesn't hang on error

* WIP#3 - better early exit

* use concurrent executor in log poller tests

* use stronger runners

* use newer CTF

* use free runners, limit parallel to 30

* try with 10x max parallelism

* do not gather workflow metrics

* notify bix-scale-eng about compat failures, use latest tagged ctf
  • Loading branch information
Tofel authored Jul 11, 2024
1 parent 1dd4df7 commit 3f8ce53
Show file tree
Hide file tree
Showing 7 changed files with 26 additions and 11 deletions.
8 changes: 7 additions & 1 deletion .github/workflows/client-compatibility-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,7 @@ jobs:
TEST_LOG_LEVEL: debug
strategy:
fail-fast: false
max-parallel: 10
matrix:
evm_node: ${{fromJson(needs.prepare-compatibility-matrix.outputs.matrix)}}
runs-on: ubuntu-latest
Expand Down Expand Up @@ -570,6 +571,11 @@ jobs:
echo "$modified_string"
}
echo "TEST_LOG_NAME=$(replace_special_chars "${{ matrix.evm_node.product }}-${{ matrix.evm_node.docker_image }}-test-logs")" >> $GITHUB_ENV
# - name: Collect Workflow Telemetry
# uses: catchpoint/workflow-telemetry-action@v2
# with:
# comment_on_pr: false
# theme: 'dark'
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
with:
Expand Down Expand Up @@ -648,7 +654,7 @@ jobs:
"type": "section",
"text": {
"type": "mrkdwn",
"text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@U060CGGPY8H>' || 'All Good!' }}"
"text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@S0536FX9ZDZ>' || 'All Good!' }}"
}
},
{
Expand Down
13 changes: 11 additions & 2 deletions integration-tests/docker/test_env/cl_node.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,14 @@ func WithNodeEnvVars(ev map[string]string) ClNodeOption {
}
}

func WithStartupTimeout(timeout time.Duration) ClNodeOption {
return func(n *ClNode) {
if timeout != 0 {
n.StartupTimeout = timeout
}
}
}

// Sets custom node container name if name is not empty
func WithNodeContainerName(name string) ClNodeOption {
return func(c *ClNode) {
Expand Down Expand Up @@ -130,6 +138,7 @@ func NewClNode(networks []string, imageName, imageVersion string, nodeConfig *ch
ContainerVersion: imageVersion,
Networks: networks,
LogStream: logStream,
StartupTimeout: 3 * time.Minute,
},
UserEmail: "[email protected]",
UserPassword: "localdevpassword",
Expand Down Expand Up @@ -447,9 +456,9 @@ func (n *ClNode) getContainerRequest(secrets string) (
"-a", apiCredsPath,
},
Networks: append(n.Networks, "tracing"),
WaitingFor: tcwait.ForHTTP("/health").
WaitingFor: tcwait.ForHTTP("/readyz").
WithPort("6688/tcp").
WithStartupTimeout(90 * time.Second).
WithStartupTimeout(n.StartupTimeout).
WithPollInterval(1 * time.Second),
Files: []tc.ContainerFile{
{
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ require (
github.com/slack-go/slack v0.12.2
github.com/smartcontractkit/chainlink-automation v1.0.4
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240709155552-d76403db14e5
github.com/smartcontractkit/chainlink-testing-framework v1.31.10
github.com/smartcontractkit/chainlink-testing-framework v1.32.0
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239
github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1526,8 +1526,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead6
github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead68/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU=
github.com/smartcontractkit/chainlink-testing-framework v1.31.10 h1:zFsXW04ceKMg37oR0EaxlTPH7vL9YnnNA4VntvsaOZ4=
github.com/smartcontractkit/chainlink-testing-framework v1.31.10/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k=
github.com/smartcontractkit/chainlink-testing-framework v1.32.0 h1:6emeE/YuMNmdd1pZ3NlJGu94QCa+NUayx26NX0jAQDY=
github.com/smartcontractkit/chainlink-testing-framework v1.32.0/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k=
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/load/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
github.com/slack-go/slack v0.12.2
github.com/smartcontractkit/chainlink-automation v1.0.4
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240709155552-d76403db14e5
github.com/smartcontractkit/chainlink-testing-framework v1.31.10
github.com/smartcontractkit/chainlink-testing-framework v1.32.0
github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/load/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1516,8 +1516,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead6
github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead68/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU=
github.com/smartcontractkit/chainlink-testing-framework v1.31.10 h1:zFsXW04ceKMg37oR0EaxlTPH7vL9YnnNA4VntvsaOZ4=
github.com/smartcontractkit/chainlink-testing-framework v1.31.10/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k=
github.com/smartcontractkit/chainlink-testing-framework v1.32.0 h1:6emeE/YuMNmdd1pZ3NlJGu94QCa+NUayx26NX0jAQDY=
github.com/smartcontractkit/chainlink-testing-framework v1.32.0/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k=
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8=
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU=
github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/testconfig/log_poller/log_poller.toml
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ use_finality_tag = true

[LogPoller.Looped]
execution_count = 100
min_emit_wait_time_ms = 200
max_emit_wait_time_ms = 500
min_emit_wait_time_ms = 400
max_emit_wait_time_ms = 600

# test-specific
[TestLogPollerFewFiltersFixedDepth.NodeConfig]
Expand Down

0 comments on commit 3f8ce53

Please sign in to comment.