From 3f8ce53332ffedef5cceba4268bffe67fb7a92c9 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Thu, 11 Jul 2024 16:09:50 +0200 Subject: [PATCH] [TT-1376] more stable compatibility pipeline (#13720) * increase cl node startup timeout * use newer print summary action * use /readyz endpoint not /health, when waiting for CL node to be ready * slightly increase min time between emitting events to avoid 'alreadyKnown' tx error, when processing is slow * add step that gathers resource metric * move where telemetry action runs * use latest CTF * put collection in correct place, really * WIP#1 - doesn't hang on error * WIP#3 - better early exit * use concurrent executor in log poller tests * use stronger runners * use newer CTF * use free runners, limit parallel to 30 * try with 10x max parallelism * do not gather workflow metrics * notify bix-scale-eng about compat failures, use latest tagged ctf --- .github/workflows/client-compatibility-tests.yml | 8 +++++++- integration-tests/docker/test_env/cl_node.go | 13 +++++++++++-- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 ++-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 ++-- .../testconfig/log_poller/log_poller.toml | 4 ++-- 7 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml index a9168ff1026..b744ffcc83d 100644 --- a/.github/workflows/client-compatibility-tests.yml +++ b/.github/workflows/client-compatibility-tests.yml @@ -520,6 +520,7 @@ jobs: TEST_LOG_LEVEL: debug strategy: fail-fast: false + max-parallel: 10 matrix: evm_node: ${{fromJson(needs.prepare-compatibility-matrix.outputs.matrix)}} runs-on: ubuntu-latest @@ -570,6 +571,11 @@ jobs: echo "$modified_string" } echo "TEST_LOG_NAME=$(replace_special_chars "${{ matrix.evm_node.product }}-${{ matrix.evm_node.docker_image }}-test-logs")" >> $GITHUB_ENV +# - name: Collect Workflow Telemetry +# uses: catchpoint/workflow-telemetry-action@v2 +# with: +# comment_on_pr: false +# theme: 'dark' - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 with: @@ -648,7 +654,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@U060CGGPY8H>' || 'All Good!' }}" + "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@S0536FX9ZDZ>' || 'All Good!' }}" } }, { diff --git a/integration-tests/docker/test_env/cl_node.go b/integration-tests/docker/test_env/cl_node.go index 1de16c9c6c3..b28d954688a 100644 --- a/integration-tests/docker/test_env/cl_node.go +++ b/integration-tests/docker/test_env/cl_node.go @@ -75,6 +75,14 @@ func WithNodeEnvVars(ev map[string]string) ClNodeOption { } } +func WithStartupTimeout(timeout time.Duration) ClNodeOption { + return func(n *ClNode) { + if timeout != 0 { + n.StartupTimeout = timeout + } + } +} + // Sets custom node container name if name is not empty func WithNodeContainerName(name string) ClNodeOption { return func(c *ClNode) { @@ -130,6 +138,7 @@ func NewClNode(networks []string, imageName, imageVersion string, nodeConfig *ch ContainerVersion: imageVersion, Networks: networks, LogStream: logStream, + StartupTimeout: 3 * time.Minute, }, UserEmail: "local@local.com", UserPassword: "localdevpassword", @@ -447,9 +456,9 @@ func (n *ClNode) getContainerRequest(secrets string) ( "-a", apiCredsPath, }, Networks: append(n.Networks, "tracing"), - WaitingFor: tcwait.ForHTTP("/health"). + WaitingFor: tcwait.ForHTTP("/readyz"). WithPort("6688/tcp"). - WithStartupTimeout(90 * time.Second). + WithStartupTimeout(n.StartupTimeout). WithPollInterval(1 * time.Second), Files: []tc.ContainerFile{ { diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 4c09a23aa4b..764f41ed765 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -29,7 +29,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240709155552-d76403db14e5 - github.com/smartcontractkit/chainlink-testing-framework v1.31.10 + github.com/smartcontractkit/chainlink-testing-framework v1.32.0 github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/havoc/k8schaos v0.0.0-20240409145249-e78d20847e37 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 2aa60ceab5f..432c30e8e9c 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1526,8 +1526,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead6 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead68/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= -github.com/smartcontractkit/chainlink-testing-framework v1.31.10 h1:zFsXW04ceKMg37oR0EaxlTPH7vL9YnnNA4VntvsaOZ4= -github.com/smartcontractkit/chainlink-testing-framework v1.31.10/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k= +github.com/smartcontractkit/chainlink-testing-framework v1.32.0 h1:6emeE/YuMNmdd1pZ3NlJGu94QCa+NUayx26NX0jAQDY= +github.com/smartcontractkit/chainlink-testing-framework v1.32.0/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 862e1ed9949..2d1c6d5d12d 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.4 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240709155552-d76403db14e5 - github.com/smartcontractkit/chainlink-testing-framework v1.31.10 + github.com/smartcontractkit/chainlink-testing-framework v1.32.0 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240702141926-063ceef8c42e diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index e9c8d750aa7..3d24dc6c750 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1516,8 +1516,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead6 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240708150922-0546185ead68/go.mod h1:NbXXQaNFskVMYRut0MvBlcHu/vDgipGMwYjamvjVB9Y= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799 h1:HyLTySm7BR+oNfZqDTkVJ25wnmcTtxBBD31UkFL+kEM= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240709043547-03612098f799/go.mod h1:UVFRacRkP7O7TQAzFmR52v5mUlxf+G1ovMlCQAB/cHU= -github.com/smartcontractkit/chainlink-testing-framework v1.31.10 h1:zFsXW04ceKMg37oR0EaxlTPH7vL9YnnNA4VntvsaOZ4= -github.com/smartcontractkit/chainlink-testing-framework v1.31.10/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k= +github.com/smartcontractkit/chainlink-testing-framework v1.32.0 h1:6emeE/YuMNmdd1pZ3NlJGu94QCa+NUayx26NX0jAQDY= +github.com/smartcontractkit/chainlink-testing-framework v1.32.0/go.mod h1:wSRZGoukZliwfTkghdF4cI1RLHz5k3aDnL+rXhJ6f7k= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239 h1:Kk5OVlx/5g9q3Z3lhxytZS4/f8ds1MiNM8yaHgK3Oe8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240405215812-5a72bc9af239/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= diff --git a/integration-tests/testconfig/log_poller/log_poller.toml b/integration-tests/testconfig/log_poller/log_poller.toml index d2277d619a5..8c74a9a4cdb 100644 --- a/integration-tests/testconfig/log_poller/log_poller.toml +++ b/integration-tests/testconfig/log_poller/log_poller.toml @@ -81,8 +81,8 @@ use_finality_tag = true [LogPoller.Looped] execution_count = 100 -min_emit_wait_time_ms = 200 -max_emit_wait_time_ms = 500 +min_emit_wait_time_ms = 400 +max_emit_wait_time_ms = 600 # test-specific [TestLogPollerFewFiltersFixedDepth.NodeConfig]