From 7ba2d847a6e18766f61c18a09380a6290da33cfb Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:25:03 +0100 Subject: [PATCH 1/8] Run test with logs --- .github/workflows/ci-core.yml | 2 +- .github/workflows/flakeguard.yml | 4 ++-- .../evmregistry/v21/transmit/event_provider_test.go | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 9134a8c9b56..651c59e9183 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -463,7 +463,7 @@ jobs: findByTestFilesDiff: true findByAffectedPackages: false slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications - extraArgs: '{ "skipped_tests": "TestChainComponents", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "3", "min_pass_ratio": "0.01" }' + extraArgs: '{ "skipped_tests": "TestChainComponents", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "30", "omit_test_outputs_on_success": "false" }' secrets: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/flakeguard.yml b/.github/workflows/flakeguard.yml index 82d33e3e124..044f16486a2 100644 --- a/.github/workflows/flakeguard.yml +++ b/.github/workflows/flakeguard.yml @@ -68,7 +68,7 @@ env: ALL_TESTS_RUNNER: ${{ fromJSON(inputs.extraArgs)['all_tests_runner'] || 'ubuntu22.04-32cores-128GB' }} # The runner to use for running all tests. DEFAULT_RUNNER: 'ubuntu-latest' # The default runner to use for running tests. UPLOAD_ALL_TEST_RESULTS: ${{ fromJSON(inputs.extraArgs)['upload_all_test_results'] || 'false' }} # Whether to upload all test results as artifacts. - + OMIT_TEST_OUTPUTS_ON_SUCCESS: ${{ fromJSON(inputs.extraArgs)['omit_test_outputs_on_success'] || 'true' }} jobs: get-tests: @@ -286,7 +286,7 @@ jobs: - name: Run tests with flakeguard shell: bash - run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --max-pass-ratio=${{ inputs.maxPassRatio }} --race=${{ env.RUN_WITH_RACE }} --shuffle=${{ env.RUN_WITH_SHUFFLE }} --shuffle-seed=${{ env.SHUFFLE_SEED }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json --omit-test-outputs-on-success=true + run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.TEST_REPEAT_COUNT }} --max-pass-ratio=${{ inputs.maxPassRatio }} --race=${{ env.RUN_WITH_RACE }} --shuffle=${{ env.RUN_WITH_SHUFFLE }} --shuffle-seed=${{ env.SHUFFLE_SEED }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json --omit-test-outputs-on-success=${{ env.OMIT_TEST_OUTPUTS_ON_SUCCESS }} env: CL_DATABASE_URL: ${{ env.DB_URL }} diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go index fb6ebf24462..fa2ae8a3d30 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go @@ -24,6 +24,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core" ) +// test + func TestTransmitEventProvider_Sanity(t *testing.T) { ctx := testutils.Context(t) From 9211ba51142a10da3d7e702f6680775a359222ce Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:34:56 +0100 Subject: [PATCH 2/8] bump flakeguard --- .github/workflows/flakeguard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flakeguard.yml b/.github/workflows/flakeguard.yml index 044f16486a2..afb40492b6a 100644 --- a/.github/workflows/flakeguard.yml +++ b/.github/workflows/flakeguard.yml @@ -123,7 +123,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@fc2d7e38486853d2bed06e9074868087f5b55506 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@361bf9373e406470603a30a4268a7f95f2deaf60 # flakguard@0.1.0 - name: Find new or updated test packages if: ${{ inputs.runAllTests == false }} @@ -282,7 +282,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@fc2d7e38486853d2bed06e9074868087f5b55506 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@361bf9373e406470603a30a4268a7f95f2deaf60 # flakguard@0.1.0 - name: Run tests with flakeguard shell: bash @@ -329,7 +329,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@fc2d7e38486853d2bed06e9074868087f5b55506 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@361bf9373e406470603a30a4268a7f95f2deaf60 # flakguard@0.1.0 - name: Aggregate Flakeguard Results id: results From b95033500bf13815d726af975557d1d019dfa47f Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:45:45 +0100 Subject: [PATCH 3/8] bump flakeguard --- .github/workflows/flakeguard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flakeguard.yml b/.github/workflows/flakeguard.yml index afb40492b6a..d0a819a00b6 100644 --- a/.github/workflows/flakeguard.yml +++ b/.github/workflows/flakeguard.yml @@ -123,7 +123,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@361bf9373e406470603a30a4268a7f95f2deaf60 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@edfd2a4f5d83fb5e2596332a41bc172379e90cdc # flakguard@0.1.0 - name: Find new or updated test packages if: ${{ inputs.runAllTests == false }} @@ -282,7 +282,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@361bf9373e406470603a30a4268a7f95f2deaf60 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@edfd2a4f5d83fb5e2596332a41bc172379e90cdc # flakguard@0.1.0 - name: Run tests with flakeguard shell: bash @@ -329,7 +329,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@361bf9373e406470603a30a4268a7f95f2deaf60 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@edfd2a4f5d83fb5e2596332a41bc172379e90cdc # flakguard@0.1.0 - name: Aggregate Flakeguard Results id: results From 2d679c49a577b017ec817d10ba0a23dd3c92bd3c Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Mon, 16 Dec 2024 12:33:55 +0100 Subject: [PATCH 4/8] bump --- .github/workflows/flakeguard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flakeguard.yml b/.github/workflows/flakeguard.yml index d0a819a00b6..99ff5de30ab 100644 --- a/.github/workflows/flakeguard.yml +++ b/.github/workflows/flakeguard.yml @@ -123,7 +123,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@edfd2a4f5d83fb5e2596332a41bc172379e90cdc # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@16cbe0d2d2532e5ffa1b95b97d7a018103ebb767 # flakguard@0.1.0 - name: Find new or updated test packages if: ${{ inputs.runAllTests == false }} @@ -282,7 +282,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@edfd2a4f5d83fb5e2596332a41bc172379e90cdc # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@16cbe0d2d2532e5ffa1b95b97d7a018103ebb767 # flakguard@0.1.0 - name: Run tests with flakeguard shell: bash @@ -329,7 +329,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@edfd2a4f5d83fb5e2596332a41bc172379e90cdc # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@16cbe0d2d2532e5ffa1b95b97d7a018103ebb767 # flakguard@0.1.0 - name: Aggregate Flakeguard Results id: results From 2eb6e4089efd3523d9e29141be1f14fb1ce07564 Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:05:23 +0100 Subject: [PATCH 5/8] bump --- .github/workflows/flakeguard.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flakeguard.yml b/.github/workflows/flakeguard.yml index 99ff5de30ab..3951c356a3b 100644 --- a/.github/workflows/flakeguard.yml +++ b/.github/workflows/flakeguard.yml @@ -123,7 +123,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@16cbe0d2d2532e5ffa1b95b97d7a018103ebb767 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@ea4ffd8c51ce02efebf5ea6bca503fe10b6cee92 # flakguard@0.1.0 - name: Find new or updated test packages if: ${{ inputs.runAllTests == false }} @@ -282,7 +282,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@16cbe0d2d2532e5ffa1b95b97d7a018103ebb767 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@ea4ffd8c51ce02efebf5ea6bca503fe10b6cee92 # flakguard@0.1.0 - name: Run tests with flakeguard shell: bash @@ -329,7 +329,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@16cbe0d2d2532e5ffa1b95b97d7a018103ebb767 # flakguard@0.1.0 + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@ea4ffd8c51ce02efebf5ea6bca503fe10b6cee92 # flakguard@0.1.0 - name: Aggregate Flakeguard Results id: results From 2275efdb785960f4a46303645bc47821b988097b Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:06:04 +0100 Subject: [PATCH 6/8] fix --- .github/workflows/ci-core.yml | 4 ++-- .../evmregistry/v21/transmit/event_provider_test.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 651c59e9183..726b6b14074 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -463,7 +463,7 @@ jobs: findByTestFilesDiff: true findByAffectedPackages: false slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications - extraArgs: '{ "skipped_tests": "TestChainComponents", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "30", "omit_test_outputs_on_success": "false" }' + extraArgs: '{ "skipped_tests": "TestChainComponents", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "3", "omit_test_outputs_on_success": "true" }' secrets: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -482,7 +482,7 @@ jobs: findByTestFilesDiff: true findByAffectedPackages: false slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications - extraArgs: '{ "skipped_tests": "TestAddLane", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "3", "min_pass_ratio": "0.01" }' + extraArgs: '{ "skipped_tests": "TestAddLane", "run_with_race": "true", "print_failed_tests": "true", "test_repeat_count": "3", "omit_test_outputs_on_success": "true" }' secrets: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go index fa2ae8a3d30..fb6ebf24462 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/transmit/event_provider_test.go @@ -24,8 +24,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core" ) -// test - func TestTransmitEventProvider_Sanity(t *testing.T) { ctx := testutils.Context(t) From 04833e06cc0f7184445a0119296c86983a85c511 Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:48:34 +0100 Subject: [PATCH 7/8] fail test --- core/web/resolver/spec_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/web/resolver/spec_test.go b/core/web/resolver/spec_test.go index 69d6a56509c..b0dbfa1aa73 100644 --- a/core/web/resolver/spec_test.go +++ b/core/web/resolver/spec_test.go @@ -42,7 +42,7 @@ func TestResolver_CronSpec(t *testing.T) { Type: job.Cron, CronSpec: &job.CronSpec{ CronSchedule: "CRON_TZ=UTC 0 0 1 1 *", - EVMChainID: ubig.NewI(42), + EVMChainID: ubig.NewI(45), CreatedAt: f.Timestamp(), }, }, nil) From 37ece36b8fc8fb8a60d2fc653514320370f89ab3 Mon Sep 17 00:00:00 2001 From: lukaszcl <120112546+lukaszcl@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:17:49 +0100 Subject: [PATCH 8/8] fix test --- core/web/resolver/spec_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/web/resolver/spec_test.go b/core/web/resolver/spec_test.go index b0dbfa1aa73..69d6a56509c 100644 --- a/core/web/resolver/spec_test.go +++ b/core/web/resolver/spec_test.go @@ -42,7 +42,7 @@ func TestResolver_CronSpec(t *testing.T) { Type: job.Cron, CronSpec: &job.CronSpec{ CronSchedule: "CRON_TZ=UTC 0 0 1 1 *", - EVMChainID: ubig.NewI(45), + EVMChainID: ubig.NewI(42), CreatedAt: f.Timestamp(), }, }, nil)