diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 15a03a1b222..02c1359cdc2 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -414,12 +414,12 @@ jobs: projectPath: '.' baseRef: ${{ github.base_ref }} headRef: ${{ github.head_ref }} - runCount: '5' runThreshold: '1' runWithRace: true findByTestFilesDiff: true findByAffectedPackages: false slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications + extraArgs: '{ "skipped_tests": "TestChainComponents" }' secrets: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} @@ -433,12 +433,12 @@ jobs: projectPath: 'deployment' baseRef: ${{ github.base_ref }} headRef: ${{ github.head_ref }} - runCount: '5' runThreshold: '1' runWithRace: true findByTestFilesDiff: true findByAffectedPackages: false slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications + extraArgs: '{ "skipped_tests": "TestAddLane" }' secrets: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} diff --git a/.github/workflows/find-new-flaky-tests.yml b/.github/workflows/find-new-flaky-tests.yml index 9f964655f84..bbae987f948 100644 --- a/.github/workflows/find-new-flaky-tests.yml +++ b/.github/workflows/find-new-flaky-tests.yml @@ -20,11 +20,6 @@ on: required: false type: string description: 'The head reference or branch to compare changes for detecting flaky tests. Default is the current branch.' - runCount: - required: false - type: string - description: 'The number of times to run the tests to detect flaky tests.' - default: '5' runThreshold: required: false type: string @@ -35,11 +30,6 @@ on: type: boolean description: 'Run tests with -race flag.' default: true - maxGroupSize: - required: false - type: string - description: 'The maximum number of jobs to run in parallel when running tests.' - default: '8' findByTestFilesDiff: required: false type: boolean @@ -54,13 +44,20 @@ on: description: "Slack channel ID to send the notification to for failed tests." required: false type: string + extraArgs: + required: false + type: string + default: '{}' + description: 'JSON of extra arguments for the workflow.' secrets: SLACK_BOT_TOKEN: required: false env: GIT_HEAD_REF: ${{ inputs.headRef || github.ref }} - SKIPPED_TESTS: "TestChainComponents" # Comma separated list of test names to skip running in the flaky detector. Related issue: TT-1823 + SKIPPED_TESTS: ${{ fromJson(inputs.extraArgs)['skipped_tests'] || '' }} # Comma separated list of test names to skip running in the flaky detector. Related issue: TT-1823 + MAX_GROUP_SIZE: ${{ fromJson(inputs.extraArgs)['max_group_size'] || '8' }} # The maximum number of jobs to run in parallel when running tests. + RUN_COUNT: ${{ fromJson(inputs.extraArgs)['run_count'] || '5' }} # The number of times to run the tests to detect flaky tests. jobs: find-tests: @@ -85,7 +82,7 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@88b7b797ea6ba0f44a81de87b63ee94eb6a277df + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@2a8e6cddfd618291855b210c3a9d7810e8ca99d6 - name: Find new or updated test packages id: find-tests @@ -121,7 +118,7 @@ jobs: shell: bash run: | PACKAGES=(${{ steps.find-tests.outputs.packages }}) - DESIRED_GROUP_COUNT=$((${{ inputs.maxGroupSize }})) + DESIRED_GROUP_COUNT=$((${{ env.MAX_GROUP_SIZE }})) TOTAL_PACKAGES=${#PACKAGES[@]} # Number of groups should be no more than the number of packages @@ -223,11 +220,11 @@ jobs: - name: Install flakeguard shell: bash - run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@88b7b797ea6ba0f44a81de87b63ee94eb6a277df + run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@2a8e6cddfd618291855b210c3a9d7810e8ca99d6 - name: Run tests with flakeguard shell: bash - run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ inputs.runCount }} --threshold=${{ inputs.runThreshold }} --race=${{ inputs.runWithRace }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json + run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.RUN_COUNT }} --threshold=${{ inputs.runThreshold }} --race=${{ inputs.runWithRace }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json env: CL_DATABASE_URL: ${{ env.DB_URL }} @@ -360,8 +357,9 @@ jobs: echo "| **Setting** | **Value** |" >> $GITHUB_STEP_SUMMARY echo "|-------------------------|------------|" >> $GITHUB_STEP_SUMMARY echo "| Flakiness Threshold | ${threshold_percentage}% |" >> $GITHUB_STEP_SUMMARY - echo "| Test Run Count | ${{ inputs.runCount }} |" >> $GITHUB_STEP_SUMMARY + echo "| Test Run Count | ${{ env.RUN_COUNT }} |" >> $GITHUB_STEP_SUMMARY echo "| Race Detection | ${{ inputs.runWithRace }} |" >> $GITHUB_STEP_SUMMARY + echo "| Excluded Tests | ${{ env.SKIPPED_TESTS }} |" >> $GITHUB_STEP_SUMMARY - name: Append No Tests Found Message to GitHub Summary if: ${{ fromJson(steps.set_test_results.outputs.all_tests_count) == 0 }} diff --git a/.github/workflows/run-find-new-flaky-tests.yml b/.github/workflows/run-find-new-flaky-tests.yml index 26723eede36..238da78df2b 100644 --- a/.github/workflows/run-find-new-flaky-tests.yml +++ b/.github/workflows/run-find-new-flaky-tests.yml @@ -22,11 +22,6 @@ on: required: false type: string description: 'The head reference or branch to compare changes for detecting flaky tests. Default is the current branch.' - runCount: - required: false - type: string - description: 'The number of times to run the tests to detect flaky tests.' - default: '5' runThreshold: required: false type: string @@ -51,6 +46,11 @@ on: description: "Slack channel ID to send the notification to for failed tests." required: false type: string + extraArgs: + required: false + type: string + default: '{}' + description: 'JSON of extra arguments for the workflow.' jobs: trigger-flaky-test-detection: @@ -61,11 +61,11 @@ jobs: baseRef: ${{ inputs.baseRef }} projectPath: ${{ inputs.projectPath }} headRef: ${{ inputs.headRef }} - runCount: ${{ inputs.runCount }} runThreshold: ${{ inputs.runThreshold }} runWithRace: ${{ inputs.runWithRace }} findByTestFilesDiff: ${{ inputs.findByTestFilesDiff }} findByAffectedPackages: ${{ inputs.findByAffectedPackages }} slackNotificationAfterTestsChannelId: ${{ inputs.slack_notification_after_tests_channel_id }} + extraArgs: ${{ inputs.extraArgs }} secrets: SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} \ No newline at end of file