TT-1825 Allow to pass extra args to flaky detector workflow (#15050)

* Allow to pass extra args to flaky detector workflow * Bump * Fix * Update summary
smartcontractkit · Oct 31, 2024 · acc3ca0 · acc3ca0
1 parent f585064
commit acc3ca0
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 24 deletions.
diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml
@@ -414,12 +414,12 @@ jobs:
       projectPath: '.'
       baseRef: ${{ github.base_ref }}
       headRef: ${{ github.head_ref }}
-      runCount: '5'
       runThreshold: '1'
       runWithRace: true
       findByTestFilesDiff: true
       findByAffectedPackages: false
       slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications
+      extraArgs: '{ "skipped_tests": "TestChainComponents" }'
     secrets:
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
 
@@ -433,12 +433,12 @@ jobs:
       projectPath: 'deployment'
       baseRef: ${{ github.base_ref }}
       headRef: ${{ github.head_ref }}
-      runCount: '5'
       runThreshold: '1'
       runWithRace: true
       findByTestFilesDiff: true
       findByAffectedPackages: false
       slackNotificationAfterTestsChannelId: 'C07TRF65CNS' #flaky-test-detector-notifications
+      extraArgs: '{ "skipped_tests": "TestAddLane" }'
     secrets:
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
 

diff --git a/.github/workflows/find-new-flaky-tests.yml b/.github/workflows/find-new-flaky-tests.yml
@@ -20,11 +20,6 @@ on:
         required: false
         type: string
         description: 'The head reference or branch to compare changes for detecting flaky tests. Default is the current branch.'  
-      runCount:
-        required: false
-        type: string
-        description: 'The number of times to run the tests to detect flaky tests.'
-        default: '5'
       runThreshold:
         required: false
         type: string
@@ -35,11 +30,6 @@ on:
         type: boolean
         description: 'Run tests with -race flag.'
         default: true
-      maxGroupSize:
-        required: false
-        type: string
-        description: 'The maximum number of jobs to run in parallel when running tests.'
-        default: '8'
       findByTestFilesDiff:
         required: false
         type: boolean
@@ -54,13 +44,20 @@ on:
         description: "Slack channel ID to send the notification to for failed tests."
         required: false
         type: string
+      extraArgs:
+        required: false
+        type: string
+        default: '{}'
+        description: 'JSON of extra arguments for the workflow.'
     secrets:
       SLACK_BOT_TOKEN:
         required: false
 
 env:
   GIT_HEAD_REF: ${{ inputs.headRef || github.ref }}
-  SKIPPED_TESTS: "TestChainComponents" # Comma separated list of test names to skip running in the flaky detector. Related issue: TT-1823
+  SKIPPED_TESTS: ${{ fromJson(inputs.extraArgs)['skipped_tests'] || '' }} # Comma separated list of test names to skip running in the flaky detector. Related issue: TT-1823
+  MAX_GROUP_SIZE: ${{ fromJson(inputs.extraArgs)['max_group_size'] || '8' }} # The maximum number of jobs to run in parallel when running tests.
+  RUN_COUNT: ${{ fromJson(inputs.extraArgs)['run_count'] || '5' }} # The number of times to run the tests to detect flaky tests.
 
 jobs:
   find-tests:
@@ -85,7 +82,7 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@88b7b797ea6ba0f44a81de87b63ee94eb6a277df
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@2a8e6cddfd618291855b210c3a9d7810e8ca99d6
 
       - name: Find new or updated test packages
         id: find-tests
@@ -121,7 +118,7 @@ jobs:
         shell: bash
         run: |
           PACKAGES=(${{ steps.find-tests.outputs.packages }})
-          DESIRED_GROUP_COUNT=$((${{ inputs.maxGroupSize }}))
+          DESIRED_GROUP_COUNT=$((${{ env.MAX_GROUP_SIZE }}))
           TOTAL_PACKAGES=${#PACKAGES[@]}
 
           # Number of groups should be no more than the number of packages
@@ -223,11 +220,11 @@ jobs:
 
       - name: Install flakeguard
         shell: bash
-        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@88b7b797ea6ba0f44a81de87b63ee94eb6a277df
+        run: go install github.com/smartcontractkit/chainlink-testing-framework/tools/flakeguard@2a8e6cddfd618291855b210c3a9d7810e8ca99d6
 
       - name: Run tests with flakeguard
         shell: bash
-        run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ inputs.runCount }} --threshold=${{ inputs.runThreshold }} --race=${{ inputs.runWithRace }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json
+        run: flakeguard run --project-path=${{ inputs.projectPath }} --test-packages=${{ matrix.testPackages }} --run-count=${{ env.RUN_COUNT }} --threshold=${{ inputs.runThreshold }} --race=${{ inputs.runWithRace }} --skip-tests=${{ env.SKIPPED_TESTS }} --output-json=test-result.json
         env:
           CL_DATABASE_URL: ${{ env.DB_URL }}
 
@@ -360,8 +357,9 @@ jobs:
           echo "| **Setting**             | **Value**  |" >> $GITHUB_STEP_SUMMARY
           echo "|-------------------------|------------|" >> $GITHUB_STEP_SUMMARY
           echo "| Flakiness Threshold     | ${threshold_percentage}%       |" >> $GITHUB_STEP_SUMMARY
-          echo "| Test Run Count          | ${{ inputs.runCount }}         |" >> $GITHUB_STEP_SUMMARY
+          echo "| Test Run Count          | ${{ env.RUN_COUNT }}         |" >> $GITHUB_STEP_SUMMARY
           echo "| Race Detection          | ${{ inputs.runWithRace }}      |" >> $GITHUB_STEP_SUMMARY
+          echo "| Excluded Tests          | ${{ env.SKIPPED_TESTS }}      |" >> $GITHUB_STEP_SUMMARY
           
       - name: Append No Tests Found Message to GitHub Summary
         if: ${{ fromJson(steps.set_test_results.outputs.all_tests_count) == 0 }}

diff --git a/.github/workflows/run-find-new-flaky-tests.yml b/.github/workflows/run-find-new-flaky-tests.yml
@@ -22,11 +22,6 @@ on:
         required: false
         type: string
         description: 'The head reference or branch to compare changes for detecting flaky tests. Default is the current branch.'        
-      runCount:
-        required: false
-        type: string
-        description: 'The number of times to run the tests to detect flaky tests.'
-        default: '5'
       runThreshold:
         required: false
         type: string
@@ -51,6 +46,11 @@ on:
         description: "Slack channel ID to send the notification to for failed tests."
         required: false
         type: string
+      extraArgs:
+        required: false
+        type: string
+        default: '{}'
+        description: 'JSON of extra arguments for the workflow.'        
 
 jobs:
   trigger-flaky-test-detection:
@@ -61,11 +61,11 @@ jobs:
       baseRef: ${{ inputs.baseRef }}
       projectPath: ${{ inputs.projectPath }}
       headRef: ${{ inputs.headRef }}
-      runCount: ${{ inputs.runCount }}
       runThreshold: ${{ inputs.runThreshold }}
       runWithRace: ${{ inputs.runWithRace }}
       findByTestFilesDiff: ${{ inputs.findByTestFilesDiff }}
       findByAffectedPackages: ${{ inputs.findByAffectedPackages }}
       slackNotificationAfterTestsChannelId: ${{ inputs.slack_notification_after_tests_channel_id }}
+      extraArgs: ${{ inputs.extraArgs }}
     secrets:
       SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}