CCIP Chaos Tests #932
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CCIP Chaos Tests | |
| on: | |
| workflow_run: | |
| workflows: [ CCIP Load Test ] | |
| types: [ completed ] | |
| branches: [ ccip-develop ] | |
| workflow_dispatch: | |
| # Only run 1 of this workflow at a time per PR | |
| concurrency: | |
| group: chaos-ccip-tests-chainlink-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CL_ECR: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip-tests:${{ github.sha }} | |
| MOD_CACHE_VERSION: 1 | |
| jobs: | |
| build-chainlink: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Chainlink Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Check if image exists | |
| id: check-image | |
| uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| repository: chainlink | |
| tag: ${{ github.sha }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Build Image | |
| if: steps.check-image.outputs.exists == 'false' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| with: | |
| cl_repo: smartcontractkit/chainlink-ccip | |
| cl_ref: ${{ github.sha }} | |
| push_tag: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:${{ github.sha }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-chaos-tests-build-chainlink-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Chainlink Image | |
| continue-on-error: true | |
| build-test-image: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Test Image | |
| runs-on: ubuntu20.04-16cores-64GB | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-chaos-tests-build-test-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Test Image | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Build Test Image | |
| uses: ./.github/actions/build-test-image | |
| with: | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| ccip-chaos-tests: | |
| environment: integration | |
| permissions: | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| name: CCIP Chaos Tests | |
| runs-on: ubuntu-latest | |
| needs: [ build-chainlink, build-test-image ] | |
| env: | |
| TEST_SUITE: chaos | |
| TEST_ARGS: -test.timeout 30m | |
| CHAINLINK_COMMIT_SHA: ${{ github.sha }} | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| TEST_TRIGGERED_BY: ccip-cron-chaos-eth | |
| TEST_LOG_LEVEL: debug | |
| DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-chaos-tests | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: CCIP Chaos Tests | |
| test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Prepare Base64 TOML override for CCIP secrets | |
| uses: ./.github/actions/setup-create-base64-config-ccip | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| chainlinkVersion: ${{ github.sha }} | |
| lokiEndpoint: ${{ secrets.LOKI_URL }} | |
| lokiTenantId: ${{ vars.LOKI_TENANT_ID }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| grafanaUrl: ${{ vars.GRAFANA_URL }} | |
| grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
| - name: Run Chaos Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| test_command_to_run: cd ./integration-tests && go test -timeout 1h -count=1 -json -test.parallel 11 -run 'TestChaosCCIP' ./chaos 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci | |
| test_download_vendor_packages_command: make gomod | |
| cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| cl_image_tag: ${{ github.sha }} | |
| artifacts_location: ./integration-tests/chaos/logs | |
| publish_check_name: CCIP Chaos Test Results | |
| publish_report_paths: ./tests-chaos-report.xml | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| CGO_ENABLED: "1" | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| ## Notify in slack if the job fails | |
| - name: Notify Slack | |
| if: failure() && github.event_name != 'workflow_dispatch' | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} | |
| with: | |
| channel-id: "#ccip-testing" | |
| slack-message: ":x: :mild-panic-intensifies: CCIP chaos tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" | |
| ## Run Cleanup if the job succeeds | |
| - name: cleanup | |
| if: always() | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
| ccip-chaos-with-load-tests: | |
| environment: integration | |
| permissions: | |
| issues: read | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| name: CCIP Load With Chaos Tests | |
| if: false # Disabled until CCIP-2555 is resolved | |
| runs-on: ubuntu-latest | |
| needs: [ build-chainlink, build-test-image ] | |
| env: | |
| TEST_SUITE: load | |
| TEST_ARGS: -test.timeout 1h | |
| CHAINLINK_COMMIT_SHA: ${{ github.sha }} | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| TEST_TRIGGERED_BY: ccip-cron-chaos-and-load-eth | |
| TEST_LOG_LEVEL: debug | |
| DATABASE_URL: postgresql://postgres:node@localhost:5432/chainlink_test?sslmode=disable | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: ccip-chaos-tests-with-load-test | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: CCIP load with chaos test | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| - name: Prepare Base64 TOML override for CCIP secrests | |
| uses: ./.github/actions/setup-create-base64-config-ccip | |
| with: | |
| runId: ${{ github.run_id }} | |
| testLogCollect: ${{ vars.TEST_LOG_COLLECT }} | |
| chainlinkImage: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| chainlinkVersion: ${{ github.sha }} | |
| lokiEndpoint: ${{ secrets.LOKI_URL }} | |
| lokiTenantId: ${{ vars.LOKI_TENANT_ID }} | |
| logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| grafanaUrl: ${{ vars.GRAFANA_URL }} | |
| grafanaDashboardUrl: "/d/6vjVx-1V8/ccip-long-running-tests" | |
| - name: Run Load With Chaos Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| test_command_to_run: cd ./integration-tests/ccip-tests && go test -timeout 2h -count=1 -json -test.parallel 4 -run '^TestLoadCCIPStableWithPodChaosDiffCommitAndExec' ./load 2>&1 | tee /tmp/gotest.log | gotestfmt | |
| test_download_vendor_packages_command: make gomod | |
| cl_repo: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| cl_image_tag: ${{ github.sha }} | |
| artifacts_location: ./integration-tests/load/logs | |
| publish_check_name: CCIP Chaos With Load Test Results | |
| publish_report_paths: ./tests-chaos-with-load-report.xml | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| CGO_ENABLED: "1" | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| cache_key_id: ccip-load-${{ env.MOD_CACHE_VERSION }} | |
| cache_restore_only: "true" | |
| ## Notify in slack if the job fails | |
| - name: Notify Slack | |
| if: failure() && github.event_name != 'workflow_dispatch' | |
| uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0 | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }} | |
| with: | |
| channel-id: "#ccip-testing" | |
| slack-message: ":x: :mild-panic-intensifies: CCIP chaos with load tests failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}" | |
| ## Run Cleanup if the job succeeds | |
| - name: cleanup | |
| if: always() | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/cleanup@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| triggered_by: ${{ env.TEST_TRIGGERED_BY }} |