Run Automation On Demand Tests #114
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Automation On Demand Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| chainlinkVersion: | |
| description: Chainlink image version to use | |
| required: false | |
| type: string | |
| chainlinkImage: | |
| description: Chainlink image repo to use (Leave empty to build from head/ref) | |
| required: false | |
| type: string | |
| chainlinkVersionUpdate: | |
| description: Chainlink image version to use initially for upgrade test | |
| default: latest | |
| required: true | |
| type: string | |
| chainlinkImageUpdate: | |
| description: Chainlink image repo to use initially for upgrade test | |
| required: true | |
| default: public.ecr.aws/chainlink/chainlink | |
| type: string | |
| enableChaos: | |
| description: Check to enable chaos tests | |
| type: boolean | |
| default: false | |
| required: true | |
| enableReorg: | |
| description: Check to enable reorg tests | |
| type: boolean | |
| default: false | |
| required: true | |
| env: | |
| ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} | |
| CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
| jobs: | |
| build-chainlink: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| strategy: | |
| matrix: | |
| image: | |
| - name: "" | |
| dockerfile: core/chainlink.Dockerfile | |
| tag-suffix: "" | |
| - name: (plugins) | |
| dockerfile: plugins/chainlink.Dockerfile | |
| tag-suffix: -plugins | |
| name: Build Chainlink Image ${{ matrix.image.name }} | |
| runs-on: ubuntu22.04-16cores-64GB | |
| steps: | |
| - name: Collect Metrics | |
| if: inputs.chainlinkImage == '' | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: automation-on-demand-build-chainlink | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Chainlink Image ${{ matrix.image.name }} | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ github.head_ref || github.ref_name }} | |
| - name: Check if image exists | |
| if: inputs.chainlinkImage == '' | |
| id: check-image | |
| uses: smartcontractkit/chainlink-github-actions/docker/image-exists@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| repository: chainlink | |
| tag: ${{ github.sha }}${{ matrix.image.tag-suffix }} | |
| AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Build Image | |
| if: steps.check-image.outputs.exists == 'false' && inputs.chainlinkImage == '' | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| with: | |
| cl_repo: smartcontractkit/chainlink | |
| cl_ref: ${{ github.sha }} | |
| cl_dockerfile: ${{ matrix.image.dockerfile }} | |
| push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| - name: Print Chainlink Image Built | |
| if: inputs.chainlinkImage == '' | |
| run: | | |
| echo "### chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
| echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY | |
| build-test-image: | |
| environment: integration | |
| permissions: | |
| id-token: write | |
| contents: read | |
| name: Build Test Image | |
| runs-on: ubuntu22.04-16cores-64GB | |
| steps: | |
| - name: Collect Metrics | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: automation-on-demand-build-test-image | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Build Test Image | |
| continue-on-error: true | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ github.head_ref || github.ref_name }} | |
| - name: Build Test Image | |
| if: inputs.enableChaos || inputs.enableReorg | |
| uses: ./.github/actions/build-test-image | |
| with: | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| automation-on-demand-tests: | |
| environment: integration | |
| permissions: | |
| checks: write | |
| pull-requests: write | |
| id-token: write | |
| contents: read | |
| needs: [build-chainlink, build-test-image] | |
| env: | |
| CHAINLINK_COMMIT_SHA: ${{ github.sha }} | |
| CHAINLINK_ENV_USER: ${{ github.actor }} | |
| TEST_LOG_LEVEL: info | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tests: | |
| - name: chaos | |
| id: chaos | |
| suite: chaos | |
| nodes: 15 | |
| os: ubuntu-latest | |
| enabled: ${{ inputs.enableChaos }} | |
| pyroscope_env: ci-automation-on-demand-chaos | |
| network: SIMULATED | |
| command: -run ^TestAutomationChaos$ ./chaos | |
| - name: reorg | |
| id: reorg | |
| suite: reorg | |
| nodes: 5 | |
| os: ubuntu-latest | |
| enabled: ${{ inputs.enableReorg }} | |
| pyroscope_env: ci-automation-on-demand-reorg | |
| network: SIMULATED_NONDEV | |
| command: -run ^TestAutomationReorg$ ./reorg | |
| - name: upgrade 2.0 | |
| id: upgrade-2.0 | |
| type: upgrade | |
| suite: smoke | |
| nodes: 1 | |
| os: ubuntu22.04-8cores-32GB | |
| enabled: true | |
| pyroscope_env: ci-automation-on-demand-upgrade | |
| network: SIMULATED | |
| command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke | |
| - name: upgrade 2.1 | |
| id: upgrade-2.1 | |
| type: upgrade | |
| suite: smoke | |
| nodes: 5 | |
| os: ubuntu22.04-8cores-32GB | |
| enabled: true | |
| pyroscope_env: ci-automation-on-demand-upgrade | |
| network: SIMULATED | |
| command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke | |
| - name: upgrade 2.2 | |
| id: upgrade-2.2 | |
| type: upgrade | |
| suite: smoke | |
| nodes: 5 | |
| os: ubuntu22.04-8cores-32GB | |
| enabled: true | |
| pyroscope_env: ci-automation-on-demand-upgrade | |
| network: SIMULATED | |
| command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke | |
| runs-on: ${{ matrix.tests.os }} | |
| name: Automation On Demand ${{ matrix.tests.name }} Test | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
| with: | |
| ref: ${{ github.head_ref || github.ref_name }} | |
| - name: Determine build to use | |
| id: determine-build | |
| shell: bash | |
| run: | | |
| if [[ "${{ inputs.chainlinkImage }}" == "" ]]; then | |
| echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT | |
| echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT | |
| echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT | |
| echo "upgrade_image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT | |
| else | |
| READ_CL_IMAGE=$(jq -r '.inputs.chainlinkImage' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$READ_CL_IMAGE | |
| echo "image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT | |
| echo "version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT | |
| echo "upgrade_version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT | |
| echo "upgrade_image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT | |
| fi | |
| if [[ "${{ matrix.tests.type }}" == "upgrade" ]]; then | |
| READ_CL_UPGR_IMAGE=$(jq -r '.inputs.chainlinkImageUpdate' $GITHUB_EVENT_PATH) | |
| echo ::add-mask::$READ_CL_UPGR_IMAGE | |
| echo "image=$READ_CL_UPGR_IMAGE" >>$GITHUB_OUTPUT | |
| echo "version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT | |
| fi | |
| - name: Setup GAP for Grafana | |
| uses: smartcontractkit/.github/actions/setup-gap@main | |
| with: | |
| # aws inputs | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} | |
| api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} | |
| # other inputs | |
| duplicate-authorization-header: "true" | |
| # metrics inputs | |
| metrics-job-name: "grafana" | |
| gc-host: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| gc-basic-auth: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} | |
| gc-org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| - name: Prepare Base64 TOML config | |
| env: | |
| SELECTED_NETWORKS: ${{ matrix.tests.network }} | |
| OLD_IMAGE: ${{ steps.determine-build.outputs.image }} | |
| OLD_VERSION: ${{ steps.determine-build.outputs.version }} | |
| UPGRADE_VERSION: ${{ steps.determine-build.outputs.upgrade_version }} | |
| UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }} | |
| PYROSCOPE_SERVER: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 | |
| PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} | |
| PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} | |
| RUN_ID: ${{ github.run_id }} | |
| TEST_LOG_COLLECT: "true" | |
| LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push | |
| LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
| GRAFANA_URL: "http://localhost:8080/primary" | |
| GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
| $GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} | |
| run: | | |
| echo ::add-mask::$UPGRADE_IMAGE | |
| echo ::add-mask::$OLD_IMAGE | |
| # load reusable functions | |
| source ./.github/scripts/functions.sh | |
| selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS") | |
| if [ -n "$PYROSCOPE_SERVER" ]; then | |
| pyroscope_enabled=true | |
| else | |
| pyroscope_enabled=false | |
| fi | |
| if [ -n "$TEST_LOG_COLLECT" ]; then | |
| test_log_collect=true | |
| else | |
| test_log_collect=false | |
| fi | |
| log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS") | |
| grafana_bearer_token="" | |
| if [ -n "$GRAFANA_BEARER_TOKEN" ]; then | |
| grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\"" | |
| fi | |
| cat << EOF > config.toml | |
| [Network] | |
| selected_networks=$selected_networks | |
| [ChainlinkImage] | |
| image="$OLD_IMAGE" | |
| version="$OLD_VERSION" | |
| [ChainlinkUpgradeImage] | |
| image="$UPGRADE_IMAGE" | |
| version="$UPGRADE_VERSION" | |
| [Logging] | |
| test_log_collect=$test_log_collect | |
| run_id="$RUN_ID" | |
| [Logging.LogStream] | |
| log_targets=$log_targets | |
| [Logging.Loki] | |
| tenant_id="$LOKI_TENANT_ID" | |
| endpoint="$LOKI_ENDPOINT" | |
| basic_auth_secret="$LOKI_BASIC_AUTH" | |
| [Logging.Grafana] | |
| base_url="$GRAFANA_URL" | |
| dashboard_url="$GRAFANA_DASHBOARD_URL" | |
| $grafana_bearer_token | |
| [Pyroscope] | |
| enabled=$pyroscope_enabled | |
| server_url="$PYROSCOPE_SERVER" | |
| environment="$PYROSCOPE_ENVIRONMENT" | |
| key_secret="$PYROSCOPE_KEY" | |
| EOF | |
| BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0) | |
| echo ::add-mask::$BASE64_CONFIG_OVERRIDE | |
| echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV | |
| - name: Run Tests | |
| uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@b49a9d04744b0237908831730f8553f26d73a94b # v2.3.17 | |
| if: ${{ matrix.tests.enabled == true }} | |
| env: | |
| TEST_SUITE: ${{ matrix.tests.suite }} | |
| with: | |
| test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage | |
| test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
| cl_repo: ${{ steps.determine-build.outputs.image }} | |
| cl_image_tag: ${{ steps.determine-build.outputs.version }} | |
| aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
| artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs | |
| publish_check_name: Automation On Demand Results ${{ matrix.tests.name }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| go_mod_path: ./integration-tests/go.mod | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
| QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
| - name: Upload test log | |
| uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
| if: failure() | |
| with: | |
| name: test-log-${{ matrix.tests.name }} | |
| path: /tmp/gotest.log | |
| retention-days: 7 | |
| continue-on-error: true | |
| - name: Collect Metrics | |
| if: always() | |
| id: collect-gha-metrics | |
| uses: smartcontractkit/push-gha-metrics-action@dea9b546553cb4ca936607c2267a09c004e4ab3f # v3.0.0 | |
| with: | |
| id: automation-on-demand-tests-${{ matrix.tests.id }} | |
| org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
| basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
| hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
| this-job-name: Automation On Demand ${{ matrix.tests.name }} Test | |
| test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | |
| continue-on-error: true |