Skip to content

Run Automation On Demand Tests #212

Run Automation On Demand Tests

Run Automation On Demand Tests #212

name: Automation On Demand Tests
on:
workflow_dispatch:
inputs:
chainlinkVersionUpdate:
description: Chainlink image version to upgrade to
required: false
type: string
chainlinkImageUpdate:
description: Chainlink image repo to upgrade to (Leave empty to build from head/ref)
options:
- public.ecr.aws/chainlink/chainlink
- QA_ECR
type: choice
chainlinkVersion:
description: Chainlink image version to use initially for upgrade test
default: latest
required: true
type: string
chainlinkImage:
description: Chainlink image repo to use initially for upgrade test
required: true
options:
- public.ecr.aws/chainlink/chainlink
- QA_ECR
type: choice
enableChaos:
description: Check to enable chaos tests
type: boolean
default: false
required: true
enableReorg:
description: Check to enable reorg tests
type: boolean
default: false
required: true
env:
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
jobs:
build-chainlink:
environment: integration
permissions:
id-token: write
contents: read
strategy:
matrix:
image:
- name: ""
dockerfile: core/chainlink.Dockerfile
tag-suffix: ""
- name: (plugins)
dockerfile: plugins/chainlink.Dockerfile
tag-suffix: -plugins
name: Build Chainlink Image ${{ matrix.image.name }}
runs-on: ubuntu22.04-16cores-64GB
steps:
- name: Collect Metrics
if: inputs.chainlinkImage == ''
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
with:
id: automation-on-demand-build-chainlink
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: Build Chainlink Image ${{ matrix.image.name }}
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ github.head_ref || github.ref_name }}
- name: Check if image exists
if: inputs.chainlinkImage == ''
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
with:
repository: chainlink
tag: ${{ github.sha }}${{ matrix.image.tag-suffix }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Build Image
if: steps.check-image.outputs.exists == 'false' && inputs.chainlinkImage == ''
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
with:
cl_repo: smartcontractkit/chainlink
cl_ref: ${{ github.sha }}
cl_dockerfile: ${{ matrix.image.dockerfile }}
push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Print Chainlink Image Built
if: inputs.chainlinkImage == ''
run: |
echo "### chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY
echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
build-test-image:
environment: integration
permissions:
id-token: write
contents: read
name: Build Test Image
runs-on: ubuntu22.04-16cores-64GB
steps:
- name: Collect Metrics
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
with:
id: automation-on-demand-build-test-image
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: Build Test Image
continue-on-error: true
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ github.head_ref || github.ref_name }}
- name: Build Test Image
if: inputs.enableChaos || inputs.enableReorg
uses: ./.github/actions/build-test-image
with:
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
automation-on-demand-tests:
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
needs: [build-chainlink, build-test-image]
env:
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_LOG_LEVEL: info
strategy:
fail-fast: false
matrix:
tests:
- name: chaos
id: chaos
suite: chaos
nodes: 15
os: ubuntu-latest
enabled: ${{ inputs.enableChaos }}
pyroscope_env: ci-automation-on-demand-chaos
network: SIMULATED
command: -run ^TestAutomationChaos$ ./chaos
- name: reorg 2.0
id: reorg-2.0
suite: reorg
nodes: 1
os: ubuntu-latest
enabled: ${{ inputs.enableReorg }}
pyroscope_env: ci-automation-on-demand-reorg
network: SIMULATED
command: -run ^TestAutomationReorg/registry_2_0 ./reorg
- name: reorg 2.1
id: reorg-2.1
suite: reorg
nodes: 2
os: ubuntu-latest
enabled: ${{ inputs.enableReorg }}
pyroscope_env: ci-automation-on-demand-reorg
network: SIMULATED
command: -run ^TestAutomationReorg/registry_2_1 ./reorg
- name: reorg 2.2
id: reorg-2.2
suite: reorg
nodes: 2
os: ubuntu-latest
enabled: ${{ inputs.enableReorg }}
pyroscope_env: ci-automation-on-demand-reorg
network: SIMULATED
command: -run ^TestAutomationReorg/registry_2_2 ./reorg
- name: reorg 2.3
id: reorg-2.3
suite: reorg
nodes: 2
os: ubuntu-latest
enabled: ${{ inputs.enableReorg }}
pyroscope_env: ci-automation-on-demand-reorg
network: SIMULATED
command: -run ^TestAutomationReorg/registry_2_3 ./reorg
- name: upgrade 2.0
id: upgrade-2.0
type: upgrade
suite: smoke
nodes: 1
os: ubuntu22.04-8cores-32GB
enabled: true
pyroscope_env: ci-automation-on-demand-upgrade
network: SIMULATED
command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke
- name: upgrade 2.1
id: upgrade-2.1
type: upgrade
suite: smoke
nodes: 5
os: ubuntu22.04-8cores-32GB
enabled: true
pyroscope_env: ci-automation-on-demand-upgrade
network: SIMULATED
command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke
- name: upgrade 2.2
id: upgrade-2.2
type: upgrade
suite: smoke
nodes: 5
os: ubuntu22.04-8cores-32GB
enabled: true
pyroscope_env: ci-automation-on-demand-upgrade
network: SIMULATED
command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke
runs-on: ${{ matrix.tests.os }}
name: Automation On Demand ${{ matrix.tests.name }} Test
steps:
- name: Checkout the repo
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
ref: ${{ github.head_ref || github.ref_name }}
- name: Determine build to use
id: determine-build
shell: bash
run: |
if [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then
echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT
else
echo "image=${{ inputs.chainlinkImage }}" >>$GITHUB_OUTPUT
fi
if [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then
echo "upgrade_image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT
else
echo "upgrade_image=${{ inputs.chainlinkImageUpdate }}" >>$GITHUB_OUTPUT
fi
if [[ -z "${{ inputs.chainlinkVersion }}" ]] && [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then
echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT
else
echo "version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT
fi
if [[ -z "${{ inputs.chainlinkVersionUpdate }}" ]] && [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then
echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT
else
echo "upgrade_version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT
fi
- name: Setup GAP for Grafana
uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected]
with:
# aws inputs
aws-region: ${{ secrets.AWS_REGION }}
aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
# other inputs
duplicate-authorization-header: "true"
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28
if: ${{ matrix.tests.enabled == true }}
with:
test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
test_download_vendor_packages_command: cd ./integration-tests && go mod download
test_suite: ${{ matrix.tests.suite }}
test_config_chainlink_version: ${{ steps.determine-build.outputs.version }}
test_config_chainlink_upgrade_version: ${{ steps.determine-build.outputs.upgrade_version }}
test_config_selected_networks: ${{ matrix.tests.network }}
test_config_logging_run_id: ${{ github.run_id }}
test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }}
test_config_test_log_collect: "true"
cl_repo: ${{ steps.determine-build.outputs.image }}
cl_image_tag: ${{ steps.determine-build.outputs.version }}
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs
publish_check_name: Automation On Demand Results ${{ matrix.tests.name }}
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
DEFAULT_CHAINLINK_IMAGE: ${{ steps.determine-build.outputs.image }}
DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }}
DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }}
- name: Upload test log
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
if: failure()
with:
name: test-log-${{ matrix.tests.name }}
path: /tmp/gotest.log
retention-days: 7
continue-on-error: true
- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
with:
id: automation-on-demand-tests-${{ matrix.tests.id }}
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
this-job-name: Automation On Demand ${{ matrix.tests.name }} Test
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true