Run Automation On Demand Tests #212
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Automation On Demand Tests | |
on: | |
workflow_dispatch: | |
inputs: | |
chainlinkVersionUpdate: | |
description: Chainlink image version to upgrade to | |
required: false | |
type: string | |
chainlinkImageUpdate: | |
description: Chainlink image repo to upgrade to (Leave empty to build from head/ref) | |
options: | |
- public.ecr.aws/chainlink/chainlink | |
- QA_ECR | |
type: choice | |
chainlinkVersion: | |
description: Chainlink image version to use initially for upgrade test | |
default: latest | |
required: true | |
type: string | |
chainlinkImage: | |
description: Chainlink image repo to use initially for upgrade test | |
required: true | |
options: | |
- public.ecr.aws/chainlink/chainlink | |
- QA_ECR | |
type: choice | |
enableChaos: | |
description: Check to enable chaos tests | |
type: boolean | |
default: false | |
required: true | |
enableReorg: | |
description: Check to enable reorg tests | |
type: boolean | |
default: false | |
required: true | |
env: | |
ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} | |
CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink | |
jobs: | |
build-chainlink: | |
environment: integration | |
permissions: | |
id-token: write | |
contents: read | |
strategy: | |
matrix: | |
image: | |
- name: "" | |
dockerfile: core/chainlink.Dockerfile | |
tag-suffix: "" | |
- name: (plugins) | |
dockerfile: plugins/chainlink.Dockerfile | |
tag-suffix: -plugins | |
name: Build Chainlink Image ${{ matrix.image.name }} | |
runs-on: ubuntu22.04-16cores-64GB | |
steps: | |
- name: Collect Metrics | |
if: inputs.chainlinkImage == '' | |
id: collect-gha-metrics | |
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
with: | |
id: automation-on-demand-build-chainlink | |
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
this-job-name: Build Chainlink Image ${{ matrix.image.name }} | |
continue-on-error: true | |
- name: Checkout the repo | |
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
with: | |
ref: ${{ github.head_ref || github.ref_name }} | |
- name: Check if image exists | |
if: inputs.chainlinkImage == '' | |
id: check-image | |
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 | |
with: | |
repository: chainlink | |
tag: ${{ github.sha }}${{ matrix.image.tag-suffix }} | |
AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
- name: Build Image | |
if: steps.check-image.outputs.exists == 'false' && inputs.chainlinkImage == '' | |
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 | |
with: | |
cl_repo: smartcontractkit/chainlink | |
cl_ref: ${{ github.sha }} | |
cl_dockerfile: ${{ matrix.image.dockerfile }} | |
push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }} | |
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
- name: Print Chainlink Image Built | |
if: inputs.chainlinkImage == '' | |
run: | | |
echo "### chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY | |
echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY | |
build-test-image: | |
environment: integration | |
permissions: | |
id-token: write | |
contents: read | |
name: Build Test Image | |
runs-on: ubuntu22.04-16cores-64GB | |
steps: | |
- name: Collect Metrics | |
id: collect-gha-metrics | |
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
with: | |
id: automation-on-demand-build-test-image | |
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
this-job-name: Build Test Image | |
continue-on-error: true | |
- name: Checkout the repo | |
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
with: | |
ref: ${{ github.head_ref || github.ref_name }} | |
- name: Build Test Image | |
if: inputs.enableChaos || inputs.enableReorg | |
uses: ./.github/actions/build-test-image | |
with: | |
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
automation-on-demand-tests: | |
environment: integration | |
permissions: | |
checks: write | |
pull-requests: write | |
id-token: write | |
contents: read | |
needs: [build-chainlink, build-test-image] | |
env: | |
CHAINLINK_COMMIT_SHA: ${{ github.sha }} | |
CHAINLINK_ENV_USER: ${{ github.actor }} | |
TEST_LOG_LEVEL: info | |
strategy: | |
fail-fast: false | |
matrix: | |
tests: | |
- name: chaos | |
id: chaos | |
suite: chaos | |
nodes: 15 | |
os: ubuntu-latest | |
enabled: ${{ inputs.enableChaos }} | |
pyroscope_env: ci-automation-on-demand-chaos | |
network: SIMULATED | |
command: -run ^TestAutomationChaos$ ./chaos | |
- name: reorg 2.0 | |
id: reorg-2.0 | |
suite: reorg | |
nodes: 1 | |
os: ubuntu-latest | |
enabled: ${{ inputs.enableReorg }} | |
pyroscope_env: ci-automation-on-demand-reorg | |
network: SIMULATED | |
command: -run ^TestAutomationReorg/registry_2_0 ./reorg | |
- name: reorg 2.1 | |
id: reorg-2.1 | |
suite: reorg | |
nodes: 2 | |
os: ubuntu-latest | |
enabled: ${{ inputs.enableReorg }} | |
pyroscope_env: ci-automation-on-demand-reorg | |
network: SIMULATED | |
command: -run ^TestAutomationReorg/registry_2_1 ./reorg | |
- name: reorg 2.2 | |
id: reorg-2.2 | |
suite: reorg | |
nodes: 2 | |
os: ubuntu-latest | |
enabled: ${{ inputs.enableReorg }} | |
pyroscope_env: ci-automation-on-demand-reorg | |
network: SIMULATED | |
command: -run ^TestAutomationReorg/registry_2_2 ./reorg | |
- name: reorg 2.3 | |
id: reorg-2.3 | |
suite: reorg | |
nodes: 2 | |
os: ubuntu-latest | |
enabled: ${{ inputs.enableReorg }} | |
pyroscope_env: ci-automation-on-demand-reorg | |
network: SIMULATED | |
command: -run ^TestAutomationReorg/registry_2_3 ./reorg | |
- name: upgrade 2.0 | |
id: upgrade-2.0 | |
type: upgrade | |
suite: smoke | |
nodes: 1 | |
os: ubuntu22.04-8cores-32GB | |
enabled: true | |
pyroscope_env: ci-automation-on-demand-upgrade | |
network: SIMULATED | |
command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke | |
- name: upgrade 2.1 | |
id: upgrade-2.1 | |
type: upgrade | |
suite: smoke | |
nodes: 5 | |
os: ubuntu22.04-8cores-32GB | |
enabled: true | |
pyroscope_env: ci-automation-on-demand-upgrade | |
network: SIMULATED | |
command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke | |
- name: upgrade 2.2 | |
id: upgrade-2.2 | |
type: upgrade | |
suite: smoke | |
nodes: 5 | |
os: ubuntu22.04-8cores-32GB | |
enabled: true | |
pyroscope_env: ci-automation-on-demand-upgrade | |
network: SIMULATED | |
command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke | |
runs-on: ${{ matrix.tests.os }} | |
name: Automation On Demand ${{ matrix.tests.name }} Test | |
steps: | |
- name: Checkout the repo | |
uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 | |
with: | |
ref: ${{ github.head_ref || github.ref_name }} | |
- name: Determine build to use | |
id: determine-build | |
shell: bash | |
run: | | |
if [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then | |
echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT | |
else | |
echo "image=${{ inputs.chainlinkImage }}" >>$GITHUB_OUTPUT | |
fi | |
if [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then | |
echo "upgrade_image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT | |
else | |
echo "upgrade_image=${{ inputs.chainlinkImageUpdate }}" >>$GITHUB_OUTPUT | |
fi | |
if [[ -z "${{ inputs.chainlinkVersion }}" ]] && [[ "${{ inputs.chainlinkImage }}" == "QA_ECR" ]]; then | |
echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT | |
else | |
echo "version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT | |
fi | |
if [[ -z "${{ inputs.chainlinkVersionUpdate }}" ]] && [[ "${{ inputs.chainlinkImageUpdate }}" == "QA_ECR" ]]; then | |
echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT | |
else | |
echo "upgrade_version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT | |
fi | |
- name: Setup GAP for Grafana | |
uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # [email protected] | |
with: | |
# aws inputs | |
aws-region: ${{ secrets.AWS_REGION }} | |
aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} | |
api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} | |
# other inputs | |
duplicate-authorization-header: "true" | |
- name: Run Tests | |
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 | |
if: ${{ matrix.tests.enabled == true }} | |
with: | |
test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }} | |
test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs | |
test_download_vendor_packages_command: cd ./integration-tests && go mod download | |
test_suite: ${{ matrix.tests.suite }} | |
test_config_chainlink_version: ${{ steps.determine-build.outputs.version }} | |
test_config_chainlink_upgrade_version: ${{ steps.determine-build.outputs.upgrade_version }} | |
test_config_selected_networks: ${{ matrix.tests.network }} | |
test_config_logging_run_id: ${{ github.run_id }} | |
test_config_logstream_log_targets: ${{ vars.LOGSTREAM_LOG_TARGETS }} | |
test_config_test_log_collect: "true" | |
cl_repo: ${{ steps.determine-build.outputs.image }} | |
cl_image_tag: ${{ steps.determine-build.outputs.version }} | |
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} | |
artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs | |
publish_check_name: Automation On Demand Results ${{ matrix.tests.name }} | |
token: ${{ secrets.GITHUB_TOKEN }} | |
go_mod_path: ./integration-tests/go.mod | |
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} | |
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} | |
DEFAULT_CHAINLINK_IMAGE: ${{ steps.determine-build.outputs.image }} | |
DEFAULT_CHAINLINK_UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }} | |
DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push | |
DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" | |
DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" | |
DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} | |
DEFAULT_PYROSCOPE_SERVER_URL: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 | |
DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} | |
DEFAULT_PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }} | |
DEFAULT_PYROSCOPE_ENABLED: ${{ matrix.tests.pyroscope_env == '' || !startsWith(github.ref, 'refs/tags/') && 'false' || 'true' }} | |
- name: Upload test log | |
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 | |
if: failure() | |
with: | |
name: test-log-${{ matrix.tests.name }} | |
path: /tmp/gotest.log | |
retention-days: 7 | |
continue-on-error: true | |
- name: Collect Metrics | |
if: always() | |
id: collect-gha-metrics | |
uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 | |
with: | |
id: automation-on-demand-tests-${{ matrix.tests.id }} | |
org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} | |
basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} | |
hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} | |
this-job-name: Automation On Demand ${{ matrix.tests.name }} Test | |
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' | |
continue-on-error: true |