Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[TT-524] Separate Live Testnet Tests #11196

Merged
merged 5 commits into from
Nov 7, 2023
Merged

[TT-524] Separate Live Testnet Tests #11196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7d71ec0
Separate Live Testnet Tests
kalverra Nov 6, 2023
51b83b7
\n
kalverra Nov 6, 2023
bb40efc
Remove typo
kalverra Nov 6, 2023
3c5ddc6
Extrapolates building Chainlink image
kalverra Nov 7, 2023
1a2875c
Merge branch 'develop' of github.com:smartcontractkit/chainlink into …
kalverra Nov 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions .github/actions/build-chainlink-image/action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Build Chainlink Image
description: A composite action that allows building and publishing the Chainlink image for integration testing

inputs:
tag_suffix:
description: The suffix to append to the image tag (usually blank or "-plugins")
default: ""
dockerfile:
description: The path to the Dockerfile to use (usually core/chainlink.Dockerfile or plugins/chainlink.Dockerfile)
default: core/chainlink.Dockerfile
git_commit_sha:
description: The git commit sha to use for the image tag
default: ${{ github.sha }}
GRAFANA_CLOUD_BASIC_AUTH:
description: "grafana cloud basic auth"
GRAFANA_CLOUD_HOST:
description: "grafana cloud hostname"
AWS_REGION:
description: "AWS region to use for ECR"
AWS_ROLE_TO_ASSUME:
description: "AWS role to assume for ECR"

runs:
using: composite
steps:
- name: Check if image exists
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
with:
repository: chainlink
tag: ${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
AWS_REGION: ${{ inputs.AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
- name: Build Image
if: steps.check-image.outputs.exists == 'false' && needs.changes.outputs.src == 'true'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
with:
cl_repo: smartcontractkit/chainlink
cl_ref: ${{ inputs.git_commit_sha }}
cl_dockerfile: ${{ inputs.dockerfile }}
push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ inputs.git_commit_sha }}${{ inputs.tag_suffix }}
QA_AWS_REGION: ${{ inputs.AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ inputs.AWS_ROLE_TO_ASSUME }}
- name: Print Chainlink Image Built
shell: sh
run: |
echo "### Chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY
echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY
243 changes: 9 additions & 234 deletions .github/workflows/integration-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,10 @@ name: Integration Tests
on:
merge_group:
pull_request:
schedule:
- cron: "0 0 * * *"
# - cron: "0 * * * *" # DEBUG: Run every hour to nail down flakes
push:
tags:
- "*"
workflow_dispatch:
inputs:
simulatedNetwork:
description: "Run Simulated Network Tests"
required: false
type: boolean
default: true
liveNetwork:
description: "Run Live Network Tests"
required: false
type: boolean

# Only run 1 of this workflow at a time per PR
concurrency:
Expand Down Expand Up @@ -62,9 +49,9 @@ jobs:
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: Check Paths That Require Tests To Run
continue-on-error: true

outputs:
src: ${{ steps.changes.outputs.src }}

build-chainlink:
environment: integration
permissions:
Expand Down Expand Up @@ -96,30 +83,16 @@ jobs:
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
- name: Check if image exists
if: needs.changes.outputs.src == 'true'
id: check-image
uses: smartcontractkit/chainlink-github-actions/docker/image-exists@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
with:
repository: chainlink
tag: ${{ github.sha }}${{ matrix.image.tag-suffix }}
- name: Build Chainlink Image
uses: ./.github/actions/build-chainlink-image
with:
tag_suffix: ${{ matrix.image.tag-suffix }}
dockerfile: ${{ matrix.image.dockerfile }}
git_commit_sha: ${{ github.sha }}
GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }}
AWS_REGION: ${{ secrets.QA_AWS_REGION }}
AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Build Image
if: steps.check-image.outputs.exists == 'false' && needs.changes.outputs.src == 'true'
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@eccde1970eca69f079d3efb3409938a72ade8497 # v2.2.13
with:
cl_repo: smartcontractkit/chainlink
cl_ref: ${{ github.sha }}
cl_dockerfile: ${{ matrix.image.dockerfile }}
push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
- name: Print Chainlink Image Built
if: needs.changes.outputs.src == 'true'
run: |
echo "### Chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY
echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY

build-test-image:
if: startsWith(github.ref, 'refs/tags/') || github.event_name == 'schedule' || contains(join(github.event.pull_request.labels.*.name, ' '), 'build-test-image')
Expand Down Expand Up @@ -896,201 +869,3 @@ jobs:
matrix-aggregator-status: ${{ needs.solana-smoke-tests-matrix.result }}
continue-on-error: true
### End Solana Section

### Start Live Testnet Section

testnet-smoke-tests-matrix:
if: ${{ github.event_name == 'schedule' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || (github.event_name == 'workflow_dispatch' && inputs.liveNetwork) }} ## Only run live tests on new tags, schedule, or on manual request
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
needs: [build-chainlink]
env:
SELECTED_NETWORKS: ${{ matrix.testnet }}
CHAINLINK_COMMIT_SHA: ${{ github.sha }}
CHAINLINK_ENV_USER: ${{ github.actor }}
TEST_LOG_LEVEL: debug
EVM_KEYS: ${{ secrets.QA_EVM_KEYS }}

OPTIMISM_GOERLI_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_URLS }}
OPTIMISM_GOERLI_HTTP_URLS: ${{ secrets.QA_OPTIMISM_GOERLI_HTTP_URLS }}

ARBITRUM_GOERLI_URLS: ${{ secrets.QA_ARBITRUM_GOERLI_URLS }}
ARBITRUM_GOERLI_HTTP_URLS: ${{ secrets.QA_ARBITRUM_GOERLI_HTTP_URLS }}
strategy:
fail-fast: false
matrix:
# NOTE: If changing this matrix, make sure to update the matrix in the testnet-smoke-tests-notify job to be the same
# otherwise reporting will be broken. Getting a single matrix for multiple jobs is a pain
# https://github.com/orgs/community/discussions/26284#discussioncomment-3251198
testnet: [OPTIMISM_GOERLI, ARBITRUM_GOERLI]
name: Live Testnet Smoke Tests ${{ matrix.testnet }}
runs-on: ubuntu-latest
steps:
- name: Checkout the repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
## Only run OCR smoke test for now
- name: Run Tests
uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15
env:
PYROSCOPE_SERVER: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
PYROSCOPE_ENVIRONMENT: ci-smoke-ocr-evm-${{ matrix.testnet }} # TODO: Only for OCR for now
PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
with:
test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=1 ./smoke/ocr_test.go 2>&1 | tee /tmp/gotest.log | gotestfmt
test_download_vendor_packages_command: cd ./integration-tests && go mod download
cl_repo: ${{ env.CHAINLINK_IMAGE }}
cl_image_tag: ${{ github.sha }}
aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
artifacts_location: ./integration-tests/smoke/logs
publish_check_name: ${{ matrix.testnet }} OCR Smoke Test Results
token: ${{ secrets.GITHUB_TOKEN }}
go_mod_path: ./integration-tests/go.mod
cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
cache_restore_only: "true"
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}

- name: Collect Metrics
if: always()
id: collect-gha-metrics
uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2
with:
basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
this-job-name: Live Testnet Smoke Tests ${{ matrix.testnet }}
test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
continue-on-error: true

testnet-smoke-tests-notify:
name: Live Testnet Start Slack Thread
if: ${{ always() && needs.testnet-smoke-tests-matrix.result != 'skipped' && needs.testnet-smoke-tests-matrix.result != 'cancelled' && github.event_name != 'workflow_dispatch' }}
environment: integration
outputs:
thread_ts: ${{ steps.slack.outputs.thread_ts }}
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
runs-on: ubuntu-latest
needs: testnet-smoke-tests-matrix
steps:
- name: Debug Result
run: echo ${{needs.testnet-smoke-tests-matrix.result}}
- name: Main Slack Notification
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
id: slack
with:
channel-id: ${{ secrets.QA_SLACK_CHANNEL }}
payload: |
{
"attachments": [
{
"color": "${{ needs.testnet-smoke-tests-matrix.result == 'success' && '#2E7D32' || '#C62828' }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "Live Smoke Test Results ${{ needs.testnet-smoke-tests-matrix.result == 'success' && ':white_check_mark:' || ':x:'}}",
"emoji": true
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "<${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|${{ github.ref_name }}> | <${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}|${{ github.sha }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Run>"
}
}
]
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

testnet-smoke-tests-results:
name: Post Live Testnet Smoke Test Results
if: ${{ always() && needs.testnet-smoke-tests-matrix.result != 'skipped' && needs.testnet-smoke-tests-matrix.result != 'cancelled' }}
environment: integration
permissions:
checks: write
pull-requests: write
id-token: write
contents: read
runs-on: ubuntu-latest
needs: testnet-smoke-tests-notify
strategy:
fail-fast: false
matrix:
# NOTE: If changing this matrix, make sure to update the matrix in the testnet-smoke-tests-matrix job to be the same
# otherwise reporting will be broken. Getting a single matrix for multiple jobs is a pain
# https://github.com/orgs/community/discussions/26284#discussioncomment-3251198
testnet: [OPTIMISM_GOERLI, ARBITRUM_GOERLI]
steps:
- name: Get Results
id: test-results
run: |
echo "Querying test results"

echo "status=$(curl \
-H "Authorization: Bearer ${{ github.token }}" \
'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
| jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet}}").steps[] | select(.name == "Run Tests").conclusion')" >> $GITHUB_OUTPUT

echo "status=$(curl \
-H "Authorization: Bearer ${{ github.token }}" \
'https://api.github.com/repos/${{github.repository}}/actions/runs/${{ github.run_id }}/jobs' \
| jq -r '.jobs[] | select(.name == "Live Testnet Smoke Tests ${{ matrix.testnet}}").steps[] | select(.name == "Run Tests").conclusion')"
echo "thread_ts=${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}"

- name: Test Details
uses: slackapi/slack-github-action@e28cf165c92ffef168d23c5c9000cffc8a25e117 # v1.24.0
with:
channel-id: ${{ secrets.QA_SLACK_CHANNEL }}
payload: |
{
"thread_ts": "${{ needs.testnet-smoke-tests-notify.outputs.thread_ts }}",
"attachments": [
{
"color": "${{ steps.test-results.outputs.status == 'success' && '#2E7D32' || '#C62828' }}",
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "${{ matrix.testnet }} Smoke Test Results ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}",
"emoji": true
}
},
{
"type": "divider"
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "OCR ${{ steps.test-results.outputs.status == 'success' && ':white_check_mark:' || ':x:'}}"
}
}
]
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

### End Live Testnet Section
Loading
Loading