diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml index b5b47382d72..be5944ba1a1 100644 --- a/.github/workflows/live-testnet-tests.yml +++ b/.github/workflows/live-testnet-tests.yml @@ -9,12 +9,34 @@ name: Live Testnet Tests on: - schedule: - - cron: "0 5 * * *" # Run every night at midnight EST - push: - tags: - - "*" + # Disable refular runs for now until we can fix some test client flakiness and improve stability + # schedule: + # - cron: "0 5 * * *" # Run every night at midnight EST + # push: + # tags: + # - "*" workflow_dispatch: + inputs: + slack_user_id: + description: "The Slack member ID to notify" + required: true + type: string + network: + description: "The network to run tests on" + required: true + type: choice + options: + - "All" + - "Sepolia" + - "Optimism Sepolia" + - "Arbitrum Sepolia" + - "Base Sepolia" + - "Polygon Mumbai" + - "Avalanche Fuji" + - "Fantom Testnet" + - "Celo Alfajores" + - "Linea Goerli" + - "BSC Testnet" env: CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink @@ -144,7 +166,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "${{ contains(join(needs.*.result, ','), 'failure') && 'Some tests failed, notifying <@U01Q4N37KFG>' || 'All Good!' }}" + "text": "Notifying <@${{ inputs.slack_user_id }}>" } }, { @@ -200,6 +222,7 @@ jobs: sepolia-smoke-tests: environment: integration + if: ${{ (github.event.inputs.network == 'All' || github.event.inputs.network == 'Sepolia') }} permissions: checks: write pull-requests: write @@ -272,6 +295,7 @@ jobs: bsc-testnet-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'BSC Testnet' }} permissions: checks: write pull-requests: write @@ -344,6 +368,7 @@ jobs: optimism-sepolia-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Optimism Sepolia' }} permissions: checks: write pull-requests: write @@ -416,6 +441,7 @@ jobs: arbitrum-sepolia-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Arbitrum Sepolia' }} permissions: checks: write pull-requests: write @@ -488,6 +514,7 @@ jobs: base-sepolia-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Base Sepolia' }} permissions: checks: write pull-requests: write @@ -556,6 +583,7 @@ jobs: polygon-mumbai-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Polygon Mumbai' }} permissions: checks: write pull-requests: write @@ -628,6 +656,7 @@ jobs: avalanche-fuji-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Avalanche Fuji' }} permissions: checks: write pull-requests: write @@ -700,6 +729,7 @@ jobs: fantom-testnet-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Fantom Testnet' }} permissions: checks: write pull-requests: write @@ -772,6 +802,7 @@ jobs: celo-alfajores-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Celo Alfajores' }} permissions: checks: write pull-requests: write @@ -909,6 +940,7 @@ jobs: linea-goerli-smoke-tests: environment: integration + if: ${{ github.event.inputs.network == 'All' || github.event.inputs.network == 'Linea Goerli' }} permissions: checks: write pull-requests: write diff --git a/.github/workflows/live-vrf-tests.yml b/.github/workflows/live-vrf-tests.yml new file mode 100644 index 00000000000..e5b3530820b --- /dev/null +++ b/.github/workflows/live-vrf-tests.yml @@ -0,0 +1,191 @@ +name: Live VRF Tests +on: + workflow_dispatch: + inputs: + slack_user_id: + description: "The Slack member ID to notify" + required: true + type: string + networks: + description: "Remove any networks you don't want to run tests on" + required: true + default: "SEPOLIA,OPTIMISM_SEPOLIA,ARBITRUM_SEPOLIA" + test: + description: "Choose test you want to run" + required: true + type: choice + options: + - "TestVRFBasic" + - "TestVRFv2Basic" + - "TestVRFv2Plus" + +env: + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink + INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com + MOD_CACHE_VERSION: 2 + CHAINLINK_NODE_FUNDING: .5 + PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} + LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} + LOKI_URL: ${{ secrets.LOKI_URL }} + LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} + LOGSTREAM_LOG_TARGETS: loki + GRAFANA_URL: ${{ vars.GRAFANA_URL }} + RUN_ID: ${{ github.run_id }} + + CHAINLINK_COMMIT_SHA: ${{ github.sha }} + CHAINLINK_ENV_USER: ${{ github.actor }} + TEST_LOG_LEVEL: debug + +jobs: + + # Build Test Dependencies + + build-chainlink: + environment: integration + permissions: + id-token: write + contents: read + name: Build Chainlink Image + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0 + with: + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Chainlink Image + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Build Chainlink Image + uses: ./.github/actions/build-chainlink-image + with: + tag_suffix: "" + dockerfile: core/chainlink.Dockerfile + git_commit_sha: ${{ github.sha }} + GRAFANA_CLOUD_BASIC_AUTH: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + GRAFANA_CLOUD_HOST: ${{ secrets.GRAFANA_CLOUD_HOST }} + AWS_REGION: ${{ secrets.QA_AWS_REGION }} + AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + + build-tests: + environment: integration + permissions: + id-token: write + contents: read + name: Build Tests Binary + runs-on: ubuntu-latest + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@0281b09807758be1dcc41651e44e62b353808c47 # v2.1.0 + with: + org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} + basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }} + this-job-name: Build Tests Binary + continue-on-error: true + - name: Checkout the repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + - name: Build Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-tests@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6 + with: + test_download_vendor_packages_command: cd ./integration-tests && go mod download + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + go_tags: embed + cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + binary_name: tests + + build-matrix: + environment: integration + permissions: + id-token: write + contents: read + name: Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.build-matrix.outputs.matrix }} + steps: + - id: build-matrix + run: | + NETWORKS="[\"${{ github.event.inputs.networks }}\"]" + NETWORKS="${NETWORKS//,/\",\"}" + echo "matrix=${NETWORKS}" + + # End Build Test Dependencies + + live-vrf-tests: + environment: integration + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + needs: [build-chainlink, build-tests, build-matrix] + strategy: + max-parallel: 1 + fail-fast: false + matrix: + network: ${{fromJson(needs.build-matrix.outputs.matrix)}} + name: VRF Tests on ${{ matrix.network }} + runs-on: ubuntu-latest + steps: + - name: Build Secrets Names + id: build-secrets-names + run: | + echo "HTTP_URLS_SECRET_NAME=QA_${{ matrix }}_HTTP_URLS" + echo "URLS_SECRET_NAME=QA_${{ matrix }}_URLS" + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - name: Prepare Base64 TOML override + uses: ./.github/actions/setup-create-base64-config-live-testnets + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + chainlinkImage: ${{ env.CHAINLINK_IMAGE }} + chainlinkVersion: ${{ github.sha }} + lokiEndpoint: ${{ secrets.LOKI_URL }} + lokiTenantId: ${{ vars.LOKI_TENANT_ID }} + lokiBasicAuth: ${{ secrets.LOKI_BASIC_AUTH }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} + grafanaUrl: ${{ vars.GRAFANA_URL }} + grafanaDashboardUrl: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" + network: ${{ matrix.network }} + httpEndpoints: ${{ secrets[env.HTTP_URLS_SECRET_NAME] }} + wsEndpoints: ${{ secrets[env.URLS_SECRET_NAME] }} + fundingKeys: ${{ secrets.QA_EVM_KEYS }} + - name: Download Tests Binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: tests + - name: Run Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests-binary@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6 + with: + test_command_to_run: ./tests -test.timeout 1h -test.count=1 -test.parallel=1 -test.run ${{ inputs.test }} + binary_name: tests + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ github.sha }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }} + dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }} + artifacts_location: ./logs + token: ${{ secrets.GITHUB_TOKEN }} + cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} + cache_restore_only: "true" + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@c67a09566412d153ff7640d99f96b43aa03abc04 # v2.3.6 + with: + test_directory: "./" \ No newline at end of file