diff --git a/.github/workflows/generic-test-runner.yml b/.github/workflows/generic-test-runner.yml deleted file mode 100644 index dc194ab5bec..00000000000 --- a/.github/workflows/generic-test-runner.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: Generic Test Runner -on: - workflow_dispatch: - inputs: - network: - description: 'Network to run tests against' - required: true - default: 'SIMULATED' - wsURL: - description: 'Chain WS URL (Skip with SIMULATED)' - required: false - httpURL: - description: 'Chain HTTP URL (Skip with SIMULATED)' - required: false - fundingKey: - description: 'Private key to fund test (Skip with SIMULATED)' - required: false - directory: - description: 'Directory to run tests from' - required: true - default: 'smoke' - test: - description: 'Test to run' - required: true - default: 'OCRBasic' - testInputs: - description: 'Custom test inputs' - required: false - default: '' -env: - ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }} - CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink - -jobs: - build-chainlink: - environment: integration - permissions: - id-token: write - contents: read - strategy: - matrix: - image: - - name: "" - dockerfile: core/chainlink.Dockerfile - tag-suffix: "" - - name: (plugins) - dockerfile: plugins/chainlink.Dockerfile - tag-suffix: -plugins - name: Build Chainlink Image ${{ matrix.image.name }} - runs-on: ubuntu20.04-16cores-64GB - steps: - - name: Checkout the repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - name: Check if image exists - id: check-image - uses: smartcontractkit/chainlink-github-actions/docker/image-exists@00c6214deb10a3f374c6d3430c32c5202015d463 # v2.2.12 - with: - repository: chainlink - tag: ${{ github.sha }}${{ matrix.image.tag-suffix }} - AWS_REGION: ${{ secrets.QA_AWS_REGION }} - AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Build Image - if: steps.check-image.outputs.exists == 'false' - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@00c6214deb10a3f374c6d3430c32c5202015d463 # v2.2.12 - with: - cl_repo: smartcontractkit/chainlink - cl_ref: ${{ github.sha }} - cl_dockerfile: ${{ matrix.image.dockerfile }} - push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - - name: Print Chainlink Image Built - run: | - echo "### chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY - echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY - - build-test-image: - environment: integration - permissions: - id-token: write - contents: read - name: Build Test Image - runs-on: ubuntu20.04-16cores-64GB - env: - TEST_SUITE: ${{ github.event.inputs.directory }} - steps: - - name: Checkout the repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - name: Build Test Image - uses: ./.github/actions/build-test-image - with: - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - - test: - runs-on: ubuntu-latest - needs: [build-chainlink, build-test-image] - name: Run Test - environment: integration - permissions: - id-token: write - contents: read - env: - CHAINLINK_ENV_USER: ${{ github.actor }} - CHAINLINK_COMMIT_SHA: ${{ github.sha }} - TEST_SUITE: ${{ github.event.inputs.directory }} - SELECTED_NETWORKS: ${{ github.event.inputs.network }} - TEST_LOG_LEVEL: debug - steps: - - name: Mask Inputs - run: | - EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH) - EVM_HTTP_URLS=$(jq -r '.inputs.httpURL' $GITHUB_EVENT_PATH) - EVM_KEYS=$(jq -r '.inputs.fundingKey' $GITHUB_EVENT_PATH) - - echo ::add-mask::$EVM_URLS - echo ::add-mask::$EVM_HTTP_URLS - echo ::add-mask::$EVM_KEYS - - echo EVM_URLS=$EVM_URLS >> $GITHUB_ENV - echo EVM_HTTP_URLS=$EVM_HTTP_URLS >> $GITHUB_ENV - echo EVM_KEYS=$EVM_KEYS >> $GITHUB_ENV - - name: Debug Input - run: echo ${{ github.event.inputs.testInputs }} - - name: Checkout the repo - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@00c6214deb10a3f374c6d3430c32c5202015d463 # v2.2.12 - with: - test_command_to_run: make test_need_operator_assets && cd ./integration-tests && go test -timeout 1h -count=1 ./${{ github.event.inputs.directory }} -run ${{ github.event.inputs.test }} -v -args ${{ github.event.inputs.test-inputs }} - test_download_vendor_packages_command: cd ./integration-tests && go mod download - cl_repo: ${{ env.CHAINLINK_IMAGE }} - cl_image_tag: ${{ github.sha }} - artifacts_location: ./integration-tests/smoke/logs - publish_check_name: Generic Test Run - token: ${{ secrets.GITHUB_TOKEN }} - go_mod_path: ./integration-tests/go.mod - QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} - QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} - QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 110c119df75..287bb939635 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -78,30 +78,6 @@ run: # Need to set network first in case it's unset. Doesn't matter for the runn ## All commands will use 16 threads to run tests in parallel. To change this, use -test.parallel n # Smoke -.PHONY: test_smoke_raw # Run smoke tests without any gotestfmt or default args -test_smoke_raw: - go test $(args) ./smoke - -.PHONY: test_smoke -test_smoke: install_gotestfmt ## Run all smoke tests - TEST_LOG_LEVEL="disabled" \ - go test -timeout 24h -count=1 -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt - -.PHONY: test_smoke_simulated -test_smoke_simulated: install_gotestfmt ## Run all smoke tests on simulated blockchain - TEST_LOG_LEVEL="disabled" \ - SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \ - go test -timeout 1h -count=1 -json $(args) ./smoke 2>&1 | tee /tmp/gotest.log | gotestfmt - -.PHONY: test_smoke_verbose -test_smoke_verbose: ## Run all smoke tests with verbose logging - go test -timeout 24h -count=1 -v $(args) ./smoke - -.PHONY: test_smoke_simulated_verbose -test_smoke_simulated_verbose: ## Run all smoke tests with verbose logging - SELECTED_NETWORKS="SIMULATED,SIMULATED_1,SIMULATED_2" \ - go test -timeout 24h -count=1 -v $(args) ./smoke - .PHONY: test_smoke_product test_smoke_product: ## Run smoke tests for specific product ex: make test_smoke_product product="cron" args="--focus @cron -p" ARGS="$(args)" PRODUCT=$(product) ./scripts/run_product_tests diff --git a/integration-tests/README.md b/integration-tests/README.md index 3a2a59e0fbf..b84a6a2e379 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -2,205 +2,26 @@ Here lives the integration tests for chainlink, utilizing our [chainlink-testing-framework](https://github.com/smartcontractkit/chainlink-testing-framework). -## Full Setup +## NOTE: Move to Testcontainers -Prerequisites to run the tests from your local machine. Best for debugging and developing new tests, or checking changes to the framework. This can be a bit complex however, so if you just want to run the tests, see the [Just Run](#just-run) section. +If you have previously run these smoke tests using GitHub Actions or some sort of Kubernetes setup, that method is no longer necessary. We have moved the majority of our tests to utilize plain Docker containers (with the help of [Testcontainers](https://golang.testcontainers.org/)). This should make tests faster, more stable, and enable you to run them on your local machine without much hassle. More p -
- Details +## Requirements -### Install Dependencies +1. [Go](https://go.dev/) +2. [Docker](https://www.docker.com/) +3. You'll probably want to [increase the resources available to Docker](https://stackoverflow.com/questions/44533319/how-to-assign-more-memory-to-docker-container) as most tests require quite a few containers (e.g. OCR requires 6 Chainlink nodes, 6 databases, a simulated blockchain, and a mock server). -Run the below command to install all dependencies. +## Configure -```sh -make install_qa_tools -``` +See the [example.env](./example.env) file for environment variables you can set to configure things like network settings, Chainlink version, and log level. Remember to use `source .env` to activate your settings. -Or you can choose to do it manually. +## Run -
- Install Go +`go test ./smoke/_test.go` - [Install](https://go.dev/doc/install) -
+It's generally recommended to run only one test at a time on a local machine as it needs a lot of docker containers and can peg your resources otherwise. You will see docker containers spin up on your machine for each component of the test where you can inspect logs. -
- Install NodeJS +## Running Soak, Performance, Benchmark, and Chaos Tests - [Install](https://nodejs.org/en/download/) -
- -
- Install Helm Charts - - [Install Helm](https://helm.sh/docs/intro/install/#through-package-managers) if you don't already have it. Then add necessary charts with the below commands. - - ```sh - helm repo add chainlink-qa https://raw.githubusercontent.com/smartcontractkit/qa-charts/gh-pages/ - helm repo add bitnami https://charts.bitnami.com/bitnami - helm repo update - ``` - -
- -## Connect to a Kubernetes Cluster - -Integration tests require a connection to an actively running kubernetes cluster. [Minikube](https://minikube.sigs.k8s.io/docs/start/) -can work fine for some tests, but in order to run more rigorous tests, or to run with any parallelism, you'll need to either -increase minikube's resources significantly, or get a more substantial cluster. -This is necessary to deploy ephemeral testing environments, which include external adapters, chainlink nodes and their DBs, -as well as some simulated blockchains, all depending on the types of tests and networks being used. - -### Setup Kubernetes Cluster using k3d - -[k3d](https://k3d.io/) is a lightweight wrapper to run k3s (a lightweight kubernetes distribution) in docker. It's a great way to run a local kubernetes cluster for testing. -To create a new cluster you can run: - -```sh -k3d cluster create test-k8s --registry-create test-k8s-registry:0.0.0.0:5000 -``` - -This will create a cluster with a local registry running on port 5000. You can then use the registry to push images to and pull images from. - -To build and push chainlink image to the registry you can run: - -```sh -make build_push_docker_image -```` - -To stop the cluster you can run: - -```sh -k3d cluster stop test-k8s -``` - -To start an existing cluster you can run: - -```sh -k3d cluster start test-k8s -``` - -## Configure Environment - -See the [example.env](./example.env) file and use it as a template for your own `.env` file. This allows you to configure general settings like what name to associate with your tests, and which Chainlink version to use when running them. - -You can also specify `EVM_KEYS` and `EVM_URLS` for running on live chains, or use specific identifiers as shown in the [example.env](./example.env) file. - -Other `EVM_*` variables are retrieved when running with the `@general` tag, and is helpful for doing quick sanity checks on new chains or when tweaking variables. - -**The tests will not automatically load your .env file. Remember to run `source .env` for changes to take effect.** - -## How to Run - -Most of the time, you'll want to run tests on a simulated chain, for the purposes of speed and cost. - -### Smoke - -Run all smoke tests with the below command. Will use your `SELECTED_NETWORKS` env var for which network to run on. - -```sh -make test_smoke # Run all smoke tests on the chosen SELECTED_NETWORKS -SELECTED_NETWORKS="GOERLI" make test_smoke # Run all smoke tests on GOERLI network -make test_smoke_simulated # Run all smoke tests on a simulated network -``` - -Run all smoke tests in parallel, only using simulated blockchains. *Note: As of now, you can only run tests in parallel on simulated chains, not on live ones. Running on parallel tests on live chains will give errors* - -```sh -make test_smoke_simulated args="-test.parallel=" -``` - -You can also run specific tests and debug tests in vscode by setting up your .vscode/settings.json with this information. Just replace all the "" with your information before running a test. - -```json -{ - "makefile.extensionOutputFolder": "./.vscode", - "go.testEnvVars": { - "LOG_LEVEL": "debug", - "SELECTED_NETWORKS": "SIMULATED,SIMULATED_1,SIMULATED_2", - "CHAINLINK_IMAGE":".dkr.ecr.us-west-2.amazonaws.com/chainlink", - "CHAINLINK_VERSION":"develop", - "CHAINLINK_ENV_USER":"", - "TEST_LOG_LEVEL":"debug", - "AWS_ACCESS_KEY_ID":"", - "AWS_SECRET_ACCESS_KEY":"", - "AWS_SESSION_TOKEN":"" - }, - "go.testTimeout": "900s" -} -``` - -You can also run your tests inside of kubernetes instead of from locally to reduce local resource usage and the number of ports that get forwarded to the cluster. This is not recommended for normal developement since building and pushing the image can be time heavy depending on your internet upload speeds. To do this you will want to either pull down an already built chainlink-tests image or build one yourself. To build and push one yourself you can run: - -```sh -make build_test_image tag= base_tag=latest suite="smoke soak chaos reorg migration performance" push=true -``` - -Once that is done building you can add this to your go.testEnvVars in .vscode/settings.json with the correct account number and tag filled out. - -```json - "TEST_SUITE": "smoke", - "TEST_ARGS": "-test.timeout 30m", - "ENV_JOB_IMAGE":".dkr.ecr.us-west-2.amazonaws.com/chainlink-env-tests:", -``` - -Once that is done you can run/debug your test using the vscode test view just like normal. - -### Soak - -Currently we have 2 soak tests, both can be triggered using make commands. - -```sh -make test_soak_ocr -make test_soak_keeper -``` - -Soak tests will pull all their network information from the env vars that you can set in the `.env` file. *Reminder to run `source .env` for changes to take effect.* - -To configure specific parameters of how the soak tests run (e.g. test length, number of contracts), adjust the values in your `.env` file, you can use `example.env` as reference - - -#### Running with custom image -On each PR navigate to the `integration-tests` job, here you will find the images for both chainlink-tests and core. In your env file you need to replace: - -`ENV_JOB_IMAGE="image-location/chainlink-tests:"` - -`CHAINLINK_IMAGE="public.ecr.aws/chainlink/chainlink"` - -`export CHAINLINK_VERSION=""` - -After all the env vars are exported, run the tests. This will kick off a remote runner that will be in charge of running the tests. Locally the test should pass quickly and a namespace will be displayed in the output e.g -`INF Creating new namespace Namespace=soak-ocr-goerli-testnet-957b2` - -#### Logs and monitoring -- Pod logs: `kubectl logs -n soak-ocr-goerli-testnet-957b2 -c node -f chainlink-0-1` -- Remote runner logs: `kubectl logs -n soak-ocr-goerli-testnet-957b2 -f remote-runner-cs2as` -- Navigate to Grafana chainlink testing insights for all logs - -### Performance - -Currently, all performance tests are only run on simulated blockchains. - -```sh -make test_perf -``` - -## Common Issues - -- When upgrading to a new version, it's possible the helm charts have changed. There are a myriad of errors that can result from this, so it's best to just try running `helm repo update` when encountering an error you're unsure of. -- Docker failing to pull image, make sure you are referencing the correct ECR repo in AWS since develop images are not pushed to the public one. - - If tests hang for some time this is usually the case, so make sure to check the logs each time tests are failing to start - -
- - -## Just Run - -If you're making changes to chainlink code, or just want to run some tests without a complex setup, follow the below steps. - -1. [Install Go](https://go.dev/doc/install) -2. [Install GitHub CLI](https://cli.github.com/) -3. Authenticate with GitHub CLI: `gh auth login` -4. `make run` -5. Follow the setup wizard and watch your tests run in the GitHub Action. \ No newline at end of file +These tests remain bound to a Kubernetes run environment, and require more complex setup and running instructions not documented here. We endeavor to make these easier to run and configure, but for the time being please seek a member of the QA/Test Tooling team if you want to run these. diff --git a/integration-tests/example.env b/integration-tests/example.env index 8570fa86e1e..428f76b914e 100644 --- a/integration-tests/example.env +++ b/integration-tests/example.env @@ -4,7 +4,7 @@ ########## General Test Settings ########## export KEEP_ENVIRONMENTS="Never" # Always | OnFail | Never export CHAINLINK_IMAGE="public.ecr.aws/chainlink/chainlink" # Image repo to pull the Chainlink image from -export CHAINLINK_VERSION="1.13.0" # Version of the Chainlink image to pull +export CHAINLINK_VERSION="2.4.0" # Version of the Chainlink image to pull export CHAINLINK_ENV_USER="Satoshi-Nakamoto" # Name of the person running the tests (change to your own) export TEST_LOG_LEVEL="info" # info | debug | trace