diff --git a/.gitignore b/.gitignore index 0c0b0f7a2..522e640ef 100644 --- a/.gitignore +++ b/.gitignore @@ -46,9 +46,7 @@ suite/remote_runner_config.yaml *.csv # docs -docs/_site/ -docs/.sass-cache/ -docs/Gemfile.lock +book/docs/ dist/ @@ -78,4 +76,4 @@ __debug* .tool-versions import_keys_test.go -tag.py +tag.py \ No newline at end of file diff --git a/book/README.md b/book/README.md index 35ad4e76e..44a2ffe22 100644 --- a/book/README.md +++ b/book/README.md @@ -1,15 +1,18 @@ -## MDBook -Install [mdbook](https://github.com/rust-lang/mdBook) -``` +# Chainlink Testing Framework Docs + +We use [mdBook](https://github.com/rust-lang/mdBook) for our docs. They can be found [here](https://smartcontractkit.github.io/chainlink-testing-framework/). + +## Development + +First [install Rust](https://doc.rust-lang.org/cargo/getting-started/installation.html), then [mdbook](https://github.com/rust-lang/mdBook). + +```sh +# Install mdBook cargo install mdbook && \ cargo install mdbook-alerts && \ cargo install mdbook-cmdrun -``` - -Run it locally -``` +# Run the mdBook make run ``` -Open your browser [here](http://localhost:9999/) -GitHub hosted [version](https://smartcontractkit.github.io/chainlink-testing-framework/overview.html) \ No newline at end of file +Visit [localhost:9999](http://localhost:9999) to see your local version. diff --git a/book/src/libs/havoc.md b/book/src/libs/havoc.md index b641a8b0b..1c74b345f 100644 --- a/book/src/libs/havoc.md +++ b/book/src/libs/havoc.md @@ -1,211 +1,41 @@ -## Havoc +# Havoc -The `havoc` package is a Go library designed to facilitate chaos testing within Kubernetes environments using Chaos Mesh. It offers a structured way to define, execute, and manage chaos experiments as code, directly integrated into Go applications or testing suites. This package simplifies the creation and control of Chaos Mesh experiments, including network chaos, pod failures, and stress testing on Kubernetes clusters. +[![Go Reference](https://pkg.go.dev/badge/github.com/smartcontractkit/chainlink-testing-framework/havoc.svg)](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/havoc) -### Features +The `havoc` package is designed to facilitate chaos testing within [Kubernetes](https://kubernetes.io/) environments using [Chaos Mesh](https://chaos-mesh.org/). It offers a structured way to define, execute, and manage chaos experiments as code, directly integrated into Go applications or testing suites, simplifying the creation and control of Chaos Mesh experiments. -- **Chaos Object Management:** Easily create, update, pause, resume, and delete chaos experiments using Go structures and methods. -- **Lifecycle Hooks:** Utilize chaos listeners to hook into lifecycle events of chaos experiments, such as creation, start, pause, resume, and finish. -- **Support for Various Chaos Experiments:** Create and manage different types of chaos experiments like NetworkChaos, IOChaos, StressChaos, PodChaos, and HTTPChaos. -- **Chaos Experiment Status Monitoring:** Monitor and react to the status of chaos experiments programmatically. +## Features -### Installation +- **Chaos Object Management:** Create, update, pause, resume, and delete chaos experiments using Go structures and methods. +- **Lifecycle Hooks:** Utilize chaos listeners to hook into the lifecycle of chaos experiments. +- **Different Experiments:** Create and manage different types of chaos experiments to affect network, IO, K8s pods, and more. +- **Active Monitoring:** Monitor and react to the status of chaos experiments programmatically. -To use `havoc` in your project, ensure you have a Go environment setup. Then, install the package using go get: +## Requirements -``` -go get -u github.com/smartcontractkit/chainlink-testing-framework/havoc -``` - -Ensure your Kubernetes cluster is accessible and that you have Chaos Mesh installed and configured. - -### Monitoring and Observability in Chaos Experiments - -`havoc` enhances chaos experiment observability through structured logging and Grafana annotations, facilitated by implementing the ChaosListener interface. This approach allows for detailed monitoring, debugging, and visual representation of chaos experiments' impact. - -#### Structured Logging with ChaosLogger +- [Go](https://go.dev/) +- A Kubernetes cluster with [Chaos Mesh installed](https://chaos-mesh.org/docs/quick-start/) -`ChaosLogger` leverages the zerolog library to provide structured, queryable logging of chaos events. It automatically logs key lifecycle events such as creation, start, pause, and termination of chaos experiments, including detailed contextual information. +## Installation -Instantiate `ChaosLogger` and register it as a listener to your chaos experiments: - -``` -logger := havoc.NewChaosLogger() -chaos.AddListener(logger) +```sh +go get -u github.com/smartcontractkit/chainlink-testing-framework/havoc ``` -### Default package logger +## Active Monitoring -`havoc/logger.go` contains default `Logger` instance for the package. +`havoc` enhances chaos experiment observability through structured logging and Grafana annotations by implementing the [ChaosListener](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/havoc#ChaosListener) interface. -#### Visual Monitoring with Grafana Annotations +The [ChaosLogger](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/havoc#ChaosLogger) is the default implementation. It uses [zerolog](https://github.com/rs/zerolog) to provide structured, queryable logging of chaos events. It automatically logs key lifecycle events such as creation, start, pause, and termination of chaos experiments with detailed contextual information. -`SingleLineGrafanaAnnotator` is a `ChaosListener` that annotates Grafana dashboards with chaos experiment events. This visual representation helps correlate chaos events with their effects on system metrics and logs. +### Grafana Annotations -Initialize `SingleLineGrafanaAnnotator` with your Grafana instance details and register it alongside `ChaosLogger`: +We recommend using [Grafana dashboards](https://grafana.com/) to monitor your chaos experiments, and provide the [SingleLineGrafanaAnnotator](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/havoc#SingleLineGrafanaAnnotator), a `ChaosListener` that annotates dashboards with chaos experiment events so you can see in real time what your chaos experiment is doing. -```golang -annotator := havoc.NewSingleLineGrafanaAnnotator( - "http://grafana-instance.com", - "grafana-access-token", - "dashboard-uid", -) -chaos.AddListener(annotator) -``` +You can also use the [RangeGrafanaAnnotator](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/havoc#RangeGrafanaAnnotator) to show the full range of a chaos event's duration rather than a single line. -### Creating a Chaos Experiment +## Creating a Chaos Experiment To create a chaos experiment, define the chaos object options, initialize a chaos experiment with NewChaos, and then call Create to start the experiment. -Here is an example of creating and starting a PodChaos experiment: - -```golang -package main - -import ( - "context" - "github.com/smartcontractkit/chainlink-testing-framework/havoc" - "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client" - "time" -) - -func main() { - // Initialize dependencies - client, err := havoc.NewChaosMeshClient() - if err != nil { - panic(err) - } - logger := havoc.NewChaosLogger() - annotator := havoc.NewSingleLineGrafanaAnnotator( - "http://grafana-instance.com", - "grafana-access-token", - "dashboard-uid", - ) - - // Define chaos experiment - podChaos := &v1alpha1.PodChaos{ /* PodChaos spec */ } - chaos, err := havoc.NewChaos(havoc.ChaosOpts{ - Object: podChaos, - Description: "Pod failure example", - DelayCreate: 5 * time.Second, - Client: client, - }) - if err != nil { - panic(err) - } - - // Register listeners - chaos.AddListener(logger) - chaos.AddListener(annotator) - - // Start chaos experiment - chaos.Create(context.Background()) - - // Manage chaos lifecycle... -} -``` - -### Test Example - -```golang -func TestChaosDON(t *testing.T) { - testDuration := time.Minute * 60 - - // Load test config - cfg := &config.MercuryQAEnvChaos{} - - // Define chaos experiments and their schedule - - k8sClient, err := havoc.NewChaosMeshClient() - require.NoError(t, err) - - // Test 3.2: Disable 2 nodes simultaneously - - podFailureChaos4, err := k8s_chaos.MercuryPodChaosSchedule(k8s_chaos.MercuryScheduledPodChaosOpts{ - Name: "schedule-don-ocr-node-failure-4", - Description: "Disable 2 nodes (clc-ocr-mercury-arb-testnet-qa-nodes-3 and clc-ocr-mercury-arb-testnet-qa-nodes-4)", - DelayCreate: time.Minute * 0, - Duration: time.Minute * 20, - Namespace: cfg.ChaosNodeNamespace, - PodSelector: v1alpha1.PodSelector{ - Mode: v1alpha1.AllMode, - Selector: v1alpha1.PodSelectorSpec{ - GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ - Namespaces: []string{cfg.ChaosNodeNamespace}, - ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ - { - Key: "app.kubernetes.io/instance", - Operator: "In", - Values: []string{ - "clc-ocr-mercury-arb-testnet-qa-nodes-3", - "clc-ocr-mercury-arb-testnet-qa-nodes-4", - }, - }, - }, - }, - }, - }, - Client: k8sClient, - }) - require.NoError(t, err) - - // Test 3.3: Disable 3 nodes simultaneously - - podFailureChaos5, err := k8s_chaos.MercuryPodChaosSchedule(k8s_chaos.MercuryScheduledPodChaosOpts{ - Name: "schedule-don-ocr-node-failure-5", - Description: "Disable 3 nodes (clc-ocr-mercury-arb-testnet-qa-nodes-3, clc-ocr-mercury-arb-testnet-qa-nodes-4 and clc-ocr-mercury-arb-testnet-qa-nodes-5)", - DelayCreate: time.Minute * 40, - Duration: time.Minute * 20, - Namespace: cfg.ChaosNodeNamespace, - PodSelector: v1alpha1.PodSelector{ - Mode: v1alpha1.AllMode, - Selector: v1alpha1.PodSelectorSpec{ - GenericSelectorSpec: v1alpha1.GenericSelectorSpec{ - Namespaces: []string{cfg.ChaosNodeNamespace}, - ExpressionSelectors: v1alpha1.LabelSelectorRequirements{ - { - Key: "app.kubernetes.io/instance", - Operator: "In", - Values: []string{ - "clc-ocr-mercury-arb-testnet-qa-nodes-3", - "clc-ocr-mercury-arb-testnet-qa-nodes-4", - "clc-ocr-mercury-arb-testnet-qa-nodes-5", - }, - }, - }, - }, - }, - }, - Client: k8sClient, - }) - require.NoError(t, err) - - chaosList := []havoc.ChaosEntity{ - podFailureChaos4, - podFailureChaos5, - } - - for _, chaos := range chaosList { - chaos.AddListener(havoc.NewChaosLogger()) - chaos.AddListener(havoc.NewSingleLineGrafanaAnnotator(cfg.GrafanaURL, cfg.GrafanaToken, cfg.GrafanaDashboardUID)) - - // Fail the test if the chaos object already exists - exists, err := havoc.ChaosObjectExists(chaos.GetObject(), k8sClient) - require.NoError(t, err) - require.False(t, exists, "chaos object already exists: %s. Delete it before starting the test", chaos.GetChaosName()) - - chaos.Create(context.Background()) - } - - t.Cleanup(func() { - for _, chaos := range chaosList { - // Delete chaos object if it still exists - chaos.Delete(context.Background()) - } - }) - - // Simulate user activity/load for the duration of the chaos experiments - runUserLoad(t, cfg, testDuration) -} -``` +See [this runnable example](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/havoc#ExampleNewChaos) of defining a chaos experiment. diff --git a/docs/Gemfile b/docs/Gemfile deleted file mode 100644 index 9af335eec..000000000 --- a/docs/Gemfile +++ /dev/null @@ -1,11 +0,0 @@ -source "https://rubygems.org" - -gem "jekyll", "~> 3.9" -gem "kramdown-parser-gfm", "~> 1.1.0" -gem "webrick", "~> 1.7" - -gem "just-the-docs", git: "https://github.com/pmarsceill/just-the-docs" - -group :jekyll_plugins do - gem "jekyll-remote-theme", "~> 0.4.2" -end diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index cdca7af55..000000000 --- a/docs/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Docs - -We use the handy theme [just the docs](https://just-the-docs.github.io/just-the-docs/docs/navigation-structure/) for our more extensive documentation, and host it on github pages [here](https://smartcontractkit.github.io/integrations-framework/). Anything about the framework that can't be covered in our short-and-sweet [README](../README.md) should find a home in these docs. - -## Local Development - -1. Check if you have ruby installed, `ruby -v`. If not, [install it](https://www.ruby-lang.org/en/documentation/installation/). -2. `cd docs/` if you're not already there. -3. `bundle install` -4. `bundle exec jekyll serve --trace --livereload` -5. Visit `http://127.0.0.1:4000/` for a local version of the site - -The local version should update anytime you save changes to the site files. - -## Basic Style Guidelines - -- Please use folders for top-level categories, even if they only include a single `index.md` file. This helps keep pages a bit more organized. -- `Chainlink` should be capitalized when possible, unless following a programming languages capitalization conventions in a code sample. -- Try using [Grammarly](https://app.grammarly.com/) or a similar service for spell-check and clarity suggestions. - -## Custom CSS - -Custom CSS can be found and added to in the [custom.scss](./_sass/custom/custom.scss) file. - -### Notes - -You can add note `` with some borders to make text standout as warnings, asides, or general info you'd like to draw extra attention to. - -![note example](./_static/images/note-example.png) - - -```html - -
-This is a note! -
- - -
-This is a note! -
- - -
-This is a note! -
- - -
-This is a note! -
- - -
-This is a note! -
-``` - diff --git a/docs/_config.yaml b/docs/_config.yaml deleted file mode 100755 index 201137cfa..000000000 --- a/docs/_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -remote_theme: just-the-docs/just-the-docs -color_scheme: dark - -title: Chainlink Integrations Framework -description: A framework for orchestrating blockchain and Chainlink tests -logo: https://assets-global.website-files.com/5f6b7190899f41fb70882d08/5f760a499b56c47b8fa74fbb_chainlink-logo.svg -footer_content: 'Copyright © 2022 Chainlink Labs. Distributed by an MIT license.' - -# Aux links for the upper right navigation -aux_links: - 'See on GitHub': - - '//github.com/smartcontractkit/integrations-framework' - -# Makes Aux links open in a new tab. Default is false -aux_links_new_tab: false - -# Footer last edited timestamp -last_edit_timestamp: true # show or hide edit time - page must have `last_modified_date` defined in the frontmatter -last_edit_time_format: '%b %e %Y at %I:%M %p' # uses ruby's time format: https://ruby-doc.org/stdlib-2.7.0/libdoc/time/rdoc/Time.html - -plugins: - - jekyll-remote-theme - -exclude: - - vendor/ - - README.md - - Gemfile - -search_enabled: true diff --git a/docs/_sass/custom/custom.scss b/docs/_sass/custom/custom.scss deleted file mode 100644 index a678131a1..000000000 --- a/docs/_sass/custom/custom.scss +++ /dev/null @@ -1,31 +0,0 @@ -.note { - border-left-style: solid; - padding: 0.4rem 0.4rem 0.4rem 1rem; - border-top: 1px solid rgb(81, 81, 81); - border-bottom: 1px solid rgb(81, 81, 81); - border-right: 1px solid rgb(81, 81, 81); - border-radius: 0.25em; - border-left-width: 0.5em; - margin-top: 0.5em; - margin-bottom: 0.5em; -} - -.note-blue { - border-left-color: #2869e6; -} - -.note-yellow { - border-left-color: #f7d12e; -} - -.note-green { - border-left-color: #009c7b; -} - -.note-purple { - border-left-color: #5e41d0; -} - -.note-red { - border-left-color: #e94c4c; -} diff --git a/docs/assets/images/docker-resources.png b/docs/assets/images/docker-resources.png deleted file mode 100644 index 3060e129f..000000000 Binary files a/docs/assets/images/docker-resources.png and /dev/null differ diff --git a/docs/assets/images/note-example.png b/docs/assets/images/note-example.png deleted file mode 100644 index 81e4aca9f..000000000 Binary files a/docs/assets/images/note-example.png and /dev/null differ diff --git a/docs/chainlink-nodes/index.md b/docs/chainlink-nodes/index.md deleted file mode 100644 index 35a5dd8f2..000000000 --- a/docs/chainlink-nodes/index.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -layout: default -title: Chainlink Nodes -nav_order: 4 -has_children: false ---- - -# Chainlink Nodes - -Make sure to have your environment setup code in place, which will deploy a few Chainlink nodes, and give you an `env` environment. - -```go -// Get a list of all Chainlink nodes deployed in your test environment -chainlinkNodes, err := client.ConnectChainlinkNodes(env) -``` - -From here, you can [interact with](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/client#Chainlink) each Chainlink node to manage keys, jobs, bridges, and transactions. - -## Chainlink Jobs - -The most common interaction you'll have with Chainlink nodes will likely be creating jobs, using the `chainlinkNode.CreateJob(JobSpec)` method. Chainlink jobs are how the Chainlink nodes know what actions they're expected to perform on chain, and how thy should perform them. A typical test consists of launching your resources, deploying contracts to the blockchain, and telling the Chainlink node to interact with those contracts by creating a job. Read more about Chainlink jobs and the specifics on using them [here](https://docs.chain.link/docs/jobs/). - -There are plenty of built in [JobSpecs](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/client#JobSpec) like the [Keeper Job Spec](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/client#KeeperJobSpec) and the [OCR Job Spec](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework/client#OCRTaskJobSpec) that you can use. But if for whatever reason, those don't do the job for you, you can create a raw TOML job with `CreateJobRaw(string)` like below. - -```go -jobData, err := chainlinkNode.CreateJobRaw(` -schemaVersion = 1 -otherField = true -`) -``` diff --git a/docs/contracts/index.md b/docs/contracts/index.md deleted file mode 100644 index 214f7cf22..000000000 --- a/docs/contracts/index.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -layout: default -title: Contracts -nav_order: 6 -has_children: false ---- - -# Smart Contracts - -Tests require deploying at least a few contracts to whatever chain you're running on. You can find the code used to deploy and interact with these contracts in the [contracts](https://github.com/smartcontractkit/integrations-framework/tree/main/contracts) package. Most of these are self-explanatory, but Keeper and OCR can be a bit more complicated. - -## Contract Deployer - -Each network has a contract deployer - -```go -// See the previous setup code on how to get your default network -// The contract deployer will use the first private key listed to deploy contracts from -contractDeployer, err := contracts.NewContractDeployer(defaultNetwork) -``` - -From here, you can use the `contractDeployer` on its own, as defined [here](https://pkg.go.dev/github.com/smartcontractkit/integrations-framework/contracts#ContractDeployer). For a lot of one off contracts, like the Link Token contract, that's all you need. - -```go -contractDeployer.DeployLinkTokenContract() -``` - -From there, contract interactions are defined in the [contracts package](https://pkg.go.dev/github.com/smartcontractkit/integrations-framework/contracts#pkg-overview). - -## Adding New Contracts - -
-Currently it's a bit of a laborious process to add a new contract that isn't already defined. We're planning on improving this in the future. If you have ideas to make things easier, feel free to make a PR! -
diff --git a/docs/contributing/index.md b/docs/contributing/index.md deleted file mode 100644 index 67d68245b..000000000 --- a/docs/contributing/index.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -layout: default -title: Contributing -nav_order: 99 -has_children: false ---- - -# Contributing - -See the general [Chainlink contributing guidelines](https://docs.chain.link/docs/contributing-to-chainlink/). - -If you'd like to raise a feature request or a bug, please do so using our [GitHub Issues](https://github.com/smartcontractkit/integrations-framework/issues) page, or better yet, submit a PR that fixes the issue. - -## PR Process - -1. Fork the repository -2. Make your changes on the fork -3. Submit a PR to merge your fork's branch with the `main` branch -4. Pass CI checks and review process diff --git a/docs/debug/eth2.md b/docs/debug/eth2.md deleted file mode 100644 index 8b3284207..000000000 --- a/docs/debug/eth2.md +++ /dev/null @@ -1,72 +0,0 @@ -# Prysm containers - -Docker images provided by Prysm are bare-bone and do not even contain executable shell, which makes debugging impossible. To overcome it we need to build our own image with debug tools installed. The following Dockerfile will do the job for `beacon chain`: - -``` -ARG tag -FROM --platform=linux/x86_64 gcr.io/prysmaticlabs/prysm/beacon-chain:$tag as upstream - -FROM --platform=linux/x86_64 debian:buster-slim -COPY --from=upstream /app /app - -RUN apt-get update && apt-get install -y \ - curl \ - jq \ - nano \ - netcat-openbsd \ - iputils-ping \ - && rm -rf /var/lib/apt/lists/* - - -STOPSIGNAL SIGINT - -ENTRYPOINT ["/app/cmd/beacon-chain/beacon-chain"] -``` - -And for `validator`: - -``` -ARG tag -FROM --platform=linux/x86_64 gcr.io/prysmaticlabs/prysm/validator:$tag as upstream - -FROM --platform=linux/x86_64 debian:buster-slim -COPY --from=upstream /app /app - -RUN apt-get update && apt-get install -y \ - curl \ - jq \ - nano \ - netcat-openbsd \ - iputils-ping \ - && rm -rf /var/lib/apt/lists/* - - -STOPSIGNAL SIGINT - -ENTRYPOINT ["/app/cmd/validator/validator"] -``` - -And the use as follows: - -``` -docker build -t gcr.io/prysmaticlabs/prysm/validator:debug --build-arg tag=v4.1.0 . -``` - -# Lighthouse - -No supported yet. - -# Local Kubernetes @ Docker Desktop (MacOS) - -It's very handy and easy to setup, but comes with some hurdles: `hostpath` storage class doesn't work as expected in relation to retention policy. Whether you chose `Delete` or `Recycle` old data won't be removed and your chain will start correctly only the first time, when there is no data yet (consecutive runs will fail, because they will try to generate genesis.json based on previous chain states). - -My hacky workaround is to generate a new host directory based on current timestamp every time I start the service. Not idea, but works. It has one main drawback, though: disk gets bloated, since old data is never removed. Now... you can delete it manually, but it's not as straight-fowardward as you might think, because that directory is not directly accessible from your MacOS machine, because it runs inside Docker's VM. Here's how to go about it: - -``` -docker run -it --rm --privileged --pid=host justincormack/nsenter1 -``` - -(Other options are described [here](https://gist.github.com/BretFisher/5e1a0c7bcca4c735e716abf62afad389)) -Then you need to find find a folder where `rootfs` for Docker VM is located, in my case it was `/containers/services/02-docker/rootfs` (search for a folder containing whatever hostpath you have the persistent volume mounted at, or inspect one of your pods, check the volume mount and find it in the VM, just remember that even if your volume is supposedly mounted in `/data/shared` on the host in reality that `/data/shared` folder is still relative to wherever `rootfs` folder is located). - -Once you find it, you can delete the old data and start the service again. Or you can use it for debugging to easily inspect the content of shared volumes (useful in case of containers that are bare-bone and don't even have `bash` installed, like Prysm). diff --git a/docs/favicon.ico b/docs/favicon.ico deleted file mode 100644 index 995264b21..000000000 Binary files a/docs/favicon.ico and /dev/null differ diff --git a/docs/index.md b/docs/index.md deleted file mode 100755 index 95cfe4d67..000000000 --- a/docs/index.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -layout: default -title: Chainlink Testing Framework -nav_order: 1 -description: 'A general blockchain integration testing framework geared towards Chainlink projects' -permalink: / ---- - -# Chainlink Testing Framework - -[![Go Report Card](https://goreportcard.com/badge/github.com/smartcontractkit/chainlink-testing-framework)](https://goreportcard.com/report/github.com/smartcontractkit/chainlink-testing-framework) -[![Go Reference](https://pkg.go.dev/badge/github.com/smartcontractkit/chainlink-testing-framework.svg)](https://pkg.go.dev/github.com/smartcontractkit/chainlink-testing-framework) -![Tests](https://github.com/smartcontractkit/chainlink-testing-framework/actions/workflows/test.yaml/badge.svg) -![Lint](https://github.com/smartcontractkit/chainlink-testing-framework/actions/workflows/lint.yaml/badge.svg) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - -The Chainlink Testing Framework is a blockchain development and testing framework written in [Go](https://go.dev/). While the framework is designed primarily with testing Chainlink nodes in mind, it's not at all limited to that function. With this framework, blockchain developers can create extensive integration, e2e, performance, and chaos tests for almost anything! - -Are you new to [blockchain development](https://ethereum.org/en/developers/docs/), [smart contracts](https://docs.chain.link/docs/beginners-tutorial/), or [Chainlink](https://chain.link/)? Learn more by clicking the links! - -Here you'll find some guidelines on writing blockchain tests using this framework, and some tips on contributing to it. In most code examples presented, error checking is omitted for brevity's sake. **Please check your errors**. - -Some notable packages we use include: - -- [zerolog](https://github.com/rs/zerolog) -- [Kubernetes](https://github.com/kubernetes/kubernetes) diff --git a/docs/mock-adapter/index.md b/docs/mock-adapter/index.md deleted file mode 100644 index 6d09c7d0d..000000000 --- a/docs/mock-adapter/index.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -layout: default -title: Mock Adapter -nav_order: 6 -has_children: false ---- - -# Mock Adapter - -Most Chainlink jobs involve reaching out to outside APIs (referred to as "adapters") to gather data before making on-chain transactions. So along with a simulated blockchain, we also launch a mock adapter (sometimes referred to as a "mock server") that you can set and retrieve values for. We use an implementation of the [mock-server](https://www.mock-server.com/) project, but have simplified our interaction with it through just a few methods. - -Connect to your mock adapter much like you do with your Chainlink nodes. - -```go -// Get a connection to the mock server -mockserver, err = client.ConnectMockServer(env) - -path := "/test_resp" -// Any GET calls to the path will return a Chainlink-node-readable response of 5 -err := mockserver.SetValuePath(path, 5) - -// Test some things - -// Change the response of path to 15 -err := mockserver.SetValuePath(path, 15) -``` diff --git a/docs/quickstart/index.md b/docs/quickstart/index.md deleted file mode 100644 index 945c86ccd..000000000 --- a/docs/quickstart/index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -layout: default -title: Quick Start -nav_order: 2 -has_children: true ---- - -Follow this guide to get quickly set up and running tests. diff --git a/docs/quickstart/local-k8s-setup.md b/docs/quickstart/local-k8s-setup.md deleted file mode 100644 index 057f9d3bb..000000000 --- a/docs/quickstart/local-k8s-setup.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -layout: default -title: Kubernetes Cluster -nav_order: 1 -parent: Quick Start ---- - -# Kubernetes Cluster - -In order to run tests, we need a [Kubernetes](https://kubernetes.io/) (often abbreviated as _K8s_) cluster, and a way to connect to it (see [why Kubernetes](https://smartcontractkit.github.io/integrations-framework/setup/kubernetes.html#why)). - -In order to connect to a K8s cluster, we need to [download kubectl](https://kubernetes.io/releases/download/). This will let you connect to and control any Kubernetes cluster. - -Next you'll need to get a Kubernetes cluster running. If you're lucky, you'll have one running in the cloud somewhere that you can utilize. But if you don't, or just want to get something running locally, you can use [Docker](https://www.docker.com/) and [K3D](https://k3d.io/) to launch a local Kubernetes cluster. - -
-The Kubernetes setup process, and the resources needed to run a K8s cluster for these tests is a common pain point. We're exploring ways to lessen the resources needed, and possibly expand to other systems as well as Kubernetes. -
- -## Local Kubernetes Cluster - -First up, [install Docker](https://docs.docker.com/get-docker/) if you haven't already, then [install K3D](https://k3d.io/#installation). These two technologies will enable you to run a lightweight Kubernetes cluster on your machine quickly. - -### Configure Docker - -Docker will need a little more power than its default in order to run our tests, especially if we want to run many of them at once. In Docker Desktop, you can go to `Settings (the gear in the top right) -> Resources -> Advanced` to change how much power to give Docker. It will depend on your personal machine's specs, but a good starting point for some basic tests is 4CPUs and 4GB RAM. - -![docker resources](../assets/images/docker-resources.png) - -### K3D Local Cluster - -`k3d cluster create` - -After a few minutes, the above command will deploy a local Kubernetes cluster on your machine! Use the below commands to check that things are working correctly. - -```sh -> kubectl cluster-info -Kubernetes control plane is running at https://0.0.0.0:60887 -CoreDNS is running at https://0.0.0.0:60887/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy -Metrics-server is running at https://0.0.0.0:60887/api/v1/namespaces/kube-system/services/https:metrics-server:https/proxy -> kubectl get ns -NAME STATUS AGE -default Active 114s -kube-system Active 114s -kube-public Active 114s -kube-node-lease Active 114s -``` - -You can use the `kubectl` CLI to control and view the Kubernetes cluster if you're handy with it, and we'll be using those commands for simplicities sake. You can also use [Lens](https://k8slens.dev/), a handy GUI that simplifies things a bit. Use the below command to tear down the cluster. - -`k3d cluster delete` - -
-Not deleting your cluster when not in use means that it will continue to use up your machine's resources. -
diff --git a/docs/quickstart/running-our-tests.md b/docs/quickstart/running-our-tests.md deleted file mode 100644 index 3fc0c69b6..000000000 --- a/docs/quickstart/running-our-tests.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -layout: default -title: Running our Tests -nav_order: 3 -parent: Quick Start ---- - -# Running our Tests - -We have some tests written that we keep up to date in the [main Chainlink repo](https://github.com/smartcontractkit/chainlink). In order to run them for yourself, follow the steps below. - -1. Clone the latest develop branch of the [Chainlink repo](https://github.com/smartcontractkit/chainlink) -2. See the tests and running instructions in the [integration-tests folder](https://github.com/smartcontractkit/chainlink/tree/develop/integration-tests) diff --git a/docs/quickstart/writing-your-first-test.md b/docs/quickstart/writing-your-first-test.md deleted file mode 100644 index 6b77aa136..000000000 --- a/docs/quickstart/writing-your-first-test.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: default -title: Writing Your First Test -nav_order: 2 -parent: Quick Start ---- - -# Writing Your First Test - -Tests are written entirely in [Go](https://go.dev/). If you're familiar with Go and want your own setup, feel free to do so and use the code below as examples. Otherwise, you can utilize our [template repository](https://github.com/smartcontractkit/chainlink-integration-tests-template) which contains tons of examples in a ready-to-go codebase. diff --git a/docs/setup/code.md b/docs/setup/code.md deleted file mode 100644 index 7c9ad69da..000000000 --- a/docs/setup/code.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -layout: default -title: Test Setup Code -nav_order: 3 -parent: Setup ---- - -# Test Setup Code - -Now that we've got our config and Kubernetes sorted, we can write a bit of code that will deploy an environment for our test to run. To deploy our simulated geth, mock-server, and Chainlink instances, we rely on `env` package. This package handles deploying everything our test needs to the Kubernetes cluster. - -```go -// We use the env package to make and handle deployed resources -import "github.com/smartcontractkit/chainlink-testing-framework/k8s/environment" - -// Deploy a testing environment, and receive it as the `env` variable. This is used to connect to resources. -e = environment.New(nil) -err := e. - AddHelm(mockservercfg.New(nil)). - AddHelm(mockserver.New(nil)). - AddHelm(geth.New(nil)). - AddHelm(chainlink.New(nil)). - Run() -Expect(err).ShouldNot(HaveOccurred(), "Environment deployment shouldn't fail") -// Connect to all networks specified in the networks.yaml file -networkRegistry := client.NewDefaultNetworkRegistry() -// Retrieve these networks -networks, err := networkRegistry.GetNetworks(env) -// Get the default network (the first one in your listed selected_networks) -defaultNetwork := networks.Default -``` - -Most of the setup code will be the same for all your tests. Here's a more detailed explanation as to what some of the deployment code is doing to launch a few common test resources. - -```go -e := environment.New(&environment.Config{ - Labels: []string{fmt.Sprintf("envType=%s", pkg.EnvTypeEVM5)}, // set more additional labels -}) -err := e. - AddHelm(mockservercfg.New(nil)). // add more Helm charts, all charts got merged in a manifest and deployed with kubectl when you call Run() - AddHelm(mockserver.New(nil)). - Run() -Expect(err).ShouldNot(HaveOccurred(), "Environment deployment shouldn't fail") -// do some other stuff with deployed charts if you need to interact with deployed services -err = e. - AddChart(blockscout.New(&blockscout.Props{})). // you can also add cdk8s charts if you like Go code - AddHelm(geth.New(nil)). - AddHelm(chainlink.New(nil)). - Run() -// Connect to all networks specified in the networks.yaml file -networkRegistry := client.NewDefaultNetworkRegistry() -// Retrieve these networks -networks, err := networkRegistry.GetNetworks(env) -// Get the default network (the first one in your listed selected_networks) -defaultNetwork := networks.Default -``` - -These common resources consist of - -- A simulated Geth instance -- A basic mock server that serves as a mock adapter for Chainlink nodes -- A specified number of chainlink nodes - -## Test Tear Down - -When your test is done, you'll want to have a way to tear down the test environment you launched. You'll also want to be able to see the logs from your test environment. Below is a typical test flow. - -```go -// Launch our environment -env, err := environment.DeployOrLoadEnvironment( - environment.NewChainlinkConfig(environment.ChainlinkReplicas(1, nil), "chainlink-test-setup"), - tools.ChartsRoot, -) - -// Put test logic here - -// Tear down the test environment -// Prints some handy stats on Gas usage for the test, if you'd like to see that info. -networks.Default.GasStats().PrintStats() -// Tears down the test environment, according to options you selected in the `framework.yaml` config file -err = actions.TeardownSuite( - env, // The test environment object - networks, // The list of networks obtained from `networks, err := networkRegistry.GetNetworks(env)` - utils.ProjectRoot, // The folder location you'd like logs (on test failure) to be dumped to - nil, // An optional test reporter for more custom test statistics (we'll get to that later) -) -``` diff --git a/docs/setup/config.md b/docs/setup/config.md deleted file mode 100644 index e495bf9b0..000000000 --- a/docs/setup/config.md +++ /dev/null @@ -1,147 +0,0 @@ ---- -layout: default -title: Config -nav_order: 2 -parent: Setup ---- - -# Config - -The framework draws on 2 config files, `framework.yaml` and `networks.yaml`. Whether you're working in the framework's main repo, or importing the framework as a library, it will look for these two config files in order to launch, configure, and connect all the testing resources. - -## `framework.yaml` - -This config handles how the framework should handle logging, deploying, and tearing down test environments. Check out the example below, or the most up-to-date version [here](https://github.com/smartcontractkit/integrations-framework/blob/main/framework.yaml). - -Location of this file can be overridden by setting a file path as the `FRAMEWORK_CONFIG_FILE` environment variable. - -```yaml -# Retains default and configurable name-values for the integration framework -# -# All configuration can be set at runtime with environment variables, for example: -# KEEP_ENVIRONMENTS=OnFail -keep_environments: Never # Options: Always, OnFail, Never -logging: - # panic=5, fatal=4, error=3, warn=2, info=1, debug=0, trace=-1 - level: 0 - -# Specify the image and version of the Chainlink image you want to run tests against. Leave blank for default. -chainlink_image: public.ecr.aws/chainlink/chainlink -chainlink_version: 1.2.1 -chainlink_env_values: -``` - -## `networks.yaml` - -This file handles the settings for each network you want to connect to. This is a truncated version, see the full one [here](https://github.com/smartcontractkit/integrations-framework/blob/main/networks.yaml). - -Location of this file can be overridden by setting a file path as the `NETWORKS_CONFIG_FILE` environment variable. **The first private key listed will be considered the default one to use for deploying contracts and funding addresses**. - -```yaml -private_keys: &private_keys - private_keys: # Private keys that are used for simulated networks. These are publicly known keys for use only in simulated networks. - - ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 - - 59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d - - 5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a - - 7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6 - - 47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a - - 8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba - - 92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e - - 4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356 - - dbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97 - - 2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6 - - f214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897 - - 701b615bbdfb9de65240bc28bd21bbc0d996645a3dd57e7b12bc2bdf6f192c82 - - a267530f49f8280200edf313ee7af6b827f2a8bce2897751d06a843f644967b1 - - 47c99abed3324a2707c28affff1267e45918ec8c3f20b8aa892e8b065d2942dd - - c526ee95bf44d8fc405a158bb884d9d1238d99f0612e9f33d006bb0789009aaa - - 8166f546bab6da521a8369cab06c5d2b9e46670292d85c875ee9ec20e84ffb61 - - ea6c44ac03bff858b476bba40716402b03e41b8e97e276d1baec7c37d42484a0 - - 689af8efa8c651a91ad287602527f3af2fe9f6501a7ac4b061667b5a93e037fd - - de9be858da4a475276426320d5e9262ecfc3ba460bfac56360bfa6c4c28b4ee0 - - df57089febbacf7ba0bc227dafbffa9fc08a93fdc68e1e42411a14efcf23656e - -selected_networks: # Selected network(s) for test execution - - 'geth' - -networks: ... # See full file -``` - -Each network will have an identifier that can be listed in the `selected_networks` section. - - -```yaml -geth: # Network identifier - name: "Ethereum Geth dev" # Human-readable name for network - chain_id: 1337 # ETH chain ID - type: eth_simulated # eth_simulated or eth_testnet - secret_private_keys: false # Experimental feature for storing private keys as Kubernetes secrets - namespace_for_secret: default # Experimental feature for storing private keys as Kubernetes secrets - private_keys: # List of private keys for this network, used for funding and deploying contracts - - ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 - chainlink_transaction_limit: 500000 # Estimated gas limit that a single Chainlink tx might take (used for funding Chainlink nodes) - transaction_timeout: 2m # Duration of how long for the framework to wait for a confirmed transaction before timeout - minimum_confirmations: 1 # How many blocks to wait for transaction to be confirmed - gas_estimation_buffer: 10000 # How much gas to bump transaction an contract creations by (added to auto-estimations) - block_gas_limit: 40000000 # How much gas each block of the network should be using -``` - - -There are a couple values available for launching simulated Geth instances. If you choose them in your `selected_networks` list, they will launch with the following properties: - -### `geth` - -The default geth instance, small footprint with fast block times. - -```yaml -resources: - requests: - cpu: .2 - memory: 1000Mi - -config_args: - '--dev.period': '1' - '--miner.threads': '1' - '--miner.gasprice': '10000000000' - '--miner.gastarget': '80000000000' -``` - -### `geth_performance` - -Used for performance tests, launching a powerful geth instance with large blocks and fast block times. - -```yaml -resources: - requests: - cpu: 4 - memory: 4096Mi - limits: - cpu: 4 - memory: 4096Mi - -config_args: - '--dev.period': '1' - '--miner.threads': '4' - '--miner.gasprice': '10000000000' - '--miner.gastarget': '30000000000' -``` - -### `geth_realistic` - -Launches a powerful geth instance that tries to simulate Ethereum mainnet as close as possible. - -```yaml -resources: - requests: - cpu: 4 - memory: 4096Mi - limits: - cpu: 4 - memory: 4096Mi - -config_args: - '--dev.period': '14' - '--miner.threads': '4' - '--miner.gasprice': '10000000000' - '--miner.gastarget': '15000000000' -``` diff --git a/docs/setup/index.md b/docs/setup/index.md deleted file mode 100644 index 9226c64dd..000000000 --- a/docs/setup/index.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -layout: default -title: Setup -nav_order: 3 -has_children: true ---- - -# Setup - -There's some small setup work we need to make sure is in place before we run or write any tests. Namely we need to - -- Point to a K8s cluster -- Set some config files -- Write some test-setup code diff --git a/docs/setup/kubernetes.md b/docs/setup/kubernetes.md deleted file mode 100644 index bf23befef..000000000 --- a/docs/setup/kubernetes.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -layout: default -title: Kubernetes -nav_order: 1 -parent: Setup ---- - -# Kubernetes Setup - -In order to use this framework, you must have a connection to an actively running [Kubernetes cluster](https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/) and an install of [kubectl](https://kubernetes.io/releases/download/). If you don't have a Kubernetes cluster handy, check out our quickstart guide on setting up a [local cluster](https://smartcontractkit.github.io/integrations-framework/quickstart/local-k8s-setup.html#local-kubernetes-cluster) which should work fine for smaller tests. Larger tests, or many tests run in parallel will likely render these local solutions inadequate. A Kubernetes cluster with 4 vCPU and 10 GB RAM is a good starting point. - -**The framework will use whatever your current KUBECONFIG context is**, see how to set a context [here](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/). Learn more about setting up a Kubernetes cluster [here](https://kubernetes.io/docs/setup/). Once you have the cluster setup and are running tests, you can monitor the test architecture with the [kubectl](https://kubernetes.io/docs/reference/kubectl/kubectl/) CLI, or check out [Lens](https://k8slens.dev/) for a handy GUI. - -## Why? - -There's a lot of different components to bring up for each test, most of which involve: - -- A simulated blockchain -- Some number of Chainlink nodes -- An equal number of postgres DBs to support the Chainlink nodes -- At least one external adapter - -Following the good testing practice of having clean, non-dependent test environments means we're creating a lot of these components for each test, and tearing them down soon after. In order to organize these test environments, and after finding `docker compose` to be woefully inadequate after a certain point, Kubernetes was the obvious choice. - -
-The Kubernetes setup process, and the resources needed to run a K8s cluster for these tests is a common pain point. We're exploring ways to lessen the resources needed, and possibly expand to other systems as well as Kubernetes. -
diff --git a/docs/transactions/index.md b/docs/transactions/index.md deleted file mode 100644 index c53397dd7..000000000 --- a/docs/transactions/index.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -layout: default -title: Transactions -nav_order: 5 -has_children: false ---- - -# Transactions - -In order to fund contracts and Chainlink nodes, you need to make a few transactions on the blockchain you're working with. By default, sending a transaction will block the test until the transaction is confirmed on the blockchain, then move on. If you want to change this behavior, check the section on [parallel transactions](#parallel-transactions). - -## Funding Contracts - -Most on-chain objects like contracts have some sort of `Fund(...)` method that can be used to send an amount of funds to that contract's address. - -```go -// Funds the ocrContract from the default private key. -// Fund this contract with 1 ETH -err := ocrContract.Fund(big.NewFloat(1)) -// Funds the same contract with .001 ETH -err = ocrContract.Fund(big.NewFloat(.001)) -``` - -## Funding Chainlink Nodes - -Chainlink nodes are easy to fund with the handy [actions package](https://pkg.go.dev/github.com/smartcontractkit/integrations-framework/actions). - -```go -// Funds each node in the `chainlinkNodes` array with .01 of the network's native currency -err := actions.FundChainlinkNodes(chainlinkNodes, defaultNetwork, big.NewFloat(.01)) -``` - -## Parallel Transactions - -Sometimes you'll want to fund a bunch of addresses or launch a few contracts that have no relation to each other. Rather than waiting for each transaction or contract deployment to be confirmed before moving on to the next one, you can make use of parallel transactions. - -```go -// Any transactions made after this line will be submitted instantly to the blockchain without waiting for previous ones. -defaultNetwork.ParallelTransactions(true) -``` - -This can seriously speed up some test setups and executions, for example, if you want to fund your Chainlink nodes and a couple deployed contracts at the beginning of your test. But be wary, as some events depend on previous ones being confirmed, like funding a contract only after it has been deployed and confirmed on chain. For that, utilize the `defaultNetwork.WaitForEvents()` method call to halt running. - -```go -// Set parallel transactions -defaultNetwork.ParallelTransactions(true) - -// A pretend method to deploy a bunch of contracts and return them as a list -someListOfContracts := deploySomeContracts() -// Fund Chainlink nodes, which don't depend on contracts being confirmed on chain -err := actions.FundChainlinkNodes(chainlinkNodes, defaultNetwork, big.NewFloat(.01)) - -// Wait for all on-chain events started above to complete before moving on -err = defaultNetwork.WaitForEvents() - -// Fund the deployed contracts -for _, contract := range someListOfContracts { - contract.Fund(big.NewFloat(1)) -} - -// Wait for the contract funding to go through before moving on -err = defaultNetwork.WaitForEvents() -``` - -If you see errors in funding or interacting with contracts that imply the contracts don't exist, it's likely you set parallel transactions to `true` and failed to `WaitForEvents()` at an appropriate time. This feature is handy to speed up your test setups and executions, but can trip you up if not properly monitored, so take care. diff --git a/docs/writing-a-test/index.md b/docs/writing-a-test/index.md deleted file mode 100644 index 7d4c159c4..000000000 --- a/docs/writing-a-test/index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: default -title: Writing a Test -nav_order: 7 -has_children: true ---- - -# Writing a Test - -Here we walk through writing a couple different types of tests. More examples can be found in our [test suite directory](https://github.com/smartcontractkit/integrations-framework/tree/main/suite). diff --git a/docs/writing-a-test/integration.md b/docs/writing-a-test/integration.md deleted file mode 100644 index e9f32014a..000000000 --- a/docs/writing-a-test/integration.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: default -title: Integration Tests -nav_order: 1 -parent: Writing a Test ---- - -# Integration Tests - -Integration tests refer to standard, and (relatively) quick running tests. You can see some examples in our [smoke test suite](https://github.com/smartcontractkit/integrations-framework/tree/main/suite/smoke). diff --git a/docs/writing-a-test/soak.md b/docs/writing-a-test/soak.md deleted file mode 100644 index 941ecd8ce..000000000 --- a/docs/writing-a-test/soak.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -layout: default -title: Soak Tests -nav_order: 2 -parent: Writing a Test ---- - -# Soak Tests - -Soak tests refer to running longer tests, that can take anywhere from hours to days to see how the application fares over long stretches of time. See some examples in our [soak test suite](https://github.com/smartcontractkit/integrations-framework/tree/main/suite/soak). - -The test framework is designed around you launching a test environment to a K8s cluster, then running the test from your personal machine. Your personal machine coordinates the chainlink nodes, reads the blockchain, etc. This works fine for running tests that take < 5 minutes, but soak tests often last days or weeks. So the tests become dependent on your local machine maintaining power and network connection for that time frame. This quickly becomes untenable. So the solution is to launch a `remote-test-runner` container along with the test environment. - -## Writing the Test - -Since the test is being run from a `remote-test-runner` instead of your local machine, setting up and tearing down the test environment is a little different. So is connecting to things like the blockchain networks, chainlink nodes, and the mock adapter. Most other interactions should be as normal though. - -```go -// Connects to the soak test resources from the `remote-test-runner` -env, err := environment.DeployOrLoadEnvironmentFromConfigFile( - tools.ChartsRoot, // Default location of helm charts to look for - "/root/test-env.json", // Default location for the soak-test-runner container -) -log.Info().Str("Namespace", env.Namespace).Msg("Connected to Soak Environment") - -// Run test logic - -// Teardown remote suite -if err := actions.TeardownRemoteSuite(keeperBlockTimeTest.TearDownVals()); err != nil { - log.Error().Err(err).Msg("Error tearing down environment") -} -log.Info().Msg("Soak Test Concluded") -``` - -## Running the Test - -The soak tests are triggered by the [soak_runner_test.go](https://github.com/smartcontractkit/integrations-framework/blob/main/suite/soak/soak_runner_test.go) tests, or with `make test_soak`. When running, the test will check for a local config file: `remote_runner_config.yaml`. If it's not already created, it will generate one with some default values, and then inform you that you should modify those values. - -```yaml -test_regex: '@soak-ocr' # The regex of the test name to run -test_directory: /Users/adam/Projects/integrations-framework/suite/soak/tests # The directory where the go tests you want the remote runner to run -# Slack values are covered below -``` - -Modify these values that make sense for the tests you want to run. Once the values are modified, you can run the test again. The soak runner test will then compile the tests that you pointed to by the `test_directory` into a `remote.test` executable. This executable, including your local `framework.yaml` and `networks.yaml` configs are uploaded to the remote test runner. Make sure to read the section below to take advantage of slack integration. - -## Watching the Test - -The rest of the `remote_runner_config.yaml` file holds various Slack bot params to notify you when the test finishes. - -```yaml -slack_api_key: abcdefg # A Slack API key to upload test results with. This should be the `Bot User OAuth Token` -slack_channel: C01xxxxx # The Slack Channel ID (open your Slack channel details and copy the ID there) -slack_user_id: U01xxxxx # Your Slack member ID https://zapier.com/help/doc/common-problems-slack -``` - -The Slack Bot will need to have the following: - -- Permission for [files:write](https://api.slack.com/scopes/files:write) -- Permission for [chat:write](https://api.slack.com/scopes/chat:write) -- The bot must be invited into the channel you want it to notify in: `/invite @botname` HINT: If you get the error `not_in_channel` this is likely what you need to set up. - -## After the Test - -The test environment **will stay active until you manually delete it from your Kubernetes cluster**. This keeps the test env alive so you can view the logs when the test is done. You can do so by [using kubectl](https://www.dnsstuff.com/how-to-tail-kubernetes-and-kubectl-logs), something like [Lens](https://k8slens.dev/), or use the [chainlink-testing-framework](https://github.com/smartcontractkit/chainlink-testing-framework/k8s/examples/dump/env.go) `dump` command. diff --git a/havoc/README.md b/havoc/README.md index f15820690..8ba1fe9d8 100644 --- a/havoc/README.md +++ b/havoc/README.md @@ -1,5 +1,5 @@ -## Havoc +# Havoc The `havoc` package is a Go library designed to facilitate chaos testing within Kubernetes environments using Chaos Mesh. -[![Documentation](https://img.shields.io/badge/Documentation-MDBook-blue?style=for-the-badge)](https://smartcontractkit.github.io/chainlink-testing-framework/libs/havoc.html) \ No newline at end of file +[![Documentation](https://img.shields.io/badge/Documentation-MDBook-blue?style=for-the-badge)](https://smartcontractkit.github.io/chainlink-testing-framework/libs/havoc.html) diff --git a/havoc/chaos_helper.go b/havoc/chaos_helper.go index 831c78584..73fde0a28 100644 --- a/havoc/chaos_helper.go +++ b/havoc/chaos_helper.go @@ -5,7 +5,7 @@ import ( "time" ) -// WaitForAllChaosRunning waits for all chaos experiments to be running +// WaitForAllChaosRunning blocks until chaos experiments are running func WaitForAllChaosRunning(chaosObjects []*Chaos, timeoutDuration time.Duration) error { timeout := time.NewTimer(timeoutDuration) defer timeout.Stop() diff --git a/havoc/chaos_listener.go b/havoc/chaos_listener.go index 01476d702..c3724980e 100644 --- a/havoc/chaos_listener.go +++ b/havoc/chaos_listener.go @@ -1,5 +1,6 @@ package havoc +// ChaosListener is an interface that can be implemented by clients to listen to and react to chaos events. type ChaosListener interface { OnChaosCreated(chaos Chaos) OnChaosCreationFailed(chaos Chaos, reason error) diff --git a/havoc/go.mod b/havoc/go.mod index ba1594b74..aae6cc79f 100644 --- a/havoc/go.mod +++ b/havoc/go.mod @@ -61,6 +61,7 @@ require ( golang.org/x/time v0.6.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/havoc/range_grafana_annotator.go b/havoc/range_grafana_annotator.go index c8b81bde5..4a3b9e1f5 100644 --- a/havoc/range_grafana_annotator.go +++ b/havoc/range_grafana_annotator.go @@ -239,3 +239,6 @@ func (l RangeGrafanaAnnotator) OnScheduleDeleted(chaos Schedule) { } l.chaosMap[chaos.Object.GetName()] = res.ID } + +func hey() { +} diff --git a/havoc/single_line_grafana_annotator.go b/havoc/single_line_grafana_annotator.go index c33448547..c674fd1bb 100644 --- a/havoc/single_line_grafana_annotator.go +++ b/havoc/single_line_grafana_annotator.go @@ -11,6 +11,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/lib/grafana" ) +// SingleLineGrafanaAnnotator annotates Grafana dashboards with chaos experiment events in a single line. type SingleLineGrafanaAnnotator struct { client *grafana.Client dashboardUID string diff --git a/havoc/single_line_grafana_annotator_test.go b/havoc/single_line_grafana_annotator_test.go new file mode 100644 index 000000000..a6d806273 --- /dev/null +++ b/havoc/single_line_grafana_annotator_test.go @@ -0,0 +1,39 @@ +package havoc + +import ( + "context" + "os" + "time" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/rs/zerolog" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func ExampleNewChaos() { + testLogger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger() + client := fake.NewFakeClient() + podChaos := &v1alpha1.PodChaos{ /* PodChaos spec */ } + chaos, err := NewChaos(ChaosOpts{ + Object: podChaos, + Description: "Pod failure example", + DelayCreate: 5 * time.Second, + Client: client, + Logger: &testLogger, + }) + if err != nil { + panic(err) + } + logger := NewChaosLogger(testLogger) + annotator := NewSingleLineGrafanaAnnotator( + "http://grafana-instance.com", + "grafana-access-token", + "dashboard-uid", + Logger, + ) + chaos.AddListener(logger) + chaos.AddListener(annotator) + + chaos.Create(context.Background()) + // Output: +}