Skip to content

Commit

Permalink
[TT 1024] Fixes Live Network Smoke Tests (#12603)
Browse files Browse the repository at this point in the history
* [TT-1024] Fix Live Network Smoke Test

* Use new CTF

* Typo

* go mod tidy

* Lint

---------

Co-authored-by: anirudhwarrier <[email protected]>
  • Loading branch information
kalverra and anirudhwarrier authored Mar 27, 2024
1 parent dd333e9 commit 76b72ee
Show file tree
Hide file tree
Showing 7 changed files with 47 additions and 21 deletions.
44 changes: 29 additions & 15 deletions integration-tests/docker/test_env/test_env_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
return nil, err
}

//TODO remove after fixing in CTF
// TODO: remove after fixing in CTF
networkConfig.ChainID = int64(en.EthereumChainConfig.ChainID)

if b.hasEVMClient {
Expand Down Expand Up @@ -338,27 +338,41 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
return b.te, nil
}

b.te.rpcProviders = make(map[int64]*test_env.RpcProvider)
networkConfig := networks.MustGetSelectedNetworkConfig(b.testConfig.GetNetworkConfig())[0]
// This has some hidden behavior so I'm not the biggest fan, but it matches expected behavior.
// That is, when we specify we want to run on a live network in our config, we will run on the live network and not bother with a private network.
// Even if we explicitly declare that we want to run on a private network in the test.
// Keeping this a Kludge for now as SETH transition should change all of this anyway.
if len(b.privateEthereumNetworks) == 1 {
b.te.rpcProviders = make(map[int64]*test_env.RpcProvider)
// TODO here we should save the ethereum network config to te.Cfg, but it doesn't exist at this point
// in general it seems we have no methods for saving config to file and we only load it from file
// but I don't know how that config file is to be created or whether anyone ever done that
var rpcProvider test_env.RpcProvider

b.privateEthereumNetworks[0].DockerNetworkNames = []string{b.te.DockerNetwork.Name}
networkConfig, rpcProvider, err = b.te.StartEthereumNetwork(b.privateEthereumNetworks[0])
if err != nil {
return nil, err
if networkConfig.Simulated {
// TODO here we should save the ethereum network config to te.Cfg, but it doesn't exist at this point
// in general it seems we have no methods for saving config to file and we only load it from file
// but I don't know how that config file is to be created or whether anyone ever done that
var rpcProvider test_env.RpcProvider
b.privateEthereumNetworks[0].DockerNetworkNames = []string{b.te.DockerNetwork.Name}
networkConfig, rpcProvider, err = b.te.StartEthereumNetwork(b.privateEthereumNetworks[0])
if err != nil {
return nil, err
}
b.te.rpcProviders[networkConfig.ChainID] = &rpcProvider
b.te.PrivateEthereumConfigs = b.privateEthereumNetworks

b.te.isSimulatedNetwork = true
} else { // Only start and connect to a private network if we are using a private simulated network
b.te.l.Warn().
Str("Network", networkConfig.Name).
Int64("Chain ID", networkConfig.ChainID).
Msg("Private network config provided, but we are running on a live network. Ignoring private network config.")
rpcProvider := test_env.NewRPCProvider(networkConfig.HTTPURLs, networkConfig.URLs, networkConfig.HTTPURLs, networkConfig.URLs)
b.te.rpcProviders[networkConfig.ChainID] = &rpcProvider
b.te.isSimulatedNetwork = false
}
b.te.rpcProviders[networkConfig.ChainID] = &rpcProvider
b.te.PrivateEthereumConfigs = b.privateEthereumNetworks

b.te.isSimulatedNetwork = true
}

if !b.hasSeth && !b.hasEVMClient {
return nil, errors.New("you need to specify, which evm client to use: Seth or EMVClient")
return nil, errors.New("you need to specify, which evm client to use: Seth or EVMClient")
}

if b.hasSeth && b.hasEVMClient {
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ require (
github.com/slack-go/slack v0.12.2
github.com/smartcontractkit/chainlink-automation v1.0.2-0.20240311111125-22812a072c35
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240324144450-2bc22a6738ac
github.com/smartcontractkit/chainlink-testing-framework v1.27.3
github.com/smartcontractkit/chainlink-testing-framework v1.27.6
github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868
github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000
github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1535,8 +1535,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19
github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0 h1:7m9PVtccb8/pvKTXMaGuyceFno1icRyC2SFH7KG7+70=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0/go.mod h1:SZ899lZYQ0maUulWbZg+SWqabHQ1wTbyk3jT8wJfyo8=
github.com/smartcontractkit/chainlink-testing-framework v1.27.3 h1:rUvTtLXPcxI5s3AttdpWeWTNXAprrjyyBvBEvYZp994=
github.com/smartcontractkit/chainlink-testing-framework v1.27.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
github.com/smartcontractkit/chainlink-testing-framework v1.27.6 h1:w+VS+w4N2HVA0TgJx+0KkXlW0p+Y0gOxgykaQwuHyYI=
github.com/smartcontractkit/chainlink-testing-framework v1.27.6/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=
github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs=
github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo=
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/load/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/slack-go/slack v0.12.2
github.com/smartcontractkit/chainlink-automation v1.0.2-0.20240311111125-22812a072c35
github.com/smartcontractkit/chainlink-common v0.1.7-0.20240324144450-2bc22a6738ac
github.com/smartcontractkit/chainlink-testing-framework v1.27.3
github.com/smartcontractkit/chainlink-testing-framework v1.27.6
github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c
github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8
github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052
Expand Down
4 changes: 2 additions & 2 deletions integration-tests/load/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1518,8 +1518,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19
github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0 h1:7m9PVtccb8/pvKTXMaGuyceFno1icRyC2SFH7KG7+70=
github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0/go.mod h1:SZ899lZYQ0maUulWbZg+SWqabHQ1wTbyk3jT8wJfyo8=
github.com/smartcontractkit/chainlink-testing-framework v1.27.3 h1:rUvTtLXPcxI5s3AttdpWeWTNXAprrjyyBvBEvYZp994=
github.com/smartcontractkit/chainlink-testing-framework v1.27.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
github.com/smartcontractkit/chainlink-testing-framework v1.27.6 h1:w+VS+w4N2HVA0TgJx+0KkXlW0p+Y0gOxgykaQwuHyYI=
github.com/smartcontractkit/chainlink-testing-framework v1.27.6/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA=
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea h1:ZdLmNAfKRjH8AYUvjiiDGUgiWQfq/7iNpxyTkvjx/ko=
github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea/go.mod h1:gCKC9w6XpNk6jm+XIk2psrkkfxhi421N9NSiFceXW88=
github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8=
Expand Down
10 changes: 10 additions & 0 deletions integration-tests/testconfig/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
## Introduction

Final implementation has undergone minor adjustments in comparison to the approach by Adam Hamric, Anindita Ghosh, and Sergey Kudasov stated in the ADR. The primary changes are as follows:

* `TEST_LOG_LEVEL` remains an environment variable, pending the release of version 2.
* `TEST_TYPE` is also kept as an environment variable to facilitate dynamic configuration selection by some tests.
* TOML configuration of Chainlink nodes themselves has not been added, awaiting version 2.
Expand All @@ -15,6 +16,7 @@ The `testconfig` package serves as a centralized resource for accessing configur
## Configuration and Overrides

The order of precedence for overrides is as follows:

* Environment variable `BASE64_CONFIG_OVERRIDE`
* File `overrides.toml`
* Product-specific file, e.g., `[product_name].toml`
Expand Down Expand Up @@ -119,6 +121,7 @@ Finally `default.toml` file is envisioned to contain fundamental and universally
GitHub workflows in this repository have been updated to dynamically generate and utilize base64-encoded TOML configurations derived from user inputs or environment variables. For local execution or remote Kubernetes runners, users must manually supply certain variables, which cannot be embedded in configuration files due to their sensitive or dynamic nature.

Essential variables might include:

* Chainlink image and version
* Test duration for specific tests (e.g., load, soak)
* Configuration specific to Loki (mandatory for certain tests)
Expand All @@ -127,19 +130,25 @@ Essential variables might include:
For local testing, it is advisable to place these variables in the `overrides.toml` file. For Kubernetes or remote runners, the process involves creating a TOML file with the necessary values, encoding it in base64, and setting the result as the `BASE64_CONFIG_OVERRIDE` environment variable.

## Embeded config

Because Go automatically excludes TOML files during the compilation of binaries, we must take deliberate steps to include our configuration files in the compiled binary. This can be accomplished by using a custom build tag `-o embed`. Implementing this tag will incorporate all the default configurations located in the `./testconfig` folder directly into the binary. Therefore, when executing tests from the binary, you'll only need to supply the `overrides.toml` file. This file should list only the settings you wish to modify; all other configurations will be sourced from the embedded configurations. You can access these embedded configurations [here](.integration-tests/testconfig/configs_embed.go).

## To bear in mind

### Validation failures

When the system encounters even a single setting related to a specific product or configuration within the configurations, it triggers a comprehensive validation of the entire configuration for that product. This approach is based on the assumption that if any configuration for a given product is specified, the entire set of configurations for that product must be complete and valid. This is particularly crucial when dealing with the `overrides.toml` file, where it's easy to overlook the need to comment out or adjust values when switching between configurations for different products. Essentially, the presence of any specific configuration detail necessitates that all relevant configurations for that product be fully defined and correct to prevent validation errors.

## Possible nil pointers

If no configuration values are set for a product or its logging parameters, the system won't perform validation checks. This can lead to a 'nil pointer exception' error if you attempt to access a configuration property later on. This situation arises because we use pointers to facilitate optional overrides; accessing an unset (nil) pointer will cause an error. To avoid such issues, especially when general validations might not cover every scenario, it's crucial for users to ensure that all necessary configuration options are explicitly set. Additionally, it's highly recommended to implement test-specific validations to confirm that all required values for a particular test are indeed established. This proactive approach helps prevent runtime errors and ensures smooth test execution.

## Contributing

It's crucial to incorporate all new test configuration settings directly into the TOML configuration files, steering clear of using environment variables for this purpose. Our goal is to centralize all configuration details, including examples, within the same package. This approach simplifies the process of understanding the available configuration options and identifying the appropriate values to use for each setting.

## Reusing TestConfig in other projects

To ensure the cleanliness and simplicity of your project's configuration, it's advised against using the `testconfig` code as a direct library in other projects. The reason is that much of this code is tailored specifically to its current application, which might not align with the requirements of your project. Your project might not necessitate any overrides or could perhaps benefit from a simpler configuration approach.

However, if you find a need to utilize some methods from this project, the recommended practice is to implement the required interfaces within your project's configuration package, rather than directly copying and pasting code. For instance, if you aim to incorporate a setup action similar to the `SetupVRFV2Environment` for VRFv2, like the one shown below:
Expand All @@ -164,5 +173,6 @@ func SetupVRFV2Environment(
You should not replicate the entire `TestConfig` structure. Instead, create an implementation of the `types.VRFv2TestConfig` interface in your project and use that as the parameter. This approach allows you to maintain a streamlined and focused configuration package in your project.
## Known Issues/Limitations
* Duplicate file names in different locations may lead to unpredictable configurations being selected.
* The use of pointer fields for optional configuration elements necessitates careful handling, especially for programmatic modifications, to avoid unintended consequences. The `MustCopy()` function is recommended for creating deep copies of configurations for isolated modifications. Unfortunately some of the custom types are not copied at all, you need to set them manually. It's true for example for `blockchain.StrDuration` type.
2 changes: 2 additions & 0 deletions integration-tests/testconfig/default.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ log_producer_retry_limit=10

[ChainlinkImage]
postgres_version="15.6"
image="public.ecr.aws/chainlink/chainlink"
version="2.9.1"

[Network]
selected_networks=["simulated"]
Expand Down

0 comments on commit 76b72ee

Please sign in to comment.