From 76b72ee4fe48da25b41bbd7ee0be919464e7f16d Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Wed, 27 Mar 2024 08:42:40 -0400 Subject: [PATCH] [TT 1024] Fixes Live Network Smoke Tests (#12603) * [TT-1024] Fix Live Network Smoke Test * Use new CTF * Typo * go mod tidy * Lint --------- Co-authored-by: anirudhwarrier <12178754+anirudhwarrier@users.noreply.github.com> --- .../docker/test_env/test_env_builder.go | 44 ++++++++++++------- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 +- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- integration-tests/testconfig/README.md | 10 +++++ integration-tests/testconfig/default.toml | 2 + 7 files changed, 47 insertions(+), 21 deletions(-) diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go index 07d14f1885c..14a4c1df6fa 100644 --- a/integration-tests/docker/test_env/test_env_builder.go +++ b/integration-tests/docker/test_env/test_env_builder.go @@ -298,7 +298,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { return nil, err } - //TODO remove after fixing in CTF + // TODO: remove after fixing in CTF networkConfig.ChainID = int64(en.EthereumChainConfig.ChainID) if b.hasEVMClient { @@ -338,27 +338,41 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { return b.te, nil } + b.te.rpcProviders = make(map[int64]*test_env.RpcProvider) networkConfig := networks.MustGetSelectedNetworkConfig(b.testConfig.GetNetworkConfig())[0] + // This has some hidden behavior so I'm not the biggest fan, but it matches expected behavior. + // That is, when we specify we want to run on a live network in our config, we will run on the live network and not bother with a private network. + // Even if we explicitly declare that we want to run on a private network in the test. + // Keeping this a Kludge for now as SETH transition should change all of this anyway. if len(b.privateEthereumNetworks) == 1 { - b.te.rpcProviders = make(map[int64]*test_env.RpcProvider) - // TODO here we should save the ethereum network config to te.Cfg, but it doesn't exist at this point - // in general it seems we have no methods for saving config to file and we only load it from file - // but I don't know how that config file is to be created or whether anyone ever done that - var rpcProvider test_env.RpcProvider - - b.privateEthereumNetworks[0].DockerNetworkNames = []string{b.te.DockerNetwork.Name} - networkConfig, rpcProvider, err = b.te.StartEthereumNetwork(b.privateEthereumNetworks[0]) - if err != nil { - return nil, err + if networkConfig.Simulated { + // TODO here we should save the ethereum network config to te.Cfg, but it doesn't exist at this point + // in general it seems we have no methods for saving config to file and we only load it from file + // but I don't know how that config file is to be created or whether anyone ever done that + var rpcProvider test_env.RpcProvider + b.privateEthereumNetworks[0].DockerNetworkNames = []string{b.te.DockerNetwork.Name} + networkConfig, rpcProvider, err = b.te.StartEthereumNetwork(b.privateEthereumNetworks[0]) + if err != nil { + return nil, err + } + b.te.rpcProviders[networkConfig.ChainID] = &rpcProvider + b.te.PrivateEthereumConfigs = b.privateEthereumNetworks + + b.te.isSimulatedNetwork = true + } else { // Only start and connect to a private network if we are using a private simulated network + b.te.l.Warn(). + Str("Network", networkConfig.Name). + Int64("Chain ID", networkConfig.ChainID). + Msg("Private network config provided, but we are running on a live network. Ignoring private network config.") + rpcProvider := test_env.NewRPCProvider(networkConfig.HTTPURLs, networkConfig.URLs, networkConfig.HTTPURLs, networkConfig.URLs) + b.te.rpcProviders[networkConfig.ChainID] = &rpcProvider + b.te.isSimulatedNetwork = false } - b.te.rpcProviders[networkConfig.ChainID] = &rpcProvider - b.te.PrivateEthereumConfigs = b.privateEthereumNetworks - b.te.isSimulatedNetwork = true } if !b.hasSeth && !b.hasEVMClient { - return nil, errors.New("you need to specify, which evm client to use: Seth or EMVClient") + return nil, errors.New("you need to specify, which evm client to use: Seth or EVMClient") } if b.hasSeth && b.hasEVMClient { diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 69ce3dd6191..f5a20ec63f7 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -24,7 +24,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.2-0.20240311111125-22812a072c35 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240324144450-2bc22a6738ac - github.com/smartcontractkit/chainlink-testing-framework v1.27.3 + github.com/smartcontractkit/chainlink-testing-framework v1.27.6 github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 9a11c7f2de1..09f4e0ce1af 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1535,8 +1535,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0 h1:7m9PVtccb8/pvKTXMaGuyceFno1icRyC2SFH7KG7+70= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0/go.mod h1:SZ899lZYQ0maUulWbZg+SWqabHQ1wTbyk3jT8wJfyo8= -github.com/smartcontractkit/chainlink-testing-framework v1.27.3 h1:rUvTtLXPcxI5s3AttdpWeWTNXAprrjyyBvBEvYZp994= -github.com/smartcontractkit/chainlink-testing-framework v1.27.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= +github.com/smartcontractkit/chainlink-testing-framework v1.27.6 h1:w+VS+w4N2HVA0TgJx+0KkXlW0p+Y0gOxgykaQwuHyYI= +github.com/smartcontractkit/chainlink-testing-framework v1.27.6/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868/go.mod h1:Kn1Hape05UzFZ7bOUnm3GVsHzP0TNrVmpfXYNHdqGGs= github.com/smartcontractkit/go-plugin v0.0.0-20240208201424-b3b91517de16 h1:TFe+FvzxClblt6qRfqEhUfa4kFQx5UobuoFGO2W4mMo= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 35bc1ba19aa..3f6ca2a2a93 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -16,7 +16,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-automation v1.0.2-0.20240311111125-22812a072c35 github.com/smartcontractkit/chainlink-common v0.1.7-0.20240324144450-2bc22a6738ac - github.com/smartcontractkit/chainlink-testing-framework v1.27.3 + github.com/smartcontractkit/chainlink-testing-framework v1.27.6 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 github.com/smartcontractkit/libocr v0.0.0-20240326191951-2bbe9382d052 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 76826ba53ef..2c338ce87be 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1518,8 +1518,8 @@ github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19 github.com/smartcontractkit/chainlink-solana v1.0.3-0.20240216142700-c5869534c19e/go.mod h1:JiykN+8W5TA4UD2ClrzQCVvcH3NcyLEVv7RwY0busrw= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0 h1:7m9PVtccb8/pvKTXMaGuyceFno1icRyC2SFH7KG7+70= github.com/smartcontractkit/chainlink-starknet/relayer v0.0.1-beta-test.0.20240213121419-1272736c2ac0/go.mod h1:SZ899lZYQ0maUulWbZg+SWqabHQ1wTbyk3jT8wJfyo8= -github.com/smartcontractkit/chainlink-testing-framework v1.27.3 h1:rUvTtLXPcxI5s3AttdpWeWTNXAprrjyyBvBEvYZp994= -github.com/smartcontractkit/chainlink-testing-framework v1.27.3/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= +github.com/smartcontractkit/chainlink-testing-framework v1.27.6 h1:w+VS+w4N2HVA0TgJx+0KkXlW0p+Y0gOxgykaQwuHyYI= +github.com/smartcontractkit/chainlink-testing-framework v1.27.6/go.mod h1:jN+HgXbriq6fKRlIqLw9F3I81aYImV6kBJkIfz0mdIA= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea h1:ZdLmNAfKRjH8AYUvjiiDGUgiWQfq/7iNpxyTkvjx/ko= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240227164431-18a7065e23ea/go.mod h1:gCKC9w6XpNk6jm+XIk2psrkkfxhi421N9NSiFceXW88= github.com/smartcontractkit/chainlink-vrf v0.0.0-20231120191722-fef03814f868 h1:FFdvEzlYwcuVHkdZ8YnZR/XomeMGbz5E2F2HZI3I3w8= diff --git a/integration-tests/testconfig/README.md b/integration-tests/testconfig/README.md index a86531551ff..fd985c316e2 100644 --- a/integration-tests/testconfig/README.md +++ b/integration-tests/testconfig/README.md @@ -3,6 +3,7 @@ ## Introduction Final implementation has undergone minor adjustments in comparison to the approach by Adam Hamric, Anindita Ghosh, and Sergey Kudasov stated in the ADR. The primary changes are as follows: + * `TEST_LOG_LEVEL` remains an environment variable, pending the release of version 2. * `TEST_TYPE` is also kept as an environment variable to facilitate dynamic configuration selection by some tests. * TOML configuration of Chainlink nodes themselves has not been added, awaiting version 2. @@ -15,6 +16,7 @@ The `testconfig` package serves as a centralized resource for accessing configur ## Configuration and Overrides The order of precedence for overrides is as follows: + * Environment variable `BASE64_CONFIG_OVERRIDE` * File `overrides.toml` * Product-specific file, e.g., `[product_name].toml` @@ -119,6 +121,7 @@ Finally `default.toml` file is envisioned to contain fundamental and universally GitHub workflows in this repository have been updated to dynamically generate and utilize base64-encoded TOML configurations derived from user inputs or environment variables. For local execution or remote Kubernetes runners, users must manually supply certain variables, which cannot be embedded in configuration files due to their sensitive or dynamic nature. Essential variables might include: + * Chainlink image and version * Test duration for specific tests (e.g., load, soak) * Configuration specific to Loki (mandatory for certain tests) @@ -127,19 +130,25 @@ Essential variables might include: For local testing, it is advisable to place these variables in the `overrides.toml` file. For Kubernetes or remote runners, the process involves creating a TOML file with the necessary values, encoding it in base64, and setting the result as the `BASE64_CONFIG_OVERRIDE` environment variable. ## Embeded config + Because Go automatically excludes TOML files during the compilation of binaries, we must take deliberate steps to include our configuration files in the compiled binary. This can be accomplished by using a custom build tag `-o embed`. Implementing this tag will incorporate all the default configurations located in the `./testconfig` folder directly into the binary. Therefore, when executing tests from the binary, you'll only need to supply the `overrides.toml` file. This file should list only the settings you wish to modify; all other configurations will be sourced from the embedded configurations. You can access these embedded configurations [here](.integration-tests/testconfig/configs_embed.go). ## To bear in mind + ### Validation failures + When the system encounters even a single setting related to a specific product or configuration within the configurations, it triggers a comprehensive validation of the entire configuration for that product. This approach is based on the assumption that if any configuration for a given product is specified, the entire set of configurations for that product must be complete and valid. This is particularly crucial when dealing with the `overrides.toml` file, where it's easy to overlook the need to comment out or adjust values when switching between configurations for different products. Essentially, the presence of any specific configuration detail necessitates that all relevant configurations for that product be fully defined and correct to prevent validation errors. ## Possible nil pointers + If no configuration values are set for a product or its logging parameters, the system won't perform validation checks. This can lead to a 'nil pointer exception' error if you attempt to access a configuration property later on. This situation arises because we use pointers to facilitate optional overrides; accessing an unset (nil) pointer will cause an error. To avoid such issues, especially when general validations might not cover every scenario, it's crucial for users to ensure that all necessary configuration options are explicitly set. Additionally, it's highly recommended to implement test-specific validations to confirm that all required values for a particular test are indeed established. This proactive approach helps prevent runtime errors and ensures smooth test execution. ## Contributing + It's crucial to incorporate all new test configuration settings directly into the TOML configuration files, steering clear of using environment variables for this purpose. Our goal is to centralize all configuration details, including examples, within the same package. This approach simplifies the process of understanding the available configuration options and identifying the appropriate values to use for each setting. ## Reusing TestConfig in other projects + To ensure the cleanliness and simplicity of your project's configuration, it's advised against using the `testconfig` code as a direct library in other projects. The reason is that much of this code is tailored specifically to its current application, which might not align with the requirements of your project. Your project might not necessitate any overrides or could perhaps benefit from a simpler configuration approach. However, if you find a need to utilize some methods from this project, the recommended practice is to implement the required interfaces within your project's configuration package, rather than directly copying and pasting code. For instance, if you aim to incorporate a setup action similar to the `SetupVRFV2Environment` for VRFv2, like the one shown below: @@ -164,5 +173,6 @@ func SetupVRFV2Environment( You should not replicate the entire `TestConfig` structure. Instead, create an implementation of the `types.VRFv2TestConfig` interface in your project and use that as the parameter. This approach allows you to maintain a streamlined and focused configuration package in your project. ## Known Issues/Limitations + * Duplicate file names in different locations may lead to unpredictable configurations being selected. * The use of pointer fields for optional configuration elements necessitates careful handling, especially for programmatic modifications, to avoid unintended consequences. The `MustCopy()` function is recommended for creating deep copies of configurations for isolated modifications. Unfortunately some of the custom types are not copied at all, you need to set them manually. It's true for example for `blockchain.StrDuration` type. diff --git a/integration-tests/testconfig/default.toml b/integration-tests/testconfig/default.toml index 7e518c1fbf2..d334eaa3c13 100644 --- a/integration-tests/testconfig/default.toml +++ b/integration-tests/testconfig/default.toml @@ -8,6 +8,8 @@ log_producer_retry_limit=10 [ChainlinkImage] postgres_version="15.6" +image="public.ecr.aws/chainlink/chainlink" +version="2.9.1" [Network] selected_networks=["simulated"]