even better config readme, docs for config public methods

config/README.md

@@ -21,6 +21,8 @@ It's up to the user to provide a way to read the config from file and unmarshal
 
 `Validate()` should be used to ensure that the config is valid. Some of the building blocks have also a `Default()` method that can be used to get default values.
 
+Also you might find `BytesToAnyTomlStruct(logger zerolog.Logger, filename, configurationName string, target any, content []byte) error` utility method useful for unmarshalling TOMLs read from env var or files into a struct
+
 ## Working example
 
 For a full working example making use of all the building blocks see [testconfig.go](../config/examples/testconfig.go). It provides methods for reading TOML, applying overrides and validating non-empty config blocks. It supports 4 levels of overrides, in order of precedence:
@@ -44,6 +46,8 @@ type NetworkConfig struct {
 	// list of networks that should be used for testing
 	SelectedNetworks []string            `toml:"selected_networks"`
 	// map of network name to RPC endpoints where key is network name and value is a list of RPC HTTP endpoints
+	// it doesn't matter if you use `arbitrum_sepolia` or `ARBITRUM_SEPOLIA` or even `arbitrum_SEPOLIA` as key
+	// as all keys will be uppercased when loading the Default config
 	RpcHttpUrls      map[string][]string `toml:"RpcHttpUrls"`
 	// map of network name to RPC endpoints where key is network name and value is a list of RPC WS endpoints
 	RpcWsUrls        map[string][]string `toml:"RpcWsUrls"`
@@ -56,8 +60,46 @@ func (n *NetworkConfig) Default() error {
 }
 ```
 
+Sample TOML config:
+```toml
+selected_networks = ["arbitrum_goerli", "optimism_goerli"]
+
+[RpcHttpUrls]
+arbitrum_goerli = ["https://devnet-2.mt/ABC/rpc/"]
+
+[WalletKeys]
+arbitrum_goerli = ["1810868fc221b9f50b5b3e0186d8a5f343f892e51ce12a9e818f936ec0b651ed"]
+optimism_goerli = ["1810868fc221b9f50b5b3e0186d8a5f343f892e51ce12a9e818f936ec0b651ed"]
+```
+
+If your config struct looks like that:
+```golang
+
+type TestConfig struct {
+	Network *ctf_config.NetworkConfig `toml:"Network"`
+}
+```
+
+then your TOML file should look like that:
+```toml
+[Network]
+selected_networks = ["arbitrum_goerli"]
+
+[Network.RpcHttpUrls]
+arbitrum_goerli = ["https://devnet-2.mt/ABC/rpc/"]
+
+[Network.RpcWsUrls]
+arbitrum_goerli = ["ws://devnet-2.mt/ABC/rpc/"]
+
+[Network.WalletKeys]
+arbitrum_goerli = ["1810868fc221b9f50b5b3e0186d8a5f343f892e51ce12a9e818f936ec0b651ed"]
+```
+
+
 It not only stores the configuration of selected networks and RPC endpoints and wallet keys, but via `Default()` method provides a way to read from env var `BASE64_NETWORK_CONFIG` a base64-ed configuration of RPC endpoints and wallet keys. This could prove useful in the CI, where we could store as a secret a default configuration of stable endpoints, so that when we run a test job all that we have to provide is the network name and nothing more as it's pretty tedious, especially for on-demand jobs, to have to pass the whole RPC/wallet configuration every time you run it.
 
+If in your product config you want to support case-insensitive network names and map keys remember to run `NetworkConfig.UpperCaseNetworkNames()` on your config before using it.
+
 ## Providing custom values in the CI
 
 Up to this point when we wanted to modify some dynamic tests parameters in the CI we would simply set env vars. That approach won't work anymore. The way to go around it is to build a TOML file, `base64` it, mask it and then set is as `BASE64_CONFIG_OVERRIDE` env var that will be read by tests. Here's an example of a working snippet of how that could look:

config/chainlink_image.go

@@ -9,6 +9,8 @@ type ChainlinkImageConfig struct {
 	Version *string `toml:"version"`
 }
 
+// Validate checks that the chainlink image config is valid, which means that
+// both image and version are set and non-empty
 func (c *ChainlinkImageConfig) Validate() error {
 	if c.Image == nil || *c.Image == "" {
 		return errors.New("chainlink image name must be set")

config/logging.go

@@ -16,6 +16,7 @@ type LoggingConfig struct {
 	LogStream      *LogStreamConfig `toml:"LogStream"`
 }
 
+// Validate executes config validation for LogStream, Grafana and Loki
 func (l *LoggingConfig) Validate() error {
 	if l.LogStream != nil {
 		if err := l.LogStream.Validate(); err != nil {
@@ -29,36 +30,11 @@ func (l *LoggingConfig) Validate() error {
 		}
 	}
 
-	return nil
-}
-
-func (l *LoggingConfig) ApplyOverrides(from *LoggingConfig) error {
-	if from == nil {
-		return nil
-	}
-	if from.TestLogCollect != nil {
-		l.TestLogCollect = from.TestLogCollect
-	}
-	if from.LogStream != nil && l.LogStream == nil {
-		l.LogStream = from.LogStream
-	} else if from.LogStream != nil && l.LogStream != nil {
-		if err := l.LogStream.ApplyOverrides(from.LogStream); err != nil {
-			return fmt.Errorf("error applying overrides to log stream config: %w", err)
-		}
-	}
-	if from.Loki != nil && l.Loki == nil {
-		l.Loki = from.Loki
-	} else if from.Loki != nil && l.Loki != nil {
-		if err := l.Loki.ApplyOverrides(from.Loki); err != nil {
-			return fmt.Errorf("error applying overrides to loki config: %w", err)
+	if l.Loki != nil {
+		if err := l.Loki.Validate(); err != nil {
+			return fmt.Errorf("invalid loki config: %w", err)
 		}
 	}
-	if from.Grafana != nil {
-		l.Grafana = from.Grafana
-	}
-	if from.RunId != nil {
-		l.RunId = from.RunId
-	}
 
 	return nil
 }
@@ -69,23 +45,8 @@ type LogStreamConfig struct {
 	LogProducerRetryLimit *uint                   `toml:"log_producer_retry_limit"`
 }
 
-func (l *LogStreamConfig) ApplyOverrides(from *LogStreamConfig) error {
-	if from == nil {
-		return nil
-	}
-	if from.LogTargets != nil {
-		l.LogTargets = from.LogTargets
-	}
-	if from.LogProducerTimeout != nil {
-		l.LogProducerTimeout = from.LogProducerTimeout
-	}
-	if from.LogProducerRetryLimit != nil {
-		l.LogProducerRetryLimit = from.LogProducerRetryLimit
-	}
-
-	return nil
-}
-
+// Validate checks that the log stream config is valid, which means that
+// log targets are valid and log producer timeout is greater than 0
 func (l *LogStreamConfig) Validate() error {
 	if len(l.LogTargets) > 0 {
 		for _, target := range l.LogTargets {
@@ -110,28 +71,16 @@ type LokiConfig struct {
 	BasicAuth *string `toml:"basic_auth"`
 }
 
+// Validate checks that the loki config is valid, which means that
+// endpoint is a valid URL and tenant id is not empty
 func (l *LokiConfig) Validate() error {
 	if l.Endpoint != nil {
 		if !net.IsValidURL(*l.Endpoint) {
 			return fmt.Errorf("invalid loki endpoint %s", *l.Endpoint)
 		}
 	}
-
-	return nil
-}
-
-func (l *LokiConfig) ApplyOverrides(from *LokiConfig) error {
-	if from == nil {
-		return nil
-	}
-	if from.TenantId != nil {
-		l.TenantId = from.TenantId
-	}
-	if from.Endpoint != nil {
-		l.Endpoint = from.Endpoint
-	}
-	if from.BasicAuth != nil {
-		l.BasicAuth = from.BasicAuth
+	if l.TenantId == nil || *l.TenantId == "" {
+		return errors.New("loki tenant id must be set")
 	}
 
 	return nil
@@ -143,6 +92,9 @@ type GrafanaConfig struct {
 	BearerToken  *string `toml:"bearer_token"`
 }
 
+// Validate checks that the grafana config is valid, which means that
+// base url is a valid URL and dashboard url and bearer token are not empty
+// but that only applies if they are set
 func (c *GrafanaConfig) Validate() error {
 	if c.BaseUrl != nil {
 		if !net.IsValidURL(*c.BaseUrl) {

config/network.go

@@ -50,7 +50,7 @@ func (n *NetworkConfig) applyDecoded(configDecoded string) error {
 
 	cfg.UpperCaseNetworkNames()
 
-	err = n.ApplyDefaults(&cfg)
+	err = n.applyDefaults(&cfg)
 	if err != nil {
 		return fmt.Errorf("error applying overrides from decoded network config file to config: %w", err)
 	}
@@ -71,6 +71,9 @@ func (n *NetworkConfig) applyBase64Enconded(configEncoded string) error {
 	return n.applyDecoded(string(decoded))
 }
 
+// Validate checks if all required fields are set, meaning that there must be at least
+// 1 selected network and unless it's a simulated network, there must be at least 1
+// rpc endpoint for HTTP and WS and 1 private key for funding wallet
 func (n *NetworkConfig) Validate() error {
 	if len(n.SelectedNetworks) == 0 {
 		return errors.New("selected_networks must be set")
@@ -98,6 +101,8 @@ func (n *NetworkConfig) Validate() error {
 	return nil
 }
 
+// UpperCaseNetworkNames converts all network name keys for wallet keys, rpc endpoints maps and
+// selected network slice to upper case
 func (n *NetworkConfig) UpperCaseNetworkNames() {
 	var upperCaseMapKeys = func(m map[string][]string) {
 		newMap := make(map[string][]string)
@@ -119,7 +124,7 @@ func (n *NetworkConfig) UpperCaseNetworkNames() {
 	}
 }
 
-func (n *NetworkConfig) ApplyDefaults(defaults *NetworkConfig) error {
+func (n *NetworkConfig) applyDefaults(defaults *NetworkConfig) error {
 	if defaults == nil {
 		return nil
 	}
@@ -167,6 +172,10 @@ func (n *NetworkConfig) ApplyDefaults(defaults *NetworkConfig) error {
 	return nil
 }
 
+// Default applies default values to the network config after reading it
+// from BASE64_NETWORK_CONFIG env var. It will only fill in the gaps, not override
+// meaning that if you provided WS RPC endpoint in your network config, but not the
+// HTTP one, then only HTTP will be taken from default config (provided it's there)
 func (n *NetworkConfig) Default() error {
 	return n.applySecrets()
 }

config/overrides.go

@@ -4,6 +4,8 @@ import (
 	"dario.cat/mergo"
 )
 
+// MightConfigOverrideChainlinkVersion will override the chainlink image and version
+// in property maps from the passed config. It will panic if the config is nil.
 func MustConfigOverrideChainlinkVersion(config *ChainlinkImageConfig, target interface{}) {
 	if config == nil {
 		panic("[ChainlinkImageConfig] must be present")
@@ -22,6 +24,9 @@ func MustConfigOverrideChainlinkVersion(config *ChainlinkImageConfig, target int
 	}
 }
 
+// MightConfigOverridePyroscope will override the pyroscope config in property maps
+// from the passed config. If the config is nil, or the enabled flag is not set, or
+// the key is not set, then this function will do nothing.
 func MightConfigOverridePyroscopeKey(config *PyroscopeConfig, target interface{}) {
 	if config == nil || (config.Enabled == nil || !*config.Enabled) || (config.Key == nil || *config.Key == "") {
 		return

config/pyroscope.go

@@ -14,6 +14,9 @@ type PyroscopeConfig struct {
 	Environment *string `toml:"environment"`
 }
 
+// Validate checks that the pyroscope config is valid, which means that
+// server url, environment and key are set and non-empty, but only if
+// pyroscope is enabled
 func (c *PyroscopeConfig) Validate() error {
 	if c.Enabled != nil && *c.Enabled {
 		if c.ServerUrl == nil {