From e269916e098308587654c55f60d3bf20449b7008 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Fri, 9 Aug 2024 17:33:33 +0200 Subject: [PATCH 01/16] support default config file overrides --- core/chains/evm/config/toml/defaults.go | 71 +++ main_test.go | 4 + .../node/validate/defaults-override.txtar | 405 ++++++++++++++++++ 3 files changed, 480 insertions(+) create mode 100644 testdata/scripts/node/validate/defaults-override.txtar diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index c3f087da8c5..a6e0b20e7d3 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -3,7 +3,10 @@ package toml import ( "bytes" "embed" + "fmt" + "io" "log" + "os" "path/filepath" "slices" "strings" @@ -15,17 +18,23 @@ import ( ) var ( + CUSTOM_DEFAULTS_ENV_KEY = "CL_CHAIN_DEFAULTS" + //go:embed defaults/*.toml defaultsFS embed.FS fallback Chain defaults = map[string]Chain{} defaultNames = map[string]string{} + customDefaults = map[string]Chain{} + // DefaultIDs is the set of chain ids which have defaults. DefaultIDs []*big.Big ) func init() { + // read the defaults first + fes, err := defaultsFS.ReadDir("defaults") if err != nil { log.Fatalf("failed to read defaults/: %v", err) @@ -65,6 +74,65 @@ func init() { slices.SortFunc(DefaultIDs, func(a, b *big.Big) int { return a.Cmp(b) }) + + // read the custom defaults overrides + dir := os.Getenv(CUSTOM_DEFAULTS_ENV_KEY) + if dir == "" { + // short-circuit; no default overrides provided + return + } + + // Read directory contents + entries, err := os.ReadDir(dir) + if err != nil { + fmt.Println("Error reading directory:", err) + return + } + + for _, entry := range entries { + if entry.IsDir() { + // Skip directories + continue + } + + path := dir + "/" + entry.Name() + file, err := os.Open(path) + if err != nil { + fmt.Println("Error opening file:", err) + continue + } + + // Read file contents + b, err := io.ReadAll(file) + if err != nil { + fmt.Println("Error reading file:", err) + file.Close() + continue + } + file.Close() + + var config = struct { + ChainID *big.Big + Chain + }{} + + if err := cconfig.DecodeTOML(bytes.NewReader(b), &config); err != nil { + log.Fatalf("failed to decode %q: %v", path, err) + } + + if config.ChainID == nil { + log.Fatalf("missing ChainID: %s", path) + } + + id := config.ChainID.String() + // these custom defaults are only meant to be overrides on existing chains; + // if the chain does not exist already in defaults error out + if _, ok := defaults[id]; !ok { + log.Fatalf("%q does not contain ChainID: %s", path, id) + } + customDefaults[id] = config.Chain + } + } // DefaultsNamed returns the default Chain values, optionally for the given chainID, as well as a name if the chainID is known. @@ -78,6 +146,9 @@ func DefaultsNamed(chainID *big.Big) (c Chain, name string) { c.SetFrom(&d) name = defaultNames[s] } + if overrides, ok := customDefaults[s]; ok { + c.SetFrom(&overrides) + } return } diff --git a/main_test.go b/main_test.go index 81e056e3b84..419a4af0517 100644 --- a/main_test.go +++ b/main_test.go @@ -42,6 +42,10 @@ func TestMain(m *testing.M) { })) } +// TestScripts walks through the testdata/scripts directory and runs all tests that end in +// .txt or .txtar with the testscripts library. To run an individual test, specify it in the +// -run param of go test without the txtar or txt suffix, like so: +// go test . -run TestScripts/node/validate/default func TestScripts(t *testing.T) { if testing.Short() { t.Skip("skipping testscript") diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar new file mode 100644 index 00000000000..504e890b6b8 --- /dev/null +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -0,0 +1,405 @@ +mkdir default_overrides +env CL_CHAIN_DEFAULTS=default_overrides +exec chainlink node -c config.toml -s secrets.toml validate +cmp stdout out.txt + +-- default_overrides/Ethereum_Mainnet.toml -- +ChainID = '1' +FinalityTagEnabled = true + +-- config.toml -- +Log.Level = 'debug' + +[[EVM]] +ChainID = '1' + +[[EVM.Nodes]] +Name = 'fake' +WSURL = 'wss://foo.bar/ws' +HTTPURL = 'https://foo.bar' + +-- secrets.toml -- +[Database] +URL = 'postgresql://user:pass1234567890abcd@localhost:5432/dbname?sslmode=disable' + +[Password] +Keystore = 'keystore_pass' + +-- out.txt -- +# Secrets: +[Database] +URL = 'xxxxx' +AllowSimplePasswords = false + +[Password] +Keystore = 'xxxxx' + +# Input Configuration: +[Log] +Level = 'debug' + +[[EVM]] +ChainID = '1' + +[[EVM.Nodes]] +Name = 'fake' +WSURL = 'wss://foo.bar/ws' +HTTPURL = 'https://foo.bar' + +# Effective Configuration, with defaults applied: +InsecureFastScrypt = false +RootDir = '~/.chainlink' +ShutdownGracePeriod = '5s' + +[Feature] +FeedsManager = true +LogPoller = false +UICSAKeys = false +CCIP = true + +[Database] +DefaultIdleInTxSessionTimeout = '1h0m0s' +DefaultLockTimeout = '15s' +DefaultQueryTimeout = '10s' +LogQueries = false +MaxIdleConns = 10 +MaxOpenConns = 100 +MigrateOnStartup = true + +[Database.Backup] +Dir = '' +Frequency = '1h0m0s' +Mode = 'none' +OnVersionUpgrade = true + +[Database.Listener] +MaxReconnectDuration = '10m0s' +MinReconnectInterval = '1m0s' +FallbackPollInterval = '30s' + +[Database.Lock] +Enabled = true +LeaseDuration = '10s' +LeaseRefreshInterval = '1s' + +[TelemetryIngress] +UniConn = true +Logging = false +BufferSize = 100 +MaxBatchSize = 50 +SendInterval = '500ms' +SendTimeout = '10s' +UseBatchSend = true + +[AuditLogger] +Enabled = false +ForwardToUrl = '' +JsonWrapperKey = '' +Headers = [] + +[Log] +Level = 'debug' +JSONConsole = false +UnixTS = false + +[Log.File] +Dir = '' +MaxSize = '5.12gb' +MaxAgeDays = 0 +MaxBackups = 1 + +[WebServer] +AuthenticationMethod = 'local' +AllowOrigins = 'http://localhost:3000,http://localhost:6688' +BridgeResponseURL = '' +BridgeCacheTTL = '0s' +HTTPWriteTimeout = '10s' +HTTPPort = 6688 +SecureCookies = true +SessionTimeout = '15m0s' +SessionReaperExpiration = '240h0m0s' +HTTPMaxSize = '32.77kb' +StartTimeout = '15s' +ListenIP = '0.0.0.0' + +[WebServer.LDAP] +ServerTLS = true +SessionTimeout = '15m0s' +QueryTimeout = '2m0s' +BaseUserAttr = 'uid' +BaseDN = '' +UsersDN = 'ou=users' +GroupsDN = 'ou=groups' +ActiveAttribute = '' +ActiveAttributeAllowedValue = '' +AdminUserGroupCN = 'NodeAdmins' +EditUserGroupCN = 'NodeEditors' +RunUserGroupCN = 'NodeRunners' +ReadUserGroupCN = 'NodeReadOnly' +UserApiTokenEnabled = false +UserAPITokenDuration = '240h0m0s' +UpstreamSyncInterval = '0s' +UpstreamSyncRateLimit = '2m0s' + +[WebServer.MFA] +RPID = '' +RPOrigin = '' + +[WebServer.RateLimit] +Authenticated = 1000 +AuthenticatedPeriod = '1m0s' +Unauthenticated = 5 +UnauthenticatedPeriod = '20s' + +[WebServer.TLS] +CertPath = '' +ForceRedirect = false +Host = '' +HTTPSPort = 6689 +KeyPath = '' +ListenIP = '0.0.0.0' + +[JobPipeline] +ExternalInitiatorsEnabled = false +MaxRunDuration = '10m0s' +MaxSuccessfulRuns = 10000 +ReaperInterval = '1h0m0s' +ReaperThreshold = '24h0m0s' +ResultWriteQueueDepth = 100 +VerboseLogging = true + +[JobPipeline.HTTPRequest] +DefaultTimeout = '15s' +MaxSize = '32.77kb' + +[FluxMonitor] +DefaultTransactionQueueDepth = 1 +SimulateTransactions = false + +[OCR2] +Enabled = false +ContractConfirmations = 3 +BlockchainTimeout = '20s' +ContractPollInterval = '1m0s' +ContractSubscribeInterval = '2m0s' +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +KeyBundleID = '0000000000000000000000000000000000000000000000000000000000000000' +CaptureEATelemetry = false +CaptureAutomationCustomTelemetry = true +DefaultTransactionQueueDepth = 1 +SimulateTransactions = false +TraceLogging = false + +[OCR] +Enabled = false +ObservationTimeout = '5s' +BlockchainTimeout = '20s' +ContractPollInterval = '1m0s' +ContractSubscribeInterval = '2m0s' +DefaultTransactionQueueDepth = 1 +KeyBundleID = '0000000000000000000000000000000000000000000000000000000000000000' +SimulateTransactions = false +TransmitterAddress = '' +CaptureEATelemetry = false +TraceLogging = false + +[P2P] +IncomingMessageBufferSize = 10 +OutgoingMessageBufferSize = 10 +PeerID = '' +TraceLogging = false + +[P2P.V2] +Enabled = true +AnnounceAddresses = [] +DefaultBootstrappers = [] +DeltaDial = '15s' +DeltaReconcile = '1m0s' +ListenAddresses = [] + +[Keeper] +DefaultTransactionQueueDepth = 1 +GasPriceBufferPercent = 20 +GasTipCapBufferPercent = 20 +BaseFeeBufferPercent = 20 +MaxGracePeriod = 100 +TurnLookBack = 1000 + +[Keeper.Registry] +CheckGasOverhead = 200000 +PerformGasOverhead = 300000 +MaxPerformDataSize = 5000 +SyncInterval = '30m0s' +SyncUpkeepQueueSize = 10 + +[AutoPprof] +Enabled = false +ProfileRoot = '' +PollInterval = '10s' +GatherDuration = '10s' +GatherTraceDuration = '5s' +MaxProfileSize = '100.00mb' +CPUProfileRate = 1 +MemProfileRate = 1 +BlockProfileRate = 1 +MutexProfileFraction = 1 +MemThreshold = '4.00gb' +GoroutineThreshold = 5000 + +[Pyroscope] +ServerAddress = '' +Environment = 'mainnet' + +[Sentry] +Debug = false +DSN = '' +Environment = '' +Release = '' + +[Insecure] +DevWebServer = false +OCRDevelopmentMode = false +InfiniteDepthQueries = false +DisableRateLimiting = false + +[Tracing] +Enabled = false +CollectorTarget = '' +NodeID = '' +SamplingRatio = 0.0 +Mode = 'tls' +TLSCertPath = '' + +[Mercury] +VerboseLogging = false + +[Mercury.Cache] +LatestReportTTL = '1s' +MaxStaleAge = '1h0m0s' +LatestReportDeadline = '5s' + +[Mercury.TLS] +CertFile = '' + +[Mercury.Transmitter] +TransmitQueueMaxSize = 10000 +TransmitTimeout = '5s' + +[Capabilities] +[Capabilities.Peering] +IncomingMessageBufferSize = 10 +OutgoingMessageBufferSize = 10 +PeerID = '' +TraceLogging = false + +[Capabilities.Peering.V2] +Enabled = false +AnnounceAddresses = [] +DefaultBootstrappers = [] +DeltaDial = '15s' +DeltaReconcile = '1m0s' +ListenAddresses = [] + +[Capabilities.ExternalRegistry] +Address = '' +NetworkID = 'evm' +ChainID = '1' + +[[EVM]] +ChainID = '1' +AutoCreateKey = true +BlockBackfillDepth = 10 +BlockBackfillSkip = false +FinalityDepth = 50 +FinalityTagEnabled = true +LinkContractAddress = '0x514910771AF9Ca656af840dff83E8264EcF986CA' +LogBackfillBatchSize = 1000 +LogPollInterval = '15s' +LogKeepBlocksDepth = 100000 +LogPrunePageSize = 0 +BackupLogPollerBlockDelay = 100 +MinIncomingConfirmations = 3 +MinContractPayment = '0.1 link' +NonceAutoSync = true +NoNewHeadsThreshold = '3m0s' +OperatorFactoryAddress = '0x3E64Cd889482443324F91bFA9c84fE72A511f48A' +RPCDefaultBatchSize = 250 +RPCBlockQueryDelay = 1 +FinalizedBlockOffset = 0 +NoNewFinalizedHeadsThreshold = '9m0s' + +[EVM.Transactions] +ForwardersEnabled = false +MaxInFlight = 16 +MaxQueued = 250 +ReaperInterval = '1h0m0s' +ReaperThreshold = '168h0m0s' +ResendAfterThreshold = '1m0s' + +[EVM.Transactions.AutoPurge] +Enabled = false + +[EVM.BalanceMonitor] +Enabled = true + +[EVM.GasEstimator] +Mode = 'BlockHistory' +PriceDefault = '20 gwei' +PriceMax = '115792089237316195423570985008687907853269984665.640564039457584007913129639935 tether' +PriceMin = '1 gwei' +LimitDefault = 500000 +LimitMax = 500000 +LimitMultiplier = '1' +LimitTransfer = 21000 +BumpMin = '5 gwei' +BumpPercent = 20 +BumpThreshold = 3 +EIP1559DynamicFees = true +FeeCapDefault = '100 gwei' +TipCapDefault = '1 wei' +TipCapMin = '1 wei' + +[EVM.GasEstimator.BlockHistory] +BatchSize = 25 +BlockHistorySize = 4 +CheckInclusionBlocks = 12 +CheckInclusionPercentile = 90 +TransactionPercentile = 50 + +[EVM.HeadTracker] +HistoryDepth = 100 +MaxBufferSize = 3 +SamplingInterval = '1s' +MaxAllowedFinalityDepth = 10000 +FinalityTagBypass = true + +[EVM.NodePool] +PollFailureThreshold = 5 +PollInterval = '10s' +SelectionMode = 'HighestHead' +SyncThreshold = 5 +LeaseDuration = '0s' +NodeIsSyncingEnabled = false +FinalizedBlockPollInterval = '5s' +EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' + +[EVM.OCR] +ContractConfirmations = 4 +ContractTransmitterTransmitTimeout = '10s' +DatabaseTimeout = '10s' +DeltaCOverride = '168h0m0s' +DeltaCJitterOverride = '1h0m0s' +ObservationGracePeriod = '1s' + +[EVM.OCR2] +[EVM.OCR2.Automation] +GasLimit = 10500000 + +[[EVM.Nodes]] +Name = 'fake' +WSURL = 'wss://foo.bar/ws' +HTTPURL = 'https://foo.bar' + +Valid configuration. From b4d777fb39ef94fc8fc222cbfa85b81427b89d5a Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Fri, 9 Aug 2024 18:18:55 +0200 Subject: [PATCH 02/16] lint --- core/chains/evm/config/toml/defaults.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index a6e0b20e7d3..de00f0dbf4c 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -41,17 +41,17 @@ func init() { } for _, fe := range fes { path := filepath.Join("defaults", fe.Name()) - b, err := defaultsFS.ReadFile(path) - if err != nil { - log.Fatalf("failed to read %q: %v", path, err) + b, err2 := defaultsFS.ReadFile(path) + if err2 != nil { + log.Fatalf("failed to read %q: %v", path, err2) } var config = struct { ChainID *big.Big Chain }{} - if err := cconfig.DecodeTOML(bytes.NewReader(b), &config); err != nil { - log.Fatalf("failed to decode %q: %v", path, err) + if err3 := cconfig.DecodeTOML(bytes.NewReader(b), &config); err3 != nil { + log.Fatalf("failed to decode %q: %v", path, err3) } if fe.Name() == "fallback.toml" { if config.ChainID != nil { @@ -132,7 +132,6 @@ func init() { } customDefaults[id] = config.Chain } - } // DefaultsNamed returns the default Chain values, optionally for the given chainID, as well as a name if the chainID is known. From f30500cfbee32dc98fa0fdec411efd3745d34948 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Sat, 10 Aug 2024 13:12:43 +0200 Subject: [PATCH 03/16] adding test for when default is not used --- testdata/scripts/node/validate/defaults-override.txtar | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 504e890b6b8..0a0e7b14dba 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -1,8 +1,14 @@ +# test with default override mkdir default_overrides env CL_CHAIN_DEFAULTS=default_overrides exec chainlink node -c config.toml -s secrets.toml validate cmp stdout out.txt +# remove override, FinalityTagEnabled should no longer match +env CL_CHAIN_DEFAULTS= +exec chainlink node -c config.toml -s secrets.toml validate +! cmp stdout out.txt + -- default_overrides/Ethereum_Mainnet.toml -- ChainID = '1' FinalityTagEnabled = true From 8a8919fb5d69f30102a0d3d8dde20f56f03ac80e Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Sat, 10 Aug 2024 13:58:06 +0200 Subject: [PATCH 04/16] WIP: negative test for when an override is specified for a chain that does not exist in the embedded defaults --- .../validate/defaults-override-invalid.txtar | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 testdata/scripts/node/validate/defaults-override-invalid.txtar diff --git a/testdata/scripts/node/validate/defaults-override-invalid.txtar b/testdata/scripts/node/validate/defaults-override-invalid.txtar new file mode 100644 index 00000000000..68668412588 --- /dev/null +++ b/testdata/scripts/node/validate/defaults-override-invalid.txtar @@ -0,0 +1,34 @@ +# test with default override +mkdir default_overrides +env CL_CHAIN_DEFAULTS=default_overrides +! exec chainlink node -c config.toml -s secrets.toml validate +cmp stderr err.txt + +-- default_overrides/Ethereum_Mainnet.toml -- +ChainID = '1' +FinalityTagEnabled = true + +-- default_overrides/Ethereum_FakeChain.toml -- +ChainID = '999' +FinalityTagEnabled = true + +-- config.toml -- +Log.Level = 'debug' + +[[EVM]] +ChainID = '1' + +[[EVM.Nodes]] +Name = 'fake' +WSURL = 'wss://foo.bar/ws' +HTTPURL = 'https://foo.bar' + +-- secrets.toml -- +[Database] +URL = 'postgresql://user:pass1234567890abcd@localhost:5432/dbname?sslmode=disable' + +[Password] +Keystore = 'keystore_pass' + +-- err.txt -- +does not contain ChainID: 999 \ No newline at end of file From d98d085b96908d829f9d31065aa4d229772a95d9 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Thu, 15 Aug 2024 19:41:50 -0400 Subject: [PATCH 05/16] using env package --- core/chains/evm/config/toml/defaults.go | 4 ++-- core/config/env/env.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index de00f0dbf4c..731d978cca6 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -15,10 +15,10 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/chaintype" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" + "github.com/smartcontractkit/chainlink/v2/core/config/env" ) var ( - CUSTOM_DEFAULTS_ENV_KEY = "CL_CHAIN_DEFAULTS" //go:embed defaults/*.toml defaultsFS embed.FS @@ -76,7 +76,7 @@ func init() { }) // read the custom defaults overrides - dir := os.Getenv(CUSTOM_DEFAULTS_ENV_KEY) + dir := os.Getenv(string(env.CustomDefaultsEnvKey)) if dir == "" { // short-circuit; no default overrides provided return diff --git a/core/config/env/env.go b/core/config/env/env.go index 0ebfc357bf3..7d7757a22a9 100644 --- a/core/config/env/env.go +++ b/core/config/env/env.go @@ -20,6 +20,7 @@ var ( ThresholdKeyShare = Secret("CL_THRESHOLD_KEY_SHARE") // Migrations env vars EVMChainIDNotNullMigration0195 = "CL_EVM_CHAINID_NOT_NULL_MIGRATION_0195" + CustomDefaultsEnvKey = Var("CL_CHAIN_DEFAULTS") ) // LOOPP commands and vars From eab01f70c452f7513a4444914159a6247d7b42ee Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Fri, 16 Aug 2024 12:54:05 -0400 Subject: [PATCH 06/16] relaxing strict override restriction, and improving log line details --- core/chains/evm/config/toml/defaults.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index 731d978cca6..40664bb34c3 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -3,7 +3,6 @@ package toml import ( "bytes" "embed" - "fmt" "io" "log" "os" @@ -85,7 +84,7 @@ func init() { // Read directory contents entries, err := os.ReadDir(dir) if err != nil { - fmt.Println("Error reading directory:", err) + log.Printf("error reading custom defaults override directory: %v", err) return } @@ -98,14 +97,14 @@ func init() { path := dir + "/" + entry.Name() file, err := os.Open(path) if err != nil { - fmt.Println("Error opening file:", err) + log.Printf("error opening file (name: %v) in custom defaults override directory: %w", entry.Name(), err) continue } // Read file contents b, err := io.ReadAll(file) if err != nil { - fmt.Println("Error reading file:", err) + log.Printf("error reading file (name: %v) contents in custom defaults override directory: %w", entry.Name(), err) file.Close() continue } @@ -117,19 +116,14 @@ func init() { }{} if err := cconfig.DecodeTOML(bytes.NewReader(b), &config); err != nil { - log.Fatalf("failed to decode %q: %v", path, err) + log.Fatalf("failed to decode %q in custom defaults override directory: %v", path, err) } if config.ChainID == nil { - log.Fatalf("missing ChainID: %s", path) + log.Fatalf("missing ChainID in: %s in custom defaults override directory. exiting", path) } id := config.ChainID.String() - // these custom defaults are only meant to be overrides on existing chains; - // if the chain does not exist already in defaults error out - if _, ok := defaults[id]; !ok { - log.Fatalf("%q does not contain ChainID: %s", path, id) - } customDefaults[id] = config.Chain } } From b77210b263b3e51cf0ee626823d70ab4df21ffda Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Fri, 16 Aug 2024 17:59:51 -0400 Subject: [PATCH 07/16] lint + CustomDefaultsEnvKey --> CustomDefaults --- core/chains/evm/config/toml/defaults.go | 6 +++--- core/config/env/env.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index 40664bb34c3..3f16f7062c0 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -75,7 +75,7 @@ func init() { }) // read the custom defaults overrides - dir := os.Getenv(string(env.CustomDefaultsEnvKey)) + dir := env.CustomDefaults.Get() if dir == "" { // short-circuit; no default overrides provided return @@ -97,14 +97,14 @@ func init() { path := dir + "/" + entry.Name() file, err := os.Open(path) if err != nil { - log.Printf("error opening file (name: %v) in custom defaults override directory: %w", entry.Name(), err) + log.Printf("error opening file (name: %v) in custom defaults override directory: %v", entry.Name(), err) continue } // Read file contents b, err := io.ReadAll(file) if err != nil { - log.Printf("error reading file (name: %v) contents in custom defaults override directory: %w", entry.Name(), err) + log.Printf("error reading file (name: %v) contents in custom defaults override directory: %v", entry.Name(), err) file.Close() continue } diff --git a/core/config/env/env.go b/core/config/env/env.go index 7d7757a22a9..c34cd7f4f5e 100644 --- a/core/config/env/env.go +++ b/core/config/env/env.go @@ -20,7 +20,7 @@ var ( ThresholdKeyShare = Secret("CL_THRESHOLD_KEY_SHARE") // Migrations env vars EVMChainIDNotNullMigration0195 = "CL_EVM_CHAINID_NOT_NULL_MIGRATION_0195" - CustomDefaultsEnvKey = Var("CL_CHAIN_DEFAULTS") + CustomDefaults = Var("CL_CHAIN_DEFAULTS") ) // LOOPP commands and vars From 3f9c8e8a14045f067db91f6946585cad87d2d276 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Mon, 19 Aug 2024 14:08:10 -0400 Subject: [PATCH 08/16] removing invalid test --- .../validate/defaults-override-invalid.txtar | 34 ------------------- 1 file changed, 34 deletions(-) delete mode 100644 testdata/scripts/node/validate/defaults-override-invalid.txtar diff --git a/testdata/scripts/node/validate/defaults-override-invalid.txtar b/testdata/scripts/node/validate/defaults-override-invalid.txtar deleted file mode 100644 index 68668412588..00000000000 --- a/testdata/scripts/node/validate/defaults-override-invalid.txtar +++ /dev/null @@ -1,34 +0,0 @@ -# test with default override -mkdir default_overrides -env CL_CHAIN_DEFAULTS=default_overrides -! exec chainlink node -c config.toml -s secrets.toml validate -cmp stderr err.txt - --- default_overrides/Ethereum_Mainnet.toml -- -ChainID = '1' -FinalityTagEnabled = true - --- default_overrides/Ethereum_FakeChain.toml -- -ChainID = '999' -FinalityTagEnabled = true - --- config.toml -- -Log.Level = 'debug' - -[[EVM]] -ChainID = '1' - -[[EVM.Nodes]] -Name = 'fake' -WSURL = 'wss://foo.bar/ws' -HTTPURL = 'https://foo.bar' - --- secrets.toml -- -[Database] -URL = 'postgresql://user:pass1234567890abcd@localhost:5432/dbname?sslmode=disable' - -[Password] -Keystore = 'keystore_pass' - --- err.txt -- -does not contain ChainID: 999 \ No newline at end of file From 376c677e0aa41e2b0feac496236f326d482b73ee Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Thu, 22 Aug 2024 15:01:33 -0400 Subject: [PATCH 09/16] crash the node if can't read or load all files from overriden defaults --- core/chains/evm/config/toml/defaults.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index 3f16f7062c0..ce8811dbdca 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -84,7 +84,7 @@ func init() { // Read directory contents entries, err := os.ReadDir(dir) if err != nil { - log.Printf("error reading custom defaults override directory: %v", err) + log.Fatalf("error reading custom defaults override directory: %v", err) return } @@ -97,14 +97,14 @@ func init() { path := dir + "/" + entry.Name() file, err := os.Open(path) if err != nil { - log.Printf("error opening file (name: %v) in custom defaults override directory: %v", entry.Name(), err) + log.Fatalf("error opening file (name: %v) in custom defaults override directory: %v", entry.Name(), err) continue } // Read file contents b, err := io.ReadAll(file) if err != nil { - log.Printf("error reading file (name: %v) contents in custom defaults override directory: %v", entry.Name(), err) + log.Fatalf("error reading file (name: %v) contents in custom defaults override directory: %v", entry.Name(), err) file.Close() continue } From 4bbfa5022301b41f74fcf7847562d1b94e78c702 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Fri, 23 Aug 2024 10:18:35 -0400 Subject: [PATCH 10/16] accounting for Fatalf in control flow --- core/chains/evm/config/toml/defaults.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index ce8811dbdca..492f8325575 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -98,17 +98,14 @@ func init() { file, err := os.Open(path) if err != nil { log.Fatalf("error opening file (name: %v) in custom defaults override directory: %v", entry.Name(), err) - continue } // Read file contents b, err := io.ReadAll(file) + file.Close() if err != nil { log.Fatalf("error reading file (name: %v) contents in custom defaults override directory: %v", entry.Name(), err) - file.Close() - continue } - file.Close() var config = struct { ChainID *big.Big From a6c89f61961ec6fcc05acf167560a679a9ea68cf Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Wed, 28 Aug 2024 19:00:51 -0400 Subject: [PATCH 11/16] evm defaults look for evm suffix. Duplicate chain IDs fail validation --- core/chains/evm/config/toml/defaults.go | 16 +++++++++---- .../node/validate/defaults-override.txtar | 24 ++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/core/chains/evm/config/toml/defaults.go b/core/chains/evm/config/toml/defaults.go index 492f8325575..4e1ebfb34df 100644 --- a/core/chains/evm/config/toml/defaults.go +++ b/core/chains/evm/config/toml/defaults.go @@ -3,6 +3,7 @@ package toml import ( "bytes" "embed" + "fmt" "io" "log" "os" @@ -81,10 +82,13 @@ func init() { return } - // Read directory contents - entries, err := os.ReadDir(dir) + // use evm overrides specifically + evmDir := fmt.Sprintf("%s/evm", dir) + + // Read directory contents for evm only + entries, err := os.ReadDir(evmDir) if err != nil { - log.Fatalf("error reading custom defaults override directory: %v", err) + log.Fatalf("error reading evm custom defaults override directory: %v", err) return } @@ -94,7 +98,7 @@ func init() { continue } - path := dir + "/" + entry.Name() + path := evmDir + "/" + entry.Name() file, err := os.Open(path) if err != nil { log.Fatalf("error opening file (name: %v) in custom defaults override directory: %v", entry.Name(), err) @@ -121,6 +125,10 @@ func init() { } id := config.ChainID.String() + + if _, ok := customDefaults[id]; ok { + log.Fatalf("%q contains duplicate ChainID: %s", path, id) + } customDefaults[id] = config.Chain } } diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 0a0e7b14dba..7a0b6de6adb 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -1,5 +1,6 @@ # test with default override mkdir default_overrides +mkdir default_overrides/evm env CL_CHAIN_DEFAULTS=default_overrides exec chainlink node -c config.toml -s secrets.toml validate cmp stdout out.txt @@ -9,10 +10,31 @@ env CL_CHAIN_DEFAULTS= exec chainlink node -c config.toml -s secrets.toml validate ! cmp stdout out.txt --- default_overrides/Ethereum_Mainnet.toml -- +# overrides outside of evm suffix, FinalityTagEnabled should not match as override is not applied +mkdir default_overrides2 +env CL_CHAIN_DEFAULTS=default_overrides2 +! exec chainlink node -c config.toml -s secrets.toml validate + +# duplicate chain IDs +mkdir default_overrides3 +env CL_CHAIN_DEFAULTS=default_overrides3 +! exec chainlink node -c config.toml -s secrets.toml validate + + +-- default_overrides/evm/Ethereum_Mainnet.toml -- +ChainID = '1' +FinalityTagEnabled = true + +-- default_overrides2/Ethereum_Mainnet.toml -- ChainID = '1' FinalityTagEnabled = true +-- default_overrides3/Ethereum_Mainnet.toml -- +ChainID = '1' + +-- default_overrides3/Ethereum_Testnet.toml -- +ChainID = '1' + -- config.toml -- Log.Level = 'debug' From 0a72af6ea5694df88336eef10f89fc30a855e5b0 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Thu, 29 Aug 2024 13:13:40 -0400 Subject: [PATCH 12/16] removing negative path testing due to CI failures --- .../node/validate/defaults-override.txtar | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 7a0b6de6adb..b49347d25e0 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -1,6 +1,6 @@ # test with default override -mkdir default_overrides -mkdir default_overrides/evm +# mkdir default_overrides +# mkdir default_overrides/evm env CL_CHAIN_DEFAULTS=default_overrides exec chainlink node -c config.toml -s secrets.toml validate cmp stdout out.txt @@ -10,16 +10,18 @@ env CL_CHAIN_DEFAULTS= exec chainlink node -c config.toml -s secrets.toml validate ! cmp stdout out.txt +# negative path testing with failing validate command not working in CI: BCF-3370 + # overrides outside of evm suffix, FinalityTagEnabled should not match as override is not applied -mkdir default_overrides2 -env CL_CHAIN_DEFAULTS=default_overrides2 -! exec chainlink node -c config.toml -s secrets.toml validate +# mkdir default_overrides2 +# env CL_CHAIN_DEFAULTS=default_overrides2 +# ! exec chainlink node -c config.toml -s secrets.toml validate # duplicate chain IDs -mkdir default_overrides3 -env CL_CHAIN_DEFAULTS=default_overrides3 -! exec chainlink node -c config.toml -s secrets.toml validate - +# mkdir default_overrides3 +# env CL_CHAIN_DEFAULTS=default_overrides3 +# ! exec chainlink node -c config.toml -s secrets.toml validate +# stderr 'contains duplicate ChainID' -- default_overrides/evm/Ethereum_Mainnet.toml -- ChainID = '1' @@ -29,10 +31,10 @@ FinalityTagEnabled = true ChainID = '1' FinalityTagEnabled = true --- default_overrides3/Ethereum_Mainnet.toml -- +-- default_overrides3/evm/Ethereum_Mainnet.toml -- ChainID = '1' --- default_overrides3/Ethereum_Testnet.toml -- +-- default_overrides3/evm/Ethereum_Testnet.toml -- ChainID = '1' -- config.toml -- From d045df6faac59ba5a1065c26dd0db349316a0ad2 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Thu, 29 Aug 2024 15:54:42 -0400 Subject: [PATCH 13/16] removing mkdirs --- testdata/scripts/node/validate/defaults-override.txtar | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index b49347d25e0..68dcc8e8d02 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -1,6 +1,4 @@ # test with default override -# mkdir default_overrides -# mkdir default_overrides/evm env CL_CHAIN_DEFAULTS=default_overrides exec chainlink node -c config.toml -s secrets.toml validate cmp stdout out.txt @@ -13,12 +11,10 @@ exec chainlink node -c config.toml -s secrets.toml validate # negative path testing with failing validate command not working in CI: BCF-3370 # overrides outside of evm suffix, FinalityTagEnabled should not match as override is not applied -# mkdir default_overrides2 # env CL_CHAIN_DEFAULTS=default_overrides2 # ! exec chainlink node -c config.toml -s secrets.toml validate # duplicate chain IDs -# mkdir default_overrides3 # env CL_CHAIN_DEFAULTS=default_overrides3 # ! exec chainlink node -c config.toml -s secrets.toml validate # stderr 'contains duplicate ChainID' From e1d55216185f3bf768c7be4be78de274cf2c3e98 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Thu, 29 Aug 2024 16:02:43 -0400 Subject: [PATCH 14/16] updating config after rebase --- .../scripts/node/validate/defaults-override.txtar | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 68dcc8e8d02..9ba0532a643 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -82,6 +82,7 @@ FeedsManager = true LogPoller = false UICSAKeys = false CCIP = true +MultiFeedsManagers = false [Database] DefaultIdleInTxSessionTimeout = '1h0m0s' @@ -327,6 +328,16 @@ DeltaDial = '15s' DeltaReconcile = '1m0s' ListenAddresses = [] +[Capabilities.Dispatcher] +SupportedVersion = 1 +ReceiverBufferSize = 10000 + +[Capabilities.Dispatcher.RateLimit] +GlobalRPS = 800.0 +GlobalBurst = 1000 +PerSenderRPS = 10.0 +PerSenderBurst = 50 + [Capabilities.ExternalRegistry] Address = '' NetworkID = 'evm' @@ -378,6 +389,7 @@ LimitDefault = 500000 LimitMax = 500000 LimitMultiplier = '1' LimitTransfer = 21000 +EstimateGasLimit = false BumpMin = '5 gwei' BumpPercent = 20 BumpThreshold = 3 @@ -423,6 +435,9 @@ ObservationGracePeriod = '1s' [EVM.OCR2.Automation] GasLimit = 10500000 +[EVM.Workflow] +GasLimitDefault = 400000 + [[EVM.Nodes]] Name = 'fake' WSURL = 'wss://foo.bar/ws' From ecdd10124661c035bac913d8cb66f1c5a1dbe2bb Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Thu, 29 Aug 2024 16:05:36 -0400 Subject: [PATCH 15/16] re-enabling negative path --- testdata/scripts/node/validate/defaults-override.txtar | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 9ba0532a643..37a5b67947c 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -11,13 +11,13 @@ exec chainlink node -c config.toml -s secrets.toml validate # negative path testing with failing validate command not working in CI: BCF-3370 # overrides outside of evm suffix, FinalityTagEnabled should not match as override is not applied -# env CL_CHAIN_DEFAULTS=default_overrides2 -# ! exec chainlink node -c config.toml -s secrets.toml validate +env CL_CHAIN_DEFAULTS=default_overrides2 +! exec chainlink node -c config.toml -s secrets.toml validate # duplicate chain IDs -# env CL_CHAIN_DEFAULTS=default_overrides3 -# ! exec chainlink node -c config.toml -s secrets.toml validate -# stderr 'contains duplicate ChainID' +env CL_CHAIN_DEFAULTS=default_overrides3 +! exec chainlink node -c config.toml -s secrets.toml validate +stderr 'contains duplicate ChainID' -- default_overrides/evm/Ethereum_Mainnet.toml -- ChainID = '1' From 07d294e40efebf978f5b180910f1307fd0cb9a67 Mon Sep 17 00:00:00 2001 From: patrickhuie19 Date: Thu, 29 Aug 2024 16:17:32 -0400 Subject: [PATCH 16/16] removing comment --- testdata/scripts/node/validate/defaults-override.txtar | 2 -- 1 file changed, 2 deletions(-) diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 37a5b67947c..3accd1584bb 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -8,8 +8,6 @@ env CL_CHAIN_DEFAULTS= exec chainlink node -c config.toml -s secrets.toml validate ! cmp stdout out.txt -# negative path testing with failing validate command not working in CI: BCF-3370 - # overrides outside of evm suffix, FinalityTagEnabled should not match as override is not applied env CL_CHAIN_DEFAULTS=default_overrides2 ! exec chainlink node -c config.toml -s secrets.toml validate