Bump default PG conns from 20->100; enable auto-scaling open conns fo…

…r mercury (#12697)

* Bump default PG conns from 20->100; enable auto-scaling open conns for mercury

* Try git submodule update --recursive --remote

* Reset foundry

.changeset/cool-apricots-compare.md

@@ -0,0 +1,9 @@
+---
+"chainlink": patch
+---
+
+Increase default config for postgres max open conns from 20 to 100.
+
+Also, add autoscaling for mercury jobs. The max open conns limit will be
+automatically increased to the number of mercury jobs if this exceeds the
+configured value.

core/config/docs/core.toml

@@ -30,7 +30,7 @@ MaxIdleConns = 10 # Default
 # MaxOpenConns configures the maximum number of database connections that a Chainlink node will have open at any one time. Think of this as the maximum burst upper bound limit of database connections per Chainlink node instance. Increasing this number can help to improve performance under database-heavy workloads.
 #
 # Postgres has connection limits, so you must use caution when increasing this value. If you are running several instances of a Chainlink node or another application on a single database server, you might run out of Postgres connection slots if you raise this value too high.
-MaxOpenConns = 20 # Default
+MaxOpenConns = 100 # Default
 # MigrateOnStartup controls whether a Chainlink node will attempt to automatically migrate the database on boot. If you want more control over your database migration process, set this variable to `false` and manually migrate the database using the CLI `migrate` command instead.
 MigrateOnStartup = true # Default
 

core/services/chainlink/testdata/config-empty-effective.toml

@@ -13,7 +13,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

core/services/chainlink/testdata/config-multi-chain-effective.toml

@@ -13,7 +13,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

core/services/pg/connection.go

@@ -82,8 +82,7 @@ func NewConnection(uri string, dialect dialects.DialectName, config ConnectionCo
 	if _, err = db.Exec(stmt); err != nil {
 		return nil, err
 	}
-	db.SetMaxOpenConns(config.MaxOpenConns())
-	db.SetMaxIdleConns(config.MaxIdleConns())
+	setMaxConns(db, config)
 
 	if os.Getenv("SKIP_PG_VERSION_CHECK") != "true" {
 		if err := checkVersion(db, MinRequiredPGVersion); err != nil {
@@ -94,6 +93,33 @@ func NewConnection(uri string, dialect dialects.DialectName, config ConnectionCo
 	return db, disallowReplica(db)
 }
 
+func setMaxConns(db *sqlx.DB, config ConnectionConfig) {
+	db.SetMaxOpenConns(config.MaxOpenConns())
+	db.SetMaxIdleConns(config.MaxIdleConns())
+
+	// HACK: In the case of mercury jobs, one conn is needed per job for good
+	// performance. Most nops will forget to increase the defaults to account
+	// for this so we detect it here instead.
+	//
+	// This problem will be solved by replacing mercury with parallel
+	// compositions (llo plugin).
+	//
+	// See: https://smartcontract-it.atlassian.net/browse/MERC-3654
+	var cnt int
+	if err := db.Get(&cnt, `SELECT COUNT(*) FROM ocr2_oracle_specs WHERE plugin_type = 'mercury'`); err != nil {
+		log.Printf("Error checking mercury jobs: %s", err.Error())
+		return
+	}
+	if cnt > config.MaxOpenConns() {
+		log.Printf("Detected %d mercury jobs, increasing max open connections from %d to %d", cnt, config.MaxOpenConns(), cnt)
+		db.SetMaxOpenConns(cnt)
+	}
+	if cnt > config.MaxIdleConns() {
+		log.Printf("Detected %d mercury jobs, increasing max idle connections from %d to %d", cnt, config.MaxIdleConns(), cnt)
+		db.SetMaxIdleConns(cnt)
+	}
+}
+
 type Getter interface {
 	Get(dest interface{}, query string, args ...interface{}) error
 }

core/web/resolver/testdata/config-empty-effective.toml

@@ -13,7 +13,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

core/web/resolver/testdata/config-multi-chain-effective.toml

@@ -13,7 +13,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

docs/CONFIG.md

@@ -80,7 +80,7 @@ DefaultLockTimeout = '15s' # Default
 DefaultQueryTimeout = '10s' # Default
 LogQueries = false # Default
 MaxIdleConns = 10 # Default
-MaxOpenConns = 20 # Default
+MaxOpenConns = 100 # Default
 MigrateOnStartup = true # Default
 ```
 
@@ -119,7 +119,7 @@ Postgres has connection limits, so you must use caution when increasing this val
 
 ### MaxOpenConns
 ```toml
-MaxOpenConns = 20 # Default
+MaxOpenConns = 100 # Default
 ```
 MaxOpenConns configures the maximum number of database connections that a Chainlink node will have open at any one time. Think of this as the maximum burst upper bound limit of database connections per Chainlink node instance. Increasing this number can help to improve performance under database-heavy workloads.
 

testdata/scripts/node/validate/default.txtar

@@ -25,7 +25,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

testdata/scripts/node/validate/disk-based-logging-disabled.txtar

@@ -69,7 +69,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

testdata/scripts/node/validate/disk-based-logging-no-dir.txtar

@@ -69,7 +69,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

testdata/scripts/node/validate/disk-based-logging.txtar

@@ -69,7 +69,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

testdata/scripts/node/validate/invalid-ocr-p2p.txtar

@@ -54,7 +54,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

testdata/scripts/node/validate/invalid.txtar

@@ -59,7 +59,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

testdata/scripts/node/validate/valid.txtar

@@ -66,7 +66,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]

testdata/scripts/node/validate/warnings.txtar

@@ -48,7 +48,7 @@ DefaultLockTimeout = '15s'
 DefaultQueryTimeout = '10s'
 LogQueries = false
 MaxIdleConns = 10
-MaxOpenConns = 20
+MaxOpenConns = 100
 MigrateOnStartup = true
 
 [Database.Backup]