ingest/ledgerbackend: Update captive-core config with new BucketlistD…

…B parameters (#5333)

* Update bucketlistdb config parameters
* Remove EXPERIMENTAL_BUCKETLIST_DB flag
* Add DEPRECATED_SQL_LEDGER_STATE param. Add unit tests for DEPRECATED_SQL_LEDGER_STATE.
* Update changelog
* Update Horizon changelog

ingest/CHANGELOG.md

@@ -2,8 +2,11 @@
 
 All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/).
 
-
-## Unreleased
+### Stellar Core Protocol 21 Configuration Update:
+* BucketlistDB is now the default database for stellar-core, replacing the experimental option. As a result, the `EXPERIMENTAL_BUCKETLIST_DB` configuration parameter has been deprecated.
+* A new mandatory parameter, `DEPRECATED_SQL_LEDGER_STATE`, has been added with a default value of false which equivalent to `EXPERIMENTAL_BUCKETLIST_DB` being set to true.
+* The parameter `EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` has been renamed to `BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT`.
+* The parameter `EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF` has been renamed to `BUCKETLIST_DB_INDEX_CUTOFF`.
 
 ### New Features
 * Support for Soroban and Protocol 20!

ingest/ledgerbackend/configs/captive-core-pubnet.cfg

@@ -5,8 +5,6 @@ FAILURE_SAFETY=1
 HTTP_PORT=11626
 PEER_PORT=11725
 
-EXPERIMENTAL_BUCKETLIST_DB=true
-
 [[HOME_DOMAINS]]
 HOME_DOMAIN="publicnode.org"
 QUALITY="HIGH"

ingest/ledgerbackend/configs/captive-core-testnet.cfg

@@ -2,8 +2,6 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015"
 UNSAFE_QUORUM=true
 FAILURE_SAFETY=1
 
-EXPERIMENTAL_BUCKETLIST_DB=true
-
 [[HOME_DOMAINS]]
 HOME_DOMAIN="testnet.stellar.org"
 QUALITY="HIGH"

ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg

@@ -0,0 +1,22 @@
+# Generated file, do not edit
+BUCKETLIST_DB_INDEX_CUTOFF = 20
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15
+DATABASE = "sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
+FAILURE_SAFETY = -1
+HTTP_PORT = 11626
+LOG_FILE_PATH = ""
+NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015"
+
+[[HOME_DOMAINS]]
+  HOME_DOMAIN = "testnet.stellar.org"
+  QUALITY = "MEDIUM"
+
+[[VALIDATORS]]
+  ADDRESS = "localhost:123"
+  HOME_DOMAIN = "testnet.stellar.org"
+  NAME = "sdf_testnet_1"
+  PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+
+[HISTORY.h0]
+  get = "curl -sf http://localhost:1170/{0} -o {1}"

ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg

@@ -0,0 +1,21 @@
+# Generated file, do not edit
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+DATABASE = "sqlite3://stellar.db"
+DEPRECATED_SQL_LEDGER_STATE = false
+FAILURE_SAFETY = -1
+HTTP_PORT = 11626
+LOG_FILE_PATH = ""
+NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015"
+
+[[HOME_DOMAINS]]
+  HOME_DOMAIN = "testnet.stellar.org"
+  QUALITY = "MEDIUM"
+
+[[VALIDATORS]]
+  ADDRESS = "localhost:123"
+  HOME_DOMAIN = "testnet.stellar.org"
+  NAME = "sdf_testnet_1"
+  PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+
+[HISTORY.h0]
+  get = "curl -sf http://localhost:1170/{0} -o {1}"

ingest/ledgerbackend/testdata/expected-in-mem-core.cfg

@@ -0,0 +1,19 @@
+# Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
+FAILURE_SAFETY = -1
+HTTP_PORT = 11626
+LOG_FILE_PATH = ""
+NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015"
+
+[[HOME_DOMAINS]]
+  HOME_DOMAIN = "testnet.stellar.org"
+  QUALITY = "MEDIUM"
+
+[[VALIDATORS]]
+  ADDRESS = "localhost:123"
+  HOME_DOMAIN = "testnet.stellar.org"
+  NAME = "sdf_testnet_1"
+  PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+
+[HISTORY.h0]
+  get = "curl -sf http://localhost:1170/{0} -o {1}"

ingest/ledgerbackend/testdata/expected-offline-core.cfg

@@ -1,7 +1,7 @@
 # Generated file, do not edit
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
 DATABASE = "sqlite3://stellar.db"
-EXPERIMENTAL_BUCKETLIST_DB = true
-EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12
+DEPRECATED_SQL_LEDGER_STATE = false
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""

ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true
 ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = true

ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""

ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""

ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = ""

ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = 0
 HTTP_PORT = 0
 LOG_FILE_PATH = "/var/stellar-core/test.log"

ingest/ledgerbackend/testdata/expected-online-core.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = ""

ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
 LOG_FILE_PATH = ""

ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true
 ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true
 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = true

ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 11626
 LOG_FILE_PATH = ""

ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg

@@ -1,4 +1,5 @@
 # Generated file, do not edit
+DEPRECATED_SQL_LEDGER_STATE = true
 FAILURE_SAFETY = -1
 HTTP_PORT = 6789
 LOG_FILE_PATH = "/var/stellar-core/test.log"

ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg

@@ -0,0 +1,12 @@
+BUCKETLIST_DB_INDEX_CUTOFF = 20
+BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="MEDIUM"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="localhost:123"

ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg

@@ -0,0 +1,11 @@
+DEPRECATED_SQL_LEDGER_STATE = false
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="MEDIUM"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="localhost:123"

ingest/ledgerbackend/testdata/sample-appendix-on-disk.cfg

@@ -0,0 +1,11 @@
+DEPRECATED_SQL_LEDGER_STATE = true
+
+[[HOME_DOMAINS]]
+HOME_DOMAIN="testnet.stellar.org"
+QUALITY="MEDIUM"
+
+[[VALIDATORS]]
+NAME="sdf_testnet_1"
+HOME_DOMAIN="testnet.stellar.org"
+PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y"
+ADDRESS="localhost:123"

ingest/ledgerbackend/toml.go

@@ -21,6 +21,8 @@ var (
 
 	//go:embed configs/captive-core-testnet.cfg
 	TestnetDefaultConfig []byte
+
+	defaultBucketListDBPageSize uint = 12
 )
 
 const (
@@ -95,9 +97,9 @@ type captiveCoreTomlValues struct {
 	Validators                            []Validator          `toml:"VALIDATORS,omitempty"`
 	HistoryEntries                        map[string]History   `toml:"-"`
 	QuorumSetEntries                      map[string]QuorumSet `toml:"-"`
-	UseBucketListDB                       bool                 `toml:"EXPERIMENTAL_BUCKETLIST_DB,omitempty"`
-	BucketListDBPageSizeExp               *uint                `toml:"EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT,omitempty"`
-	BucketListDBCutoff                    *uint                `toml:"EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF,omitempty"`
+	BucketListDBPageSizeExp               *uint                `toml:"BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT,omitempty"`
+	BucketListDBCutoff                    *uint                `toml:"BUCKETLIST_DB_INDEX_CUTOFF,omitempty"`
+	DeprecatedSqlLedgerState              *bool                `toml:"DEPRECATED_SQL_LEDGER_STATE,omitempty"`
 	EnableSorobanDiagnosticEvents         *bool                `toml:"ENABLE_SOROBAN_DIAGNOSTIC_EVENTS,omitempty"`
 	TestingMinimumPersistentEntryLifetime *uint                `toml:"TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME,omitempty"`
 	TestingSorobanHighLimitOverride       *bool                `toml:"TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE,omitempty"`
@@ -446,14 +448,15 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) {
 		c.Database = "sqlite3://stellar.db"
 	}
 
-	if def := c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB"); !def && params.UseDB {
-		c.UseBucketListDB = true
-	}
-
-	if c.UseBucketListDB && !c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") {
-		n := uint(12)
-		c.BucketListDBPageSizeExp = &n // Set default page size to 4KB
+	deprecatedSqlLedgerState := false
+	if !params.UseDB {
+		deprecatedSqlLedgerState = true
+	} else {
+		if !c.tree.Has("BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") {
+			c.BucketListDBPageSizeExp = &defaultBucketListDBPageSize
+		}
 	}
+	c.DeprecatedSqlLedgerState = &deprecatedSqlLedgerState
 
 	if !c.tree.Has("NETWORK_PASSPHRASE") {
 		c.NetworkPassphrase = params.NetworkPassphrase
@@ -543,10 +546,14 @@ func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error {
 		)
 	}
 
-	if def := c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB"); def && !params.UseDB {
-		return fmt.Errorf(
-			"BucketListDB enabled in captive core config file, requires Horizon flag --captive-core-use-db",
-		)
+	if c.tree.Has("DEPRECATED_SQL_LEDGER_STATE") {
+		if params.UseDB && *c.DeprecatedSqlLedgerState {
+			return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to true, indicating stellar-core on-disk mode," +
+				" in which DEPRECATED_SQL_LEDGER_STATE must be set to false")
+		} else if !params.UseDB && !*c.DeprecatedSqlLedgerState {
+			return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to false, indicating stellar-core in-memory mode," +
+				" in which DEPRECATED_SQL_LEDGER_STATE must be set to true")
+		}
 	}
 
 	homeDomainSet := map[string]HomeDomain{}

ingest/ledgerbackend/toml_test.go

@@ -2,9 +2,7 @@ package ledgerbackend
 
 import (
 	"io/ioutil"
-	"os"
 	"path/filepath"
-	"strconv"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -28,6 +26,7 @@ func TestCaptiveCoreTomlValidation(t *testing.T) {
 		peerPort          *uint
 		logPath           *string
 		expectedError     string
+		inMemory          bool
 	}{
 		{
 			name:              "mismatching NETWORK_PASSPHRASE",
@@ -203,6 +202,19 @@ func TestCaptiveCoreTomlValidation(t *testing.T) {
 			appendPath:    filepath.Join("testdata", "appendix-with-bucket-dir-path.cfg"),
 			expectedError: "could not unmarshal captive core toml: setting BUCKET_DIR_PATH is disallowed for Captive Core, use CAPTIVE_CORE_STORAGE_PATH instead",
 		},
+		{
+			name:       "invalid DEPRECATED_SQL_LEDGER_STATE on-disk",
+			appendPath: filepath.Join("testdata", "sample-appendix-on-disk.cfg"),
+			expectedError: "invalid captive core toml: CAPTIVE_CORE_USE_DB parameter is set to true, indicating " +
+				"stellar-core on-disk mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to false",
+		},
+		{
+			name:       "invalid DEPRECATED_SQL_LEDGER_STATE in-memory",
+			appendPath: filepath.Join("testdata", "sample-appendix-in-memory.cfg"),
+			expectedError: "invalid captive core toml: CAPTIVE_CORE_USE_DB parameter is set to false, indicating " +
+				"stellar-core in-memory mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to true",
+			inMemory: true,
+		},
 	} {
 		t.Run(testCase.name, func(t *testing.T) {
 			params := CaptiveCoreTomlParams{
@@ -212,25 +224,14 @@ func TestCaptiveCoreTomlValidation(t *testing.T) {
 				PeerPort:           testCase.peerPort,
 				LogPath:            testCase.logPath,
 				Strict:             true,
+				UseDB:              !testCase.inMemory,
 			}
 			_, err := NewCaptiveCoreTomlFromFile(testCase.appendPath, params)
 			assert.EqualError(t, err, testCase.expectedError)
 		})
 	}
 }
 
-func checkTestingAboveProtocol19() bool {
-	str := os.Getenv("HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL")
-	if str == "" {
-		return false
-	}
-	version, err := strconv.ParseUint(str, 10, 32)
-	if err != nil {
-		return false
-	}
-	return uint32(version) > 19
-}
-
 func TestGenerateConfig(t *testing.T) {
 	for _, testCase := range []struct {
 		name                           string
@@ -352,6 +353,22 @@ func TestGenerateConfig(t *testing.T) {
 			expectedPath: filepath.Join("testdata", "expected-online-with-appendix-minimum-persistent-entry.cfg"),
 			logPath:      nil,
 		},
+		{
+			name:         "default BucketlistDB config",
+			mode:         stellarCoreRunnerModeOnline,
+			appendPath:   filepath.Join("testdata", "sample-appendix.cfg"),
+			expectedPath: filepath.Join("testdata", "expected-default-bucketlistdb-core.cfg"),
+			useDB:        true,
+			logPath:      nil,
+		},
+		{
+			name:         "BucketlistDB config in appendix",
+			mode:         stellarCoreRunnerModeOnline,
+			appendPath:   filepath.Join("testdata", "sample-appendix-bucketlistdb.cfg"),
+			expectedPath: filepath.Join("testdata", "expected-bucketlistdb-core.cfg"),
+			useDB:        true,
+			logPath:      nil,
+		},
 	} {
 		t.Run(testCase.name, func(t *testing.T) {
 			var err error
@@ -385,6 +402,29 @@ func TestGenerateConfig(t *testing.T) {
 	}
 }
 
+func TestGenerateCoreConfigInMemory(t *testing.T) {
+	appendPath := filepath.Join("testdata", "sample-appendix.cfg")
+	expectedPath := filepath.Join("testdata", "expected-in-mem-core.cfg")
+	var err error
+	var captiveCoreToml *CaptiveCoreToml
+	params := CaptiveCoreTomlParams{
+		NetworkPassphrase:  "Public Global Stellar Network ; September 2015",
+		HistoryArchiveURLs: []string{"http://localhost:1170"},
+		Strict:             false,
+		UseDB:              false,
+	}
+	captiveCoreToml, err = NewCaptiveCoreTomlFromFile(appendPath, params)
+	assert.NoError(t, err)
+
+	configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOnline)
+	assert.NoError(t, err)
+
+	expectedByte, err := ioutil.ReadFile(expectedPath)
+	assert.NoError(t, err)
+
+	assert.Equal(t, string(expectedByte), string(configBytes))
+}
+
 func TestHistoryArchiveURLTrailingSlash(t *testing.T) {
 	httpPort := uint(8000)
 	peerPort := uint(8000)
@@ -461,6 +501,9 @@ func TestDBConfigDefaultsToSqlite(t *testing.T) {
 	toml := CaptiveCoreToml{}
 	require.NoError(t, toml.unmarshal(configBytes, true))
 	assert.Equal(t, toml.Database, "sqlite3://stellar.db")
+	assert.Equal(t, *toml.DeprecatedSqlLedgerState, false)
+	assert.Equal(t, *toml.BucketListDBPageSizeExp, defaultBucketListDBPageSize)
+	assert.Equal(t, toml.BucketListDBCutoff, (*uint)(nil))
 }
 
 func TestNonDBConfigDoesNotUpdateDatabase(t *testing.T) {