From 4a8c4a567f658dabb5fb009469a7a2863053d732 Mon Sep 17 00:00:00 2001 From: Urvi Date: Wed, 5 Jun 2024 21:54:10 -0700 Subject: [PATCH 01/11] Update bucketlistdb config parameters --- .../testdata/expected-offline-core.cfg | 4 +- ...offline-enforce-diag-events-and-metav1.cfg | 1 + ...ine-enforce-disabled-diagnostic-events.cfg | 1 + .../expected-offline-with-appendix-core.cfg | 1 + .../expected-offline-with-extra-fields.cfg | 1 + .../expected-offline-with-no-peer-port.cfg | 1 + .../testdata/expected-online-core.cfg | 1 + ...with-appendix-minimum-persistent-entry.cfg | 1 + ...e-with-no-http-port-diag-events-metav1.cfg | 1 + .../expected-online-with-no-http-port.cfg | 1 + .../expected-online-with-no-peer-port.cfg | 1 + ingest/ledgerbackend/toml.go | 112 +++++++++++++++--- ingest/ledgerbackend/toml_test.go | 7 ++ 13 files changed, 117 insertions(+), 16 deletions(-) diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg index 53838f6165..ec37e504fc 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-core.cfg @@ -1,7 +1,7 @@ # Generated file, do not edit +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 DATABASE = "sqlite3://stellar.db" -EXPERIMENTAL_BUCKETLIST_DB = true -EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +DEPRECATED_SQL_LEDGER_STATE = false FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg index 8c66b4a5ad..fa25d69b98 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-enforce-diag-events-and-metav1.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = true diff --git a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg index df307a3a00..4a971a19d2 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-enforce-disabled-diagnostic-events.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg index 124abc435b..30159c150a 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg index d7b41a0b81..eab3b67bf8 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-extra-fields.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg index 9eca1ccad1..9c5fd26769 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-no-peer-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = 0 HTTP_PORT = 0 LOG_FILE_PATH = "/var/stellar-core/test.log" diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg index 57a5e7ff2c..b8aeff746c 100644 --- a/ingest/ledgerbackend/testdata/expected-online-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 6789 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg index 278c681595..8ecc0fe958 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-appendix-minimum-persistent-entry.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 11626 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg index f2227376a9..3960c1b546 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port-diag-events-metav1.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true EMIT_SOROBAN_TRANSACTION_META_EXT_V1 = true ENABLE_DIAGNOSTICS_FOR_TX_SUBMISSION = true ENABLE_SOROBAN_DIAGNOSTIC_EVENTS = true diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg index 89e1762757..6b3830e6ff 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-http-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 11626 LOG_FILE_PATH = "" diff --git a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg index 1b65c5f318..93bb61b8be 100644 --- a/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-with-no-peer-port.cfg @@ -1,4 +1,5 @@ # Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true FAILURE_SAFETY = -1 HTTP_PORT = 6789 LOG_FILE_PATH = "/var/stellar-core/test.log" diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index b8b6634a78..0a40a56f45 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -95,9 +95,9 @@ type captiveCoreTomlValues struct { Validators []Validator `toml:"VALIDATORS,omitempty"` HistoryEntries map[string]History `toml:"-"` QuorumSetEntries map[string]QuorumSet `toml:"-"` - UseBucketListDB bool `toml:"EXPERIMENTAL_BUCKETLIST_DB,omitempty"` - BucketListDBPageSizeExp *uint `toml:"EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT,omitempty"` - BucketListDBCutoff *uint `toml:"EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF,omitempty"` + BucketListDBPageSizeExp *uint `toml:"BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT,omitempty"` + BucketListDBCutoff *uint `toml:"BUCKETLIST_DB_INDEX_CUTOFF,omitempty"` + DeprecatedSqlLedgerState bool `toml:"DEPRECATED_SQL_LEDGER_STATE"` EnableSorobanDiagnosticEvents *bool `toml:"ENABLE_SOROBAN_DIAGNOSTIC_EVENTS,omitempty"` TestingMinimumPersistentEntryLifetime *uint `toml:"TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME,omitempty"` TestingSorobanHighLimitOverride *bool `toml:"TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE,omitempty"` @@ -441,18 +441,108 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) { return offline, nil } +<<<<<<< HEAD +======= +// coreVersion helper struct identify a core version and provides the +// utilities to compare the version ( i.e. minor + major pair ) to a predefined +// version. +type coreVersion struct { + major int + minor int + ledgerProtocolVersion int +} + +// IsEqualOrAbove compares the core version to a version specific. If unable +// to make the decision, the result is always "false", leaning toward the +// common denominator. +func (c *coreVersion) IsEqualOrAbove(major, minor int) bool { + if c.major == 0 && c.minor == 0 { + return false + } + return (c.major == major && c.minor >= minor) || (c.major > major) +} + +// IsEqualOrAbove compares the core version to a version specific. If unable +// to make the decision, the result is always "false", leaning toward the +// common denominator. +func (c *coreVersion) IsProtocolVersionEqualOrAbove(protocolVer int) bool { + if c.ledgerProtocolVersion == 0 { + return false + } + return c.ledgerProtocolVersion >= protocolVer +} + +func checkCoreVersion(coreBinaryPath string) coreVersion { + if coreBinaryPath == "" { + return coreVersion{} + } + + versionBytes, err := exec.Command(coreBinaryPath, "version").Output() + if err != nil { + return coreVersion{} + } + + // starting soroban, we want to use only the first row for the version. + versionRows := strings.Split(string(versionBytes), "\n") + versionRaw := versionRows[0] + + var version [2]int + + re := regexp.MustCompile(`\D*(\d*)\.(\d*).*`) + versionStr := re.FindStringSubmatch(versionRaw) + if len(versionStr) == 3 { + for i := 1; i < len(versionStr); i++ { + val, err := strconv.Atoi((versionStr[i])) + if err != nil { + break + } + version[i-1] = val + } + } + + re = regexp.MustCompile(`^\s*ledger protocol version: (\d*)`) + var ledgerProtocol int + var ledgerProtocolStrings []string + for _, line := range versionRows { + ledgerProtocolStrings = re.FindStringSubmatch(line) + if len(ledgerProtocolStrings) > 0 { + break + } + } + if len(ledgerProtocolStrings) == 2 { + if val, err := strconv.Atoi(ledgerProtocolStrings[1]); err == nil { + ledgerProtocol = val + } + } + + return coreVersion{ + major: version[0], + minor: version[1], + ledgerProtocolVersion: ledgerProtocol, + } +} + +const MinimalSorobanProtocolSupport = 20 + func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { if params.UseDB && !c.tree.Has("DATABASE") { c.Database = "sqlite3://stellar.db" } - if def := c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB"); !def && params.UseDB { - c.UseBucketListDB = true + checkCoreVersionF := params.checkCoreVersion + if checkCoreVersionF == nil { + checkCoreVersionF = checkCoreVersion } + currentCoreVersion := checkCoreVersionF(params.CoreBinaryPath) - if c.UseBucketListDB && !c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { - n := uint(12) - c.BucketListDBPageSizeExp = &n // Set default page size to 4KB + if !params.UseDB { + c.DeprecatedSqlLedgerState = true + } else { + if !c.tree.Has("BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { + n := uint(12) + c.BucketListDBPageSizeExp = &n // Set default page size to 4KB + } + c.DeprecatedSqlLedgerState = false } if !c.tree.Has("NETWORK_PASSPHRASE") { @@ -543,12 +633,6 @@ func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error { ) } - if def := c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB"); def && !params.UseDB { - return fmt.Errorf( - "BucketListDB enabled in captive core config file, requires Horizon flag --captive-core-use-db", - ) - } - homeDomainSet := map[string]HomeDomain{} for _, hd := range c.HomeDomains { if _, ok := homeDomainSet[hd.HomeDomain]; ok { diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index 39b8473d4a..bf3de3229f 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -212,6 +212,13 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { PeerPort: testCase.peerPort, LogPath: testCase.logPath, Strict: true, + checkCoreVersion: func(coreBinaryPath string) coreVersion { + return coreVersion{ + major: 21, + minor: 0, + ledgerProtocolVersion: 21, + } + }, } _, err := NewCaptiveCoreTomlFromFile(testCase.appendPath, params) assert.EqualError(t, err, testCase.expectedError) From 8702bce1fe40f313c7923e019adb50c07066ef8b Mon Sep 17 00:00:00 2001 From: Urvi Date: Wed, 5 Jun 2024 22:35:07 -0700 Subject: [PATCH 02/11] Remove EXPERIMENTAL_BUCKETLIST_DB flag --- services/horizon/docker/captive-core-integration-tests.cfg | 2 -- .../docker/captive-core-integration-tests.soroban-rpc.cfg | 2 -- .../docker/captive-core-reingest-range-integration-tests.cfg | 1 - services/horizon/docker/captive-core-standalone.cfg | 2 -- .../horizon/docker/stellar-core-classic-integration-tests.cfg | 1 - services/horizon/docker/stellar-core-integration-tests.cfg | 1 - services/horizon/docker/stellar-core-pubnet.cfg | 2 -- services/horizon/docker/stellar-core-standalone.cfg | 1 - services/horizon/docker/stellar-core-testnet.cfg | 2 -- services/horizon/docker/verify-range/captive-core-pubnet.cfg | 2 -- services/horizon/docker/verify-range/stellar-core.cfg | 2 -- 11 files changed, 18 deletions(-) diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg index 02c59d7057..275599bacd 100644 --- a/services/horizon/docker/captive-core-integration-tests.cfg +++ b/services/horizon/docker/captive-core-integration-tests.cfg @@ -4,8 +4,6 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 -EXPERIMENTAL_BUCKETLIST_DB=true - ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg index cf4d514975..9a7ad9d769 100644 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg @@ -4,8 +4,6 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 -EXPERIMENTAL_BUCKETLIST_DB=true - ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg index 4744fd390e..44820f5933 100644 --- a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg @@ -2,7 +2,6 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true -EXPERIMENTAL_BUCKETLIST_DB=true [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/captive-core-standalone.cfg b/services/horizon/docker/captive-core-standalone.cfg index f5042a14df..d54b0ecae1 100644 --- a/services/horizon/docker/captive-core-standalone.cfg +++ b/services/horizon/docker/captive-core-standalone.cfg @@ -3,8 +3,6 @@ PEER_PORT=11725 UNSAFE_QUORUM=true FAILURE_SAFETY=0 -EXPERIMENTAL_BUCKETLIST_DB=true - [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg index f2b20b4927..e27cfe14ed 100644 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-classic-integration-tests.cfg @@ -13,7 +13,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" -EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg index 0d5ec5cc43..594a35b244 100644 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-integration-tests.cfg @@ -13,7 +13,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" -EXPERIMENTAL_BUCKETLIST_DB=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/stellar-core-pubnet.cfg b/services/horizon/docker/stellar-core-pubnet.cfg index 7f250b10c7..94b29c4b4e 100644 --- a/services/horizon/docker/stellar-core-pubnet.cfg +++ b/services/horizon/docker/stellar-core-pubnet.cfg @@ -9,8 +9,6 @@ DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker. NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 -EXPERIMENTAL_BUCKETLIST_DB=true - [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/stellar-core-standalone.cfg b/services/horizon/docker/stellar-core-standalone.cfg index 41c8a377aa..a2b7e806c9 100644 --- a/services/horizon/docker/stellar-core-standalone.cfg +++ b/services/horizon/docker/stellar-core-standalone.cfg @@ -14,7 +14,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker.internal port=5641 dbname=stellar" -EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-testnet.cfg b/services/horizon/docker/stellar-core-testnet.cfg index 55c5cea462..cf8546a3e9 100644 --- a/services/horizon/docker/stellar-core-testnet.cfg +++ b/services/horizon/docker/stellar-core-testnet.cfg @@ -12,8 +12,6 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=100 -EXPERIMENTAL_BUCKETLIST_DB=true - [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/verify-range/captive-core-pubnet.cfg b/services/horizon/docker/verify-range/captive-core-pubnet.cfg index cedbbc65ab..5a702711fe 100644 --- a/services/horizon/docker/verify-range/captive-core-pubnet.cfg +++ b/services/horizon/docker/verify-range/captive-core-pubnet.cfg @@ -2,8 +2,6 @@ PEER_PORT=11725 FAILURE_SAFETY=1 -EXPERIMENTAL_BUCKETLIST_DB=true - [[HOME_DOMAINS]] HOME_DOMAIN="stellar.org" QUALITY="HIGH" diff --git a/services/horizon/docker/verify-range/stellar-core.cfg b/services/horizon/docker/verify-range/stellar-core.cfg index 91bb190131..6139f9f528 100644 --- a/services/horizon/docker/verify-range/stellar-core.cfg +++ b/services/horizon/docker/verify-range/stellar-core.cfg @@ -4,8 +4,6 @@ LOG_FILE_PATH="" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 -EXPERIMENTAL_BUCKETLIST_DB=true - AUTOMATIC_MAINTENANCE_COUNT=0 NODE_NAMES=[ From 681f3a38851497cf3b7229e63f22950641d15ba8 Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 6 Jun 2024 09:50:34 -0700 Subject: [PATCH 03/11] Add DEPRECATED_SQL_LEDGER_STATE param. Add unit tests for DEPRECATED_SQL_LEDGER_STATE. --- .../testdata/expected-bucketlistdb-core.cfg | 22 ++++++++ .../expected-default-bucketlistdb-core.cfg | 21 +++++++ .../testdata/expected-in-mem-core.cfg | 19 +++++++ .../testdata/sample-appendix-bucketlistdb.cfg | 12 ++++ ingest/ledgerbackend/toml.go | 5 +- ingest/ledgerbackend/toml_test.go | 56 ++++++++++++++++--- ...ive-core-integration-tests.soroban-rpc.cfg | 1 + ...stellar-core-classic-integration-tests.cfg | 1 + .../docker/stellar-core-integration-tests.cfg | 1 + 9 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg create mode 100644 ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg create mode 100644 ingest/ledgerbackend/testdata/expected-in-mem-core.cfg create mode 100644 ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg diff --git a/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg new file mode 100644 index 0000000000..9040d6dc57 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-bucketlistdb-core.cfg @@ -0,0 +1,22 @@ +# Generated file, do not edit +BUCKETLIST_DB_INDEX_CUTOFF = 20 +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15 +DATABASE = "sqlite3://stellar.db" +DEPRECATED_SQL_LEDGER_STATE = false +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg new file mode 100644 index 0000000000..12fe5e36a5 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-default-bucketlistdb-core.cfg @@ -0,0 +1,21 @@ +# Generated file, do not edit +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +DATABASE = "sqlite3://stellar.db" +DEPRECATED_SQL_LEDGER_STATE = false +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg b/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg new file mode 100644 index 0000000000..965fa5aaa6 --- /dev/null +++ b/ingest/ledgerbackend/testdata/expected-in-mem-core.cfg @@ -0,0 +1,19 @@ +# Generated file, do not edit +DEPRECATED_SQL_LEDGER_STATE = true +FAILURE_SAFETY = -1 +HTTP_PORT = 11626 +LOG_FILE_PATH = "" +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" + +[[HOME_DOMAINS]] + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" + +[[VALIDATORS]] + ADDRESS = "localhost:123" + HOME_DOMAIN = "testnet.stellar.org" + NAME = "sdf_testnet_1" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + +[HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg b/ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg new file mode 100644 index 0000000000..e01a92cc61 --- /dev/null +++ b/ingest/ledgerbackend/testdata/sample-appendix-bucketlistdb.cfg @@ -0,0 +1,12 @@ +BUCKETLIST_DB_INDEX_CUTOFF = 20 +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 15 + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="MEDIUM" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="localhost:123" diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index 0a40a56f45..296fca391d 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -21,6 +21,8 @@ var ( //go:embed configs/captive-core-testnet.cfg TestnetDefaultConfig []byte + + DefaultBucketListDBPageSize uint = 12 ) const ( @@ -539,8 +541,7 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { c.DeprecatedSqlLedgerState = true } else { if !c.tree.Has("BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { - n := uint(12) - c.BucketListDBPageSizeExp = &n // Set default page size to 4KB + c.BucketListDBPageSizeExp = &DefaultBucketListDBPageSize } c.DeprecatedSqlLedgerState = false } diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index bf3de3229f..944ebc8f38 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -212,13 +212,6 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { PeerPort: testCase.peerPort, LogPath: testCase.logPath, Strict: true, - checkCoreVersion: func(coreBinaryPath string) coreVersion { - return coreVersion{ - major: 21, - minor: 0, - ledgerProtocolVersion: 21, - } - }, } _, err := NewCaptiveCoreTomlFromFile(testCase.appendPath, params) assert.EqualError(t, err, testCase.expectedError) @@ -359,6 +352,22 @@ func TestGenerateConfig(t *testing.T) { expectedPath: filepath.Join("testdata", "expected-online-with-appendix-minimum-persistent-entry.cfg"), logPath: nil, }, + { + name: "default BucketlistDB config", + mode: stellarCoreRunnerModeOnline, + appendPath: filepath.Join("testdata", "sample-appendix.cfg"), + expectedPath: filepath.Join("testdata", "expected-default-bucketlistdb-core.cfg"), + useDB: true, + logPath: nil, + }, + { + name: "BucketlistDB config in appendix", + mode: stellarCoreRunnerModeOnline, + appendPath: filepath.Join("testdata", "sample-appendix-bucketlistdb.cfg"), + expectedPath: filepath.Join("testdata", "expected-bucketlistdb-core.cfg"), + useDB: true, + logPath: nil, + }, } { t.Run(testCase.name, func(t *testing.T) { var err error @@ -392,6 +401,36 @@ func TestGenerateConfig(t *testing.T) { } } +func TestGenerateCoreConfigInMemory(t *testing.T) { + appendPath := filepath.Join("testdata", "sample-appendix.cfg") + expectedPath := filepath.Join("testdata", "expected-in-mem-core.cfg") + var err error + var captiveCoreToml *CaptiveCoreToml + params := CaptiveCoreTomlParams{ + NetworkPassphrase: "Public Global Stellar Network ; September 2015", + HistoryArchiveURLs: []string{"http://localhost:1170"}, + Strict: false, + UseDB: false, + checkCoreVersion: func(coreBinaryPath string) coreVersion { + return coreVersion{ + major: 21, + minor: 0, + ledgerProtocolVersion: 21, + } + }, + } + captiveCoreToml, err = NewCaptiveCoreTomlFromFile(appendPath, params) + assert.NoError(t, err) + + configBytes, err := generateConfig(captiveCoreToml, stellarCoreRunnerModeOnline) + assert.NoError(t, err) + + expectedByte, err := ioutil.ReadFile(expectedPath) + assert.NoError(t, err) + + assert.Equal(t, string(expectedByte), string(configBytes)) +} + func TestHistoryArchiveURLTrailingSlash(t *testing.T) { httpPort := uint(8000) peerPort := uint(8000) @@ -468,6 +507,9 @@ func TestDBConfigDefaultsToSqlite(t *testing.T) { toml := CaptiveCoreToml{} require.NoError(t, toml.unmarshal(configBytes, true)) assert.Equal(t, toml.Database, "sqlite3://stellar.db") + assert.Equal(t, toml.DeprecatedSqlLedgerState, false) + assert.Equal(t, *toml.BucketListDBPageSizeExp, DefaultBucketListDBPageSize) + assert.Equal(t, toml.BucketListDBCutoff, (*uint)(nil)) } func TestNonDBConfigDoesNotUpdateDatabase(t *testing.T) { diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg index 9a7ad9d769..8d76504de2 100644 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg @@ -1,3 +1,4 @@ +EXPERIMENTAL_BUCKETLIST_DB = true PEER_PORT=11725 ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg index e27cfe14ed..fe23e94e8d 100644 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-classic-integration-tests.cfg @@ -1,3 +1,4 @@ +DEPRECATED_SQL_LEDGER_STATE=false ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true NETWORK_PASSPHRASE="Standalone Network ; February 2017" diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg index 594a35b244..414ad9c1eb 100644 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-integration-tests.cfg @@ -1,3 +1,4 @@ +DEPRECATED_SQL_LEDGER_STATE=false ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true NETWORK_PASSPHRASE="Standalone Network ; February 2017" From cb5515e9032fd000cd3353ae61169cb835a01f79 Mon Sep 17 00:00:00 2001 From: Urvi Date: Fri, 7 Jun 2024 11:07:04 -0700 Subject: [PATCH 04/11] Update changelog --- ingest/CHANGELOG.md | 7 +++++-- ingest/ledgerbackend/toml.go | 7 ++++--- ingest/ledgerbackend/toml_test.go | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ingest/CHANGELOG.md b/ingest/CHANGELOG.md index 1aab4ae542..ed168de74e 100644 --- a/ingest/CHANGELOG.md +++ b/ingest/CHANGELOG.md @@ -2,8 +2,11 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). - -## Unreleased +### Stellar Core Protocol 21 Configuration Update: +* BucketlistDB is now the default database for stellar-core, replacing the experimental option. As a result, the `EXPERIMENTAL_BUCKETLIST_DB` configuration parameter has been deprecated. +* A new mandatory parameter, `DEPRECATED_SQL_LEDGER_STATE`, has been added with a default value of false which equivalent to `EXPERIMENTAL_BUCKETLIST_DB` being set to true. +* The parameter `EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` has been renamed to `BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT`. +* The parameter `EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF` has been renamed to `BUCKETLIST_DB_INDEX_CUTOFF`. ### New Features * Support for Soroban and Protocol 20! diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index 296fca391d..7f6e242f29 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -99,7 +99,7 @@ type captiveCoreTomlValues struct { QuorumSetEntries map[string]QuorumSet `toml:"-"` BucketListDBPageSizeExp *uint `toml:"BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT,omitempty"` BucketListDBCutoff *uint `toml:"BUCKETLIST_DB_INDEX_CUTOFF,omitempty"` - DeprecatedSqlLedgerState bool `toml:"DEPRECATED_SQL_LEDGER_STATE"` + DeprecatedSqlLedgerState *bool `toml:"DEPRECATED_SQL_LEDGER_STATE,omitempty"` EnableSorobanDiagnosticEvents *bool `toml:"ENABLE_SOROBAN_DIAGNOSTIC_EVENTS,omitempty"` TestingMinimumPersistentEntryLifetime *uint `toml:"TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME,omitempty"` TestingSorobanHighLimitOverride *bool `toml:"TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE,omitempty"` @@ -537,14 +537,15 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { } currentCoreVersion := checkCoreVersionF(params.CoreBinaryPath) + deprecatedSqlLedgerState := false if !params.UseDB { - c.DeprecatedSqlLedgerState = true + deprecatedSqlLedgerState = true } else { if !c.tree.Has("BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { c.BucketListDBPageSizeExp = &DefaultBucketListDBPageSize } - c.DeprecatedSqlLedgerState = false } + c.DeprecatedSqlLedgerState = &deprecatedSqlLedgerState if !c.tree.Has("NETWORK_PASSPHRASE") { c.NetworkPassphrase = params.NetworkPassphrase diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index 944ebc8f38..dd942c8665 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -507,7 +507,7 @@ func TestDBConfigDefaultsToSqlite(t *testing.T) { toml := CaptiveCoreToml{} require.NoError(t, toml.unmarshal(configBytes, true)) assert.Equal(t, toml.Database, "sqlite3://stellar.db") - assert.Equal(t, toml.DeprecatedSqlLedgerState, false) + assert.Equal(t, *toml.DeprecatedSqlLedgerState, false) assert.Equal(t, *toml.BucketListDBPageSizeExp, DefaultBucketListDBPageSize) assert.Equal(t, toml.BucketListDBCutoff, (*uint)(nil)) } From cc3cbba009ab5a11840f20076b933294f38b7696 Mon Sep 17 00:00:00 2001 From: Urvi Date: Tue, 11 Jun 2024 20:42:37 -0700 Subject: [PATCH 05/11] Update Horizon changelog --- services/horizon/CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 7a89844062..8b0edcec78 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -10,8 +10,12 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). - Change ingestion filtering logic to store transactions if any filter matches on it. ([5303](https://github.com/stellar/go/pull/5303)) - The previous behaviour was to store a tx only if both asset and account filters match together. So even if a tx matched an account filter but failed to match an asset filter, it would not be stored by Horizon. -## 2.30.0 +- Captive-core configuration parameters updated to align with [stellar-core v21](https://github.com/stellar/stellar-core/issues/3811) ([5333](https://github.com/stellar/go/pull/5333)) + - BucketlistDB is now the default database for stellar-core, deprecating `EXPERIMENTAL_BUCKETLIST_DB`. + - A new mandatory parameter `DEPRECATED_SQL_LEDGER_STATE` (default: false) will be added automatically, no action required. + - If using `EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` or `EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF`, they have been renamed to `BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` and `BUCKETLIST_DB_INDEX_CUTOFF` respectively. +## 2.30.0 **This release adds support for Protocol 21** ### Added From a10d336a1d5d585aed42ecf30600543cba817ace Mon Sep 17 00:00:00 2001 From: Urvi Date: Wed, 12 Jun 2024 11:03:55 -0700 Subject: [PATCH 06/11] Add validation for DEPRECATED_SQL_LEDGER_STATE parameter --- ingest/ledgerbackend/configs/captive-core-pubnet.cfg | 2 -- ingest/ledgerbackend/configs/captive-core-testnet.cfg | 2 -- .../sample-appendix-deprecated_sql_ledger_state.cfg | 11 +++++++++++ ingest/ledgerbackend/toml.go | 4 ++++ ingest/ledgerbackend/toml_test.go | 7 +++++++ 5 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 ingest/ledgerbackend/testdata/sample-appendix-deprecated_sql_ledger_state.cfg diff --git a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg index 6d23046f56..5af59efaf9 100644 --- a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg @@ -5,8 +5,6 @@ FAILURE_SAFETY=1 HTTP_PORT=11626 PEER_PORT=11725 -EXPERIMENTAL_BUCKETLIST_DB=true - [[HOME_DOMAINS]] HOME_DOMAIN="publicnode.org" QUALITY="HIGH" diff --git a/ingest/ledgerbackend/configs/captive-core-testnet.cfg b/ingest/ledgerbackend/configs/captive-core-testnet.cfg index 3f0dec5095..9abeecc8f5 100644 --- a/ingest/ledgerbackend/configs/captive-core-testnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-testnet.cfg @@ -2,8 +2,6 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015" UNSAFE_QUORUM=true FAILURE_SAFETY=1 -EXPERIMENTAL_BUCKETLIST_DB=true - [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" QUALITY="HIGH" diff --git a/ingest/ledgerbackend/testdata/sample-appendix-deprecated_sql_ledger_state.cfg b/ingest/ledgerbackend/testdata/sample-appendix-deprecated_sql_ledger_state.cfg new file mode 100644 index 0000000000..3ca36b223b --- /dev/null +++ b/ingest/ledgerbackend/testdata/sample-appendix-deprecated_sql_ledger_state.cfg @@ -0,0 +1,11 @@ +DEPRECATED_SQL_LEDGER_STATE = true + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="MEDIUM" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="localhost:123" diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index 7f6e242f29..3d2bdd2f4c 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -635,6 +635,10 @@ func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error { ) } + if def := c.tree.Has("DEPRECATED_SQL_LEDGER_STATE"); def && params.UseDB && *c.DeprecatedSqlLedgerState == true { + return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to true, indicating stellar-core on-disk mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to false") + } + homeDomainSet := map[string]HomeDomain{} for _, hd := range c.HomeDomains { if _, ok := homeDomainSet[hd.HomeDomain]; ok { diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index dd942c8665..a5e432f162 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -203,6 +203,12 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { appendPath: filepath.Join("testdata", "appendix-with-bucket-dir-path.cfg"), expectedError: "could not unmarshal captive core toml: setting BUCKET_DIR_PATH is disallowed for Captive Core, use CAPTIVE_CORE_STORAGE_PATH instead", }, + { + name: "invalid DEPRECATED_SQL_LEDGER_STATE", + appendPath: filepath.Join("testdata", "sample-appendix-deprecated_sql_ledger_state.cfg"), + expectedError: "invalid captive core toml: CAPTIVE_CORE_USE_DB parameter is set to true, indicating " + + "stellar-core on-disk mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to false", + }, } { t.Run(testCase.name, func(t *testing.T) { params := CaptiveCoreTomlParams{ @@ -212,6 +218,7 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { PeerPort: testCase.peerPort, LogPath: testCase.logPath, Strict: true, + UseDB: true, } _, err := NewCaptiveCoreTomlFromFile(testCase.appendPath, params) assert.EqualError(t, err, testCase.expectedError) From b1f3b587b1bd9b1f556d5e064b08d18b7e22616b Mon Sep 17 00:00:00 2001 From: Urvi Date: Wed, 12 Jun 2024 11:17:54 -0700 Subject: [PATCH 07/11] Fix linter error --- ingest/ledgerbackend/toml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index 3d2bdd2f4c..6e754e903a 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -635,7 +635,7 @@ func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error { ) } - if def := c.tree.Has("DEPRECATED_SQL_LEDGER_STATE"); def && params.UseDB && *c.DeprecatedSqlLedgerState == true { + if def := c.tree.Has("DEPRECATED_SQL_LEDGER_STATE"); def && params.UseDB && *c.DeprecatedSqlLedgerState { return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to true, indicating stellar-core on-disk mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to false") } From ce7c6ce7cac55be20f9f36e26221acd8822b589b Mon Sep 17 00:00:00 2001 From: urvisavla Date: Thu, 13 Jun 2024 12:53:24 -0700 Subject: [PATCH 08/11] Update services/horizon/CHANGELOG.md Co-authored-by: shawn --- services/horizon/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 8b0edcec78..6f127b9968 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -12,7 +12,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). - Captive-core configuration parameters updated to align with [stellar-core v21](https://github.com/stellar/stellar-core/issues/3811) ([5333](https://github.com/stellar/go/pull/5333)) - BucketlistDB is now the default database for stellar-core, deprecating `EXPERIMENTAL_BUCKETLIST_DB`. - - A new mandatory parameter `DEPRECATED_SQL_LEDGER_STATE` (default: false) will be added automatically, no action required. + - A new mandatory parameter `DEPRECATED_SQL_LEDGER_STATE` (default: false) is required by stellar-core on its configuration toml file. if the toml provided by `CAPTIVE_CORE_CONFIG_PATH` does not have this new setting, Horizon will add it automatically, therefore, no action required. - If using `EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` or `EXPERIMENTAL_BUCKETLIST_DB_INDEX_CUTOFF`, they have been renamed to `BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT` and `BUCKETLIST_DB_INDEX_CUTOFF` respectively. ## 2.30.0 From 98f8b978c9ab1f456cb91ad3a84b25103c1ec103 Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 13 Jun 2024 14:04:36 -0700 Subject: [PATCH 09/11] Addressing review comments --- .../testdata/sample-appendix-in-memory.cfg | 11 +++++++++++ ...ger_state.cfg => sample-appendix-on-disk.cfg} | 0 ingest/ledgerbackend/toml.go | 14 ++++++++++---- ingest/ledgerbackend/toml_test.go | 16 ++++++++++++---- 4 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg rename ingest/ledgerbackend/testdata/{sample-appendix-deprecated_sql_ledger_state.cfg => sample-appendix-on-disk.cfg} (100%) diff --git a/ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg b/ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg new file mode 100644 index 0000000000..958019998c --- /dev/null +++ b/ingest/ledgerbackend/testdata/sample-appendix-in-memory.cfg @@ -0,0 +1,11 @@ +DEPRECATED_SQL_LEDGER_STATE = false + +[[HOME_DOMAINS]] +HOME_DOMAIN="testnet.stellar.org" +QUALITY="MEDIUM" + +[[VALIDATORS]] +NAME="sdf_testnet_1" +HOME_DOMAIN="testnet.stellar.org" +PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" +ADDRESS="localhost:123" diff --git a/ingest/ledgerbackend/testdata/sample-appendix-deprecated_sql_ledger_state.cfg b/ingest/ledgerbackend/testdata/sample-appendix-on-disk.cfg similarity index 100% rename from ingest/ledgerbackend/testdata/sample-appendix-deprecated_sql_ledger_state.cfg rename to ingest/ledgerbackend/testdata/sample-appendix-on-disk.cfg diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index 6e754e903a..31c5a4553e 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -22,7 +22,7 @@ var ( //go:embed configs/captive-core-testnet.cfg TestnetDefaultConfig []byte - DefaultBucketListDBPageSize uint = 12 + defaultBucketListDBPageSize uint = 12 ) const ( @@ -542,7 +542,7 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { deprecatedSqlLedgerState = true } else { if !c.tree.Has("BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { - c.BucketListDBPageSizeExp = &DefaultBucketListDBPageSize + c.BucketListDBPageSizeExp = &defaultBucketListDBPageSize } } c.DeprecatedSqlLedgerState = &deprecatedSqlLedgerState @@ -635,8 +635,14 @@ func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error { ) } - if def := c.tree.Has("DEPRECATED_SQL_LEDGER_STATE"); def && params.UseDB && *c.DeprecatedSqlLedgerState { - return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to true, indicating stellar-core on-disk mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to false") + if c.tree.Has("DEPRECATED_SQL_LEDGER_STATE") { + if params.UseDB && *c.DeprecatedSqlLedgerState { + return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to true, indicating stellar-core on-disk mode," + + " in which DEPRECATED_SQL_LEDGER_STATE must be set to false") + } else if !params.UseDB && !*c.DeprecatedSqlLedgerState { + return fmt.Errorf("CAPTIVE_CORE_USE_DB parameter is set to false, indicating stellar-core in-memory mode," + + " in which DEPRECATED_SQL_LEDGER_STATE must be set to true") + } } homeDomainSet := map[string]HomeDomain{} diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index a5e432f162..05cbece3b4 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -28,6 +28,7 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { peerPort *uint logPath *string expectedError string + inMemory bool }{ { name: "mismatching NETWORK_PASSPHRASE", @@ -204,11 +205,18 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { expectedError: "could not unmarshal captive core toml: setting BUCKET_DIR_PATH is disallowed for Captive Core, use CAPTIVE_CORE_STORAGE_PATH instead", }, { - name: "invalid DEPRECATED_SQL_LEDGER_STATE", - appendPath: filepath.Join("testdata", "sample-appendix-deprecated_sql_ledger_state.cfg"), + name: "invalid DEPRECATED_SQL_LEDGER_STATE on-disk", + appendPath: filepath.Join("testdata", "sample-appendix-on-disk.cfg"), expectedError: "invalid captive core toml: CAPTIVE_CORE_USE_DB parameter is set to true, indicating " + "stellar-core on-disk mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to false", }, + { + name: "invalid DEPRECATED_SQL_LEDGER_STATE in-memory", + appendPath: filepath.Join("testdata", "sample-appendix-in-memory.cfg"), + expectedError: "invalid captive core toml: CAPTIVE_CORE_USE_DB parameter is set to false, indicating " + + "stellar-core in-memory mode, in which DEPRECATED_SQL_LEDGER_STATE must be set to true", + inMemory: true, + }, } { t.Run(testCase.name, func(t *testing.T) { params := CaptiveCoreTomlParams{ @@ -218,7 +226,7 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { PeerPort: testCase.peerPort, LogPath: testCase.logPath, Strict: true, - UseDB: true, + UseDB: !testCase.inMemory, } _, err := NewCaptiveCoreTomlFromFile(testCase.appendPath, params) assert.EqualError(t, err, testCase.expectedError) @@ -515,7 +523,7 @@ func TestDBConfigDefaultsToSqlite(t *testing.T) { require.NoError(t, toml.unmarshal(configBytes, true)) assert.Equal(t, toml.Database, "sqlite3://stellar.db") assert.Equal(t, *toml.DeprecatedSqlLedgerState, false) - assert.Equal(t, *toml.BucketListDBPageSizeExp, DefaultBucketListDBPageSize) + assert.Equal(t, *toml.BucketListDBPageSizeExp, defaultBucketListDBPageSize) assert.Equal(t, toml.BucketListDBCutoff, (*uint)(nil)) } From 0f7a987df38d0f85df41100f0b300eef83e64711 Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 20 Jun 2024 16:05:17 -0700 Subject: [PATCH 10/11] resolving merge conflict --- ingest/ledgerbackend/toml.go | 89 ------------------------------------ 1 file changed, 89 deletions(-) diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index 31c5a4553e..b7f41b03a7 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -443,100 +443,11 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) { return offline, nil } -<<<<<<< HEAD -======= -// coreVersion helper struct identify a core version and provides the -// utilities to compare the version ( i.e. minor + major pair ) to a predefined -// version. -type coreVersion struct { - major int - minor int - ledgerProtocolVersion int -} - -// IsEqualOrAbove compares the core version to a version specific. If unable -// to make the decision, the result is always "false", leaning toward the -// common denominator. -func (c *coreVersion) IsEqualOrAbove(major, minor int) bool { - if c.major == 0 && c.minor == 0 { - return false - } - return (c.major == major && c.minor >= minor) || (c.major > major) -} - -// IsEqualOrAbove compares the core version to a version specific. If unable -// to make the decision, the result is always "false", leaning toward the -// common denominator. -func (c *coreVersion) IsProtocolVersionEqualOrAbove(protocolVer int) bool { - if c.ledgerProtocolVersion == 0 { - return false - } - return c.ledgerProtocolVersion >= protocolVer -} - -func checkCoreVersion(coreBinaryPath string) coreVersion { - if coreBinaryPath == "" { - return coreVersion{} - } - - versionBytes, err := exec.Command(coreBinaryPath, "version").Output() - if err != nil { - return coreVersion{} - } - - // starting soroban, we want to use only the first row for the version. - versionRows := strings.Split(string(versionBytes), "\n") - versionRaw := versionRows[0] - - var version [2]int - - re := regexp.MustCompile(`\D*(\d*)\.(\d*).*`) - versionStr := re.FindStringSubmatch(versionRaw) - if len(versionStr) == 3 { - for i := 1; i < len(versionStr); i++ { - val, err := strconv.Atoi((versionStr[i])) - if err != nil { - break - } - version[i-1] = val - } - } - - re = regexp.MustCompile(`^\s*ledger protocol version: (\d*)`) - var ledgerProtocol int - var ledgerProtocolStrings []string - for _, line := range versionRows { - ledgerProtocolStrings = re.FindStringSubmatch(line) - if len(ledgerProtocolStrings) > 0 { - break - } - } - if len(ledgerProtocolStrings) == 2 { - if val, err := strconv.Atoi(ledgerProtocolStrings[1]); err == nil { - ledgerProtocol = val - } - } - - return coreVersion{ - major: version[0], - minor: version[1], - ledgerProtocolVersion: ledgerProtocol, - } -} - -const MinimalSorobanProtocolSupport = 20 - func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { if params.UseDB && !c.tree.Has("DATABASE") { c.Database = "sqlite3://stellar.db" } - checkCoreVersionF := params.checkCoreVersion - if checkCoreVersionF == nil { - checkCoreVersionF = checkCoreVersion - } - currentCoreVersion := checkCoreVersionF(params.CoreBinaryPath) - deprecatedSqlLedgerState := false if !params.UseDB { deprecatedSqlLedgerState = true From b15ed0a65eb8bdc7dcf5f2de5c061e932f53e228 Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 20 Jun 2024 16:12:48 -0700 Subject: [PATCH 11/11] Resolving merge conflicts --- ingest/ledgerbackend/toml_test.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index 05cbece3b4..42412b85f1 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -2,9 +2,7 @@ package ledgerbackend import ( "io/ioutil" - "os" "path/filepath" - "strconv" "testing" "github.com/stretchr/testify/assert" @@ -234,18 +232,6 @@ func TestCaptiveCoreTomlValidation(t *testing.T) { } } -func checkTestingAboveProtocol19() bool { - str := os.Getenv("HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL") - if str == "" { - return false - } - version, err := strconv.ParseUint(str, 10, 32) - if err != nil { - return false - } - return uint32(version) > 19 -} - func TestGenerateConfig(t *testing.T) { for _, testCase := range []struct { name string @@ -426,13 +412,6 @@ func TestGenerateCoreConfigInMemory(t *testing.T) { HistoryArchiveURLs: []string{"http://localhost:1170"}, Strict: false, UseDB: false, - checkCoreVersion: func(coreBinaryPath string) coreVersion { - return coreVersion{ - major: 21, - minor: 0, - ledgerProtocolVersion: 21, - } - }, } captiveCoreToml, err = NewCaptiveCoreTomlFromFile(appendPath, params) assert.NoError(t, err)