From 2b52b4d90f483bd888c4b1282aa705d76adb9059 Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 13 Jun 2024 15:42:36 -0700 Subject: [PATCH 1/4] Remove check for stellar classic --- ingest/ledgerbackend/toml.go | 107 +----------------- ingest/ledgerbackend/toml_test.go | 17 --- ...captive-core-classic-integration-tests.cfg | 15 --- ...ingest-range-classic-integration-tests.cfg | 11 -- services/horizon/internal/flags_test.go | 4 +- .../horizon/internal/integration/db_test.go | 13 +-- .../internal/test/integration/integration.go | 14 +-- 7 files changed, 9 insertions(+), 172 deletions(-) delete mode 100644 services/horizon/docker/captive-core-classic-integration-tests.cfg delete mode 100644 services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go index a4f61a454d..b8b6634a78 100644 --- a/ingest/ledgerbackend/toml.go +++ b/ingest/ledgerbackend/toml.go @@ -5,9 +5,7 @@ import ( _ "embed" "fmt" "os" - "os/exec" "regexp" - "strconv" "strings" "github.com/stellar/go/support/errors" @@ -346,8 +344,6 @@ type CaptiveCoreTomlParams struct { EnforceSorobanDiagnosticEvents bool // Enfore EnableSorobanTransactionMetaExtV1 when not disabled explicitly EnforceSorobanTransactionMetaExtV1 bool - // used for testing - checkCoreVersion func(coreBinaryPath string) coreVersion } // NewCaptiveCoreTomlFromFile constructs a new CaptiveCoreToml instance by merging configuration @@ -445,104 +441,13 @@ func (c *CaptiveCoreToml) CatchupToml() (*CaptiveCoreToml, error) { return offline, nil } -// coreVersion helper struct identify a core version and provides the -// utilities to compare the version ( i.e. minor + major pair ) to a predefined -// version. -type coreVersion struct { - major int - minor int - ledgerProtocolVersion int -} - -// IsEqualOrAbove compares the core version to a version specific. If unable -// to make the decision, the result is always "false", leaning toward the -// common denominator. -func (c *coreVersion) IsEqualOrAbove(major, minor int) bool { - if c.major == 0 && c.minor == 0 { - return false - } - return (c.major == major && c.minor >= minor) || (c.major > major) -} - -// IsEqualOrAbove compares the core version to a version specific. If unable -// to make the decision, the result is always "false", leaning toward the -// common denominator. -func (c *coreVersion) IsProtocolVersionEqualOrAbove(protocolVer int) bool { - if c.ledgerProtocolVersion == 0 { - return false - } - return c.ledgerProtocolVersion >= protocolVer -} - -func checkCoreVersion(coreBinaryPath string) coreVersion { - if coreBinaryPath == "" { - return coreVersion{} - } - - versionBytes, err := exec.Command(coreBinaryPath, "version").Output() - if err != nil { - return coreVersion{} - } - - // starting soroban, we want to use only the first row for the version. - versionRows := strings.Split(string(versionBytes), "\n") - versionRaw := versionRows[0] - - var version [2]int - - re := regexp.MustCompile(`\D*(\d*)\.(\d*).*`) - versionStr := re.FindStringSubmatch(versionRaw) - if len(versionStr) == 3 { - for i := 1; i < len(versionStr); i++ { - val, err := strconv.Atoi((versionStr[i])) - if err != nil { - break - } - version[i-1] = val - } - } - - re = regexp.MustCompile(`^\s*ledger protocol version: (\d*)`) - var ledgerProtocol int - var ledgerProtocolStrings []string - for _, line := range versionRows { - ledgerProtocolStrings = re.FindStringSubmatch(line) - if len(ledgerProtocolStrings) > 0 { - break - } - } - if len(ledgerProtocolStrings) == 2 { - if val, err := strconv.Atoi(ledgerProtocolStrings[1]); err == nil { - ledgerProtocol = val - } - } - - return coreVersion{ - major: version[0], - minor: version[1], - ledgerProtocolVersion: ledgerProtocol, - } -} - -const MinimalBucketListDBCoreSupportVersionMajor = 19 -const MinimalBucketListDBCoreSupportVersionMinor = 6 -const MinimalSorobanProtocolSupport = 20 - func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { if params.UseDB && !c.tree.Has("DATABASE") { c.Database = "sqlite3://stellar.db" } - checkCoreVersionF := params.checkCoreVersion - if checkCoreVersionF == nil { - checkCoreVersionF = checkCoreVersion - } - currentCoreVersion := checkCoreVersionF(params.CoreBinaryPath) if def := c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB"); !def && params.UseDB { - // Supports version 19.6 and above - if currentCoreVersion.IsEqualOrAbove(MinimalBucketListDBCoreSupportVersionMajor, MinimalBucketListDBCoreSupportVersionMinor) { - c.UseBucketListDB = true - } + c.UseBucketListDB = true } if c.UseBucketListDB && !c.tree.Has("EXPERIMENTAL_BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT") { @@ -584,14 +489,10 @@ func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { } if params.EnforceSorobanDiagnosticEvents { - if currentCoreVersion.IsEqualOrAbove(20, 0) { - enforceOption(&c.EnableSorobanDiagnosticEvents) - } - if currentCoreVersion.IsEqualOrAbove(20, 1) { - enforceOption(&c.EnableDiagnosticsForTxSubmission) - } + enforceOption(&c.EnableSorobanDiagnosticEvents) + enforceOption(&c.EnableDiagnosticsForTxSubmission) } - if params.EnforceSorobanTransactionMetaExtV1 && currentCoreVersion.IsEqualOrAbove(20, 4) { + if params.EnforceSorobanTransactionMetaExtV1 { enforceOption(&c.EnableEmitSorobanTransactionMetaExtV1) } } diff --git a/ingest/ledgerbackend/toml_test.go b/ingest/ledgerbackend/toml_test.go index fecf50c8d9..39b8473d4a 100644 --- a/ingest/ledgerbackend/toml_test.go +++ b/ingest/ledgerbackend/toml_test.go @@ -366,13 +366,6 @@ func TestGenerateConfig(t *testing.T) { UseDB: testCase.useDB, EnforceSorobanDiagnosticEvents: testCase.enforceSorobanDiagnosticEvents, EnforceSorobanTransactionMetaExtV1: testCase.enforceEmitMetaV1, - checkCoreVersion: func(coreBinaryPath string) coreVersion { - return coreVersion{ - major: 21, - minor: 0, - ledgerProtocolVersion: 21, - } - }, } if testCase.appendPath != "" { captiveCoreToml, err = NewCaptiveCoreTomlFromFile(testCase.appendPath, params) @@ -497,13 +490,3 @@ func TestNonDBConfigDoesNotUpdateDatabase(t *testing.T) { require.NoError(t, toml.unmarshal(configBytes, true)) assert.Equal(t, toml.Database, "") } - -func TestCheckCoreVersion(t *testing.T) { - coreBin := os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN") - if coreBin == "" { - t.SkipNow() - return - } - version := checkCoreVersion(coreBin) - require.True(t, version.IsEqualOrAbove(20, 0)) -} diff --git a/services/horizon/docker/captive-core-classic-integration-tests.cfg b/services/horizon/docker/captive-core-classic-integration-tests.cfg deleted file mode 100644 index 2f95a0ee54..0000000000 --- a/services/horizon/docker/captive-core-classic-integration-tests.cfg +++ /dev/null @@ -1,15 +0,0 @@ -PEER_PORT=11725 -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -EXPERIMENTAL_BUCKETLIST_DB=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="localhost" -QUALITY="MEDIUM" diff --git a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg deleted file mode 100644 index 735f58b739..0000000000 --- a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg +++ /dev/null @@ -1,11 +0,0 @@ -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -EXPERIMENTAL_BUCKETLIST_DB=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="localhost" -QUALITY="MEDIUM" diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index 65d1da524c..76ec1ffd8d 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -307,7 +307,7 @@ func TestEnvironmentVariables(t *testing.T) { assert.Equal(t, config.AdminPort, uint(6060)) assert.Equal(t, config.Port, uint(8001)) assert.Equal(t, config.CaptiveCoreBinaryPath, os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN")) - assert.Equal(t, config.CaptiveCoreConfigPath, "../docker/captive-core-classic-integration-tests.cfg") + assert.Equal(t, config.CaptiveCoreConfigPath, "../docker/captive-core-integration-tests.cfg") assert.Equal(t, config.CaptiveCoreConfigUseDB, true) } @@ -324,7 +324,7 @@ func horizonEnvVars() map[string]string { "ADMIN_PORT": "6060", "PORT": "8001", "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), - "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-classic-integration-tests.cfg", + "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-integration-tests.cfg", "CAPTIVE_CORE_USE_DB": "true", } } diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 98d584c8e6..331f99ee27 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -12,7 +12,6 @@ import ( "github.com/stellar/go/clients/horizonclient" "github.com/stellar/go/historyarchive" - "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/keypair" hProtocol "github.com/stellar/go/protocols/horizon" horizoncmd "github.com/stellar/go/services/horizon/cmd" @@ -535,7 +534,7 @@ func TestReingestDB(t *testing.T) { horizonConfig.CaptiveCoreConfigPath = filepath.Join( filepath.Dir(horizonConfig.CaptiveCoreConfigPath), - getCoreConfigFile(itest), + "captive-core-reingest-range-integration-tests.cfg", ) horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", @@ -703,14 +702,6 @@ func TestReingestDBWithFilterRules(t *testing.T) { }, 30*time.Second, time.Second) } -func getCoreConfigFile(itest *integration.Test) string { - coreConfigFile := "captive-core-reingest-range-classic-integration-tests.cfg" - if itest.Config().ProtocolVersion >= ledgerbackend.MinimalSorobanProtocolSupport { - coreConfigFile = "captive-core-reingest-range-integration-tests.cfg" - } - return coreConfigFile -} - func command(t *testing.T, horizonConfig horizon.Config, args ...string) []string { return append([]string{ "--stellar-core-url", @@ -848,7 +839,7 @@ func TestFillGaps(t *testing.T) { horizonConfig.CaptiveCoreConfigPath = filepath.Join( filepath.Dir(horizonConfig.CaptiveCoreConfigPath), - getCoreConfigFile(itest), + "captive-core-reingest-range-integration-tests.cfg", ) horizoncmd.RootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "--parallel-workers=1")) tt.NoError(horizoncmd.RootCmd.Execute()) diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index e12f77f693..0402301a44 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -26,7 +26,6 @@ import ( sdk "github.com/stellar/go/clients/horizonclient" "github.com/stellar/go/clients/stellarcore" - "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/keypair" proto "github.com/stellar/go/protocols/horizon" horizon "github.com/stellar/go/services/horizon/internal" @@ -184,11 +183,7 @@ func NewTest(t *testing.T, config Config) *Test { func (i *Test) configureCaptiveCore() { composePath := findDockerComposePath() i.coreConfig.binaryPath = os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN") - coreConfigFile := "captive-core-classic-integration-tests.cfg" - if i.config.ProtocolVersion >= ledgerbackend.MinimalSorobanProtocolSupport { - coreConfigFile = "captive-core-integration-tests.cfg" - } - i.coreConfig.configPath = filepath.Join(composePath, coreConfigFile) + i.coreConfig.configPath = filepath.Join(composePath, "captive-core-integration-tests.cfg") i.coreConfig.storagePath = i.CurrentTest().TempDir() i.coreConfig.useDB = true @@ -254,13 +249,6 @@ func (i *Test) runComposeCommand(args ...string) { ) } - if i.config.ProtocolVersion < ledgerbackend.MinimalSorobanProtocolSupport { - cmd.Env = append( - cmd.Environ(), - "CORE_CONFIG_FILE=stellar-core-classic-integration-tests.cfg", - ) - } - i.t.Log("Running", cmd.Args) out, innerErr := cmd.Output() if len(out) > 0 { From 5ff51c7afb22f135b382eff1699afa0b6d309ddd Mon Sep 17 00:00:00 2001 From: Urvi Date: Fri, 14 Jun 2024 11:52:21 -0700 Subject: [PATCH 2/4] Remove protocol 20 from integration tests --- .github/workflows/horizon.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index fe7b2df7a8..3ea92c8b17 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -13,7 +13,7 @@ jobs: os: [ubuntu-20.04, ubuntu-22.04] go: ["1.21", "1.22"] pg: [12, 16] - protocol-version: [20, 21] + protocol-version: [21] runs-on: ${{ matrix.os }} services: postgres: @@ -36,9 +36,6 @@ jobs: PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 - PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal - PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21 - PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 PGHOST: localhost PGPORT: 5432 PGUSER: postgres @@ -101,7 +98,7 @@ jobs: - name: Calculate the source hash id: calculate_source_hash run: | - combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_20_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_19_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1) + combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1) echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV" - name: Restore Horizon binary and integration tests source hash to cache From ffc70d0e88579143e4b8e28d495439f8940fc899 Mon Sep 17 00:00:00 2001 From: Urvi Date: Mon, 17 Jun 2024 16:34:54 -0700 Subject: [PATCH 3/4] Add shared functions to get the stellar-core build version and protocol version and update ledgerexporter and captivecorebackend to use these shared functions. Implement a version check for protocol 21 when creating a new captive-core instance. --- .../ledgerexporter/internal/config.go | 21 +-- .../ledgerexporter/internal/config_test.go | 83 +----------- ingest/ledgerbackend/captive_core_backend.go | 37 +++--- .../captive_core_backend_test.go | 33 +++++ ingest/ledgerbackend/stellar_core_version.go | 63 +++++++++ .../stellar_core_version_test.go | 123 ++++++++++++++++++ .../internal/ingest/db_integration_test.go | 2 + services/horizon/internal/ingest/main_test.go | 2 + 8 files changed, 247 insertions(+), 117 deletions(-) create mode 100644 ingest/ledgerbackend/stellar_core_version.go create mode 100644 ingest/ledgerbackend/stellar_core_version_test.go diff --git a/exp/services/ledgerexporter/internal/config.go b/exp/services/ledgerexporter/internal/config.go index b040e2dcc9..22920ed483 100644 --- a/exp/services/ledgerexporter/internal/config.go +++ b/exp/services/ledgerexporter/internal/config.go @@ -5,8 +5,6 @@ import ( _ "embed" "fmt" "os" - "os/exec" - "strings" "github.com/stellar/go/historyarchive" "github.com/stellar/go/ingest/ledgerbackend" @@ -194,26 +192,11 @@ func (config *Config) GenerateCaptiveCoreConfig(coreBinFromPath string) (ledgerb }, nil } -// By default, it points to exec.Command, overridden for testing purpose -var execCommand = exec.Command - -// Executes the "stellar-core version" command and parses its output to extract -// the core version -// The output of the "version" command is expected to be a multi-line string where the -// first line is the core version in format "vX.Y.Z-*". func (c *Config) setCoreVersionInfo() (err error) { - versionCmd := execCommand(c.StellarCoreConfig.StellarCoreBinaryPath, "version") - versionOutput, err := versionCmd.Output() + c.CoreVersion, err = ledgerbackend.GetCoreBuildVersionFunc(c.StellarCoreConfig.StellarCoreBinaryPath) if err != nil { - return fmt.Errorf("failed to execute stellar-core version command: %w", err) - } - - // Split the output into lines - rows := strings.Split(string(versionOutput), "\n") - if len(rows) == 0 || len(rows[0]) == 0 { - return fmt.Errorf("stellar-core version not found") + return fmt.Errorf("failed to set stellar-core version: %w", err) } - c.CoreVersion = rows[0] logger.Infof("stellar-core version: %s", c.CoreVersion) return nil } diff --git a/exp/services/ledgerexporter/internal/config_test.go b/exp/services/ledgerexporter/internal/config_test.go index 6523df7605..5cddd71a86 100644 --- a/exp/services/ledgerexporter/internal/config_test.go +++ b/exp/services/ledgerexporter/internal/config_test.go @@ -3,17 +3,15 @@ package ledgerexporter import ( "context" "fmt" - "os" - "os/exec" "testing" + "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/support/datastore" "github.com/stretchr/testify/require" "github.com/stellar/go/historyarchive" - "github.com/stellar/go/support/errors" ) func TestNewConfig(t *testing.T) { @@ -95,7 +93,7 @@ func TestDefaultCaptiveCoreBin(t *testing.T) { RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}) require.NoError(t, err) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("/test/default/stellar-core") require.NoError(t, err) require.Equal(t, ccConfig.BinaryPath, "/test/default/stellar-core") @@ -116,7 +114,7 @@ func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -137,7 +135,7 @@ func TestValidCaptiveCoreManualNetwork(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, "test") require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"}) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -157,7 +155,7 @@ func TestValidCaptiveCoreOverridenToml(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -179,7 +177,7 @@ func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) { require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"}) - cmdOut = "v20.2.0-2-g6e73c0a88\n" + ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88\n", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -438,72 +436,3 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) { } mockArchive.AssertExpectations(t) } - -var cmdOut = "" - -func fakeExecCommand(command string, args ...string) *exec.Cmd { - cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...) - cmd := exec.Command(os.Args[0], cs...) - cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+cmdOut) - return cmd -} - -func init() { - execCommand = fakeExecCommand -} - -func TestExecCmdHelperProcess(t *testing.T) { - if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" { - return - } - fmt.Fprint(os.Stdout, os.Getenv("CMD_OUT")) - os.Exit(0) -} - -func TestSetCoreVersionInfo(t *testing.T) { - execCommand = fakeExecCommand - tests := []struct { - name string - commandOutput string - expectedError error - expectedCoreVer string - }{ - { - name: "version found", - commandOutput: "v20.2.0-2-g6e73c0a88\n" + - "rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" + - "soroban-env-host: \n" + - " curr:\n" + - " package version: 20.2.0\n" + - " git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" + - " ledger protocol version: 20\n" + - " pre-release version: 0\n" + - " rs-stellar-xdr:\n" + - " package version: 20.1.0\n" + - " git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" + - " base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n", - expectedError: nil, - expectedCoreVer: "v20.2.0-2-g6e73c0a88", - }, - { - name: "core version not found", - commandOutput: "", - expectedError: errors.New("stellar-core version not found"), - expectedCoreVer: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - config := Config{} - cmdOut = tt.commandOutput - err := config.setCoreVersionInfo() - - if tt.expectedError != nil { - require.EqualError(t, err, tt.expectedError.Error()) - } else { - require.NoError(t, err) - require.Equal(t, tt.expectedCoreVer, config.CoreVersion) - } - }) - } -} diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go index 879049f873..75dd8cd06c 100644 --- a/ingest/ledgerbackend/captive_core_backend.go +++ b/ingest/ledgerbackend/captive_core_backend.go @@ -6,8 +6,6 @@ import ( "fmt" "net/http" "os" - "os/exec" - "strings" "sync" "time" @@ -22,6 +20,8 @@ import ( "github.com/stellar/go/xdr" ) +var minProtocolVersionSupported uint = 21 + // Ensure CaptiveStellarCore implements LedgerBackend var _ LedgerBackend = (*CaptiveStellarCore)(nil) @@ -168,6 +168,17 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) { config.Log.SetLevel(logrus.InfoLevel) } + protocolVersion, err := GetCoreProtocolVersionFunc(config.BinaryPath) + if err != nil { + return nil, fmt.Errorf("error determining stellar-core protocol version: %w", err) + } + + if protocolVersion < minProtocolVersionSupported { + return nil, fmt.Errorf("stellar-core version not supported. Installed stellar-core version is at protocol %d, but minimum "+ + "required version is %d. Please upgrade stellar-core to a version that supports protocol version %d or higher", + protocolVersion, minProtocolVersionSupported, minProtocolVersionSupported) + } + parentCtx := config.Context if parentCtx == nil { parentCtx = context.Background() @@ -250,28 +261,12 @@ func (c *CaptiveStellarCore) coreVersionMetric() float64 { return float64(info.Info.ProtocolVersion) } -// By default, it points to exec.Command, overridden for testing purpose -var execCommand = exec.Command - -// Executes the "stellar-core version" command and parses its output to extract -// the core version -// The output of the "version" command is expected to be a multi-line string where the -// first line is the core version in format "vX.Y.Z-*". func (c *CaptiveStellarCore) setCoreVersion() { - versionCmd := execCommand(c.config.BinaryPath, "version") - versionOutput, err := versionCmd.Output() + var err error + c.captiveCoreVersion, err = GetCoreBuildVersionFunc(c.config.BinaryPath) if err != nil { - c.config.Log.Errorf("failed to execute stellar-core version command: %s", err) + c.config.Log.Errorf("Failed to set stellar-core version: %s", err) } - - // Split the output into lines - rows := strings.Split(string(versionOutput), "\n") - if len(rows) == 0 || len(rows[0]) == 0 { - c.config.Log.Error("stellar-core version not found") - return - } - - c.captiveCoreVersion = rows[0] c.config.Log.Infof("stellar-core version: %s", c.captiveCoreVersion) } diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go index a367f560f1..01a58af212 100644 --- a/ingest/ledgerbackend/captive_core_backend_test.go +++ b/ingest/ledgerbackend/captive_core_backend_test.go @@ -151,6 +151,8 @@ func TestCaptiveNew(t *testing.T) { networkPassphrase := network.PublicNetworkPassphrase historyURLs := []string{server.URL} + GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ BinaryPath: executablePath, @@ -169,6 +171,35 @@ func TestCaptiveNew(t *testing.T) { assert.Equal(t, "uatest", userAgent) } +func TestCaptiveNewUnsupportedProtocolVersion(t *testing.T) { + storagePath, err := os.MkdirTemp("", "captive-core-*") + require.NoError(t, err) + defer os.RemoveAll(storagePath) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + executablePath := "/etc/stellar-core" + networkPassphrase := network.PublicNetworkPassphrase + historyURLs := []string{server.URL} + + GetCoreProtocolVersionFunc = func(string) (uint, error) { return 20, nil } + + _, err = NewCaptive( + CaptiveCoreConfig{ + BinaryPath: executablePath, + NetworkPassphrase: networkPassphrase, + HistoryArchiveURLs: historyURLs, + StoragePath: storagePath, + UserAgent: "uatest", + }, + ) + + assert.EqualError(t, err, "stellar-core version not supported. Installed stellar-core version is at protocol 20, but minimum required version is 21. Please upgrade stellar-core to a version that supports protocol version 21 or higher") +} + func TestCaptivePrepareRange(t *testing.T) { metaChan := make(chan metaResult, 100) @@ -984,6 +1015,8 @@ func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) { captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{}) assert.NoError(t, err) + GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ BinaryPath: executablePath, diff --git a/ingest/ledgerbackend/stellar_core_version.go b/ingest/ledgerbackend/stellar_core_version.go new file mode 100644 index 0000000000..dac53f3892 --- /dev/null +++ b/ingest/ledgerbackend/stellar_core_version.go @@ -0,0 +1,63 @@ +package ledgerbackend + +import ( + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" +) + +// By default, it points to exec.Command, overridden for testing purpose +var execCommand = exec.Command +var GetCoreProtocolVersionFunc = GetCoreProtocolVersion +var GetCoreBuildVersionFunc = GetCoreBuildVersion + +// GetCoreBuildVersion executes the "stellar-core version" command and parses its output to extract +// the core version +// The output of the "version" command is expected to be a multi-line string where the +// first line is the core version in format "vX.Y.Z-*". +func GetCoreBuildVersion(coreBinaryPath string) (string, error) { + versionCmd := execCommand(coreBinaryPath, "version") + versionOutput, err := versionCmd.Output() + if err != nil { + return "", fmt.Errorf("failed to execute stellar-core version command: %w", err) + } + + // Split the output into lines + rows := strings.Split(string(versionOutput), "\n") + if len(rows) == 0 || len(rows[0]) == 0 { + return "", fmt.Errorf("stellar-core version not found") + } + + return rows[0], nil +} + +// GetCoreProtocolVersion retrieves the ledger protocol version from the specified stellar-core binary. +// It executes the "stellar-core version" command and parses the output to extract the protocol version. +func GetCoreProtocolVersion(coreBinaryPath string) (uint, error) { + if coreBinaryPath == "" { + return 0, fmt.Errorf("stellar-core binary path is empty") + } + + versionBytes, err := execCommand(coreBinaryPath, "version").Output() + if err != nil { + return 0, fmt.Errorf("error executing stellar-core version command (%s): %w", coreBinaryPath, err) + } + + versionRows := strings.Split(string(versionBytes), "\n") + re := regexp.MustCompile(`^\s*ledger protocol version: (\d*)`) + var ledgerProtocolStrings []string + for _, line := range versionRows { + ledgerProtocolStrings = re.FindStringSubmatch(line) + if len(ledgerProtocolStrings) == 2 { + val, err := strconv.Atoi(ledgerProtocolStrings[1]) + if err != nil { + return 0, fmt.Errorf("error parsing protocol version from stellar-core output: %w", err) + } + return uint(val), nil + } + } + + return 0, fmt.Errorf("error parsing protocol version from stellar-core output") +} diff --git a/ingest/ledgerbackend/stellar_core_version_test.go b/ingest/ledgerbackend/stellar_core_version_test.go new file mode 100644 index 0000000000..51bf2947c1 --- /dev/null +++ b/ingest/ledgerbackend/stellar_core_version_test.go @@ -0,0 +1,123 @@ +package ledgerbackend + +import ( + "fmt" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +var fakeExecCmdOut = "" + +func fakeExecCommand(command string, args ...string) *exec.Cmd { + cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...) + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+fakeExecCmdOut) + return cmd +} + +func init() { + execCommand = fakeExecCommand +} + +func TestExecCmdHelperProcess(t *testing.T) { + if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" { + return + } + fmt.Fprint(os.Stdout, os.Getenv("CMD_OUT")) + os.Exit(0) +} + +func TestGetCoreBuildVersion(t *testing.T) { + tests := []struct { + name string + commandOutput string + expectedError error + expectedCoreVer string + }{ + { + name: "core build version found", + commandOutput: "v20.2.0-2-g6e73c0a88\n" + + "rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" + + "soroban-env-host: \n" + + " curr:\n" + + " package version: 20.2.0\n" + + " git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" + + " ledger protocol version: 20\n" + + " pre-release version: 0\n" + + " rs-stellar-xdr:\n" + + " package version: 20.1.0\n" + + " git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" + + " base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n", + expectedError: nil, + expectedCoreVer: "v20.2.0-2-g6e73c0a88", + }, + { + name: "core build version not found", + commandOutput: "", + expectedError: fmt.Errorf("stellar-core version not found"), + expectedCoreVer: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeExecCmdOut = tt.commandOutput + coreVersion, err := GetCoreBuildVersion("") + + if tt.expectedError != nil { + require.EqualError(t, err, tt.expectedError.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedCoreVer, coreVersion) + } + }) + } +} + +func TestGetCoreProtcolVersion(t *testing.T) { + tests := []struct { + name string + commandOutput string + expectedError error + expectedProtocolVersion uint + }{ + { + name: "core protocol version found", + commandOutput: "v20.2.0-2-g6e73c0a88\n" + + "rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" + + "soroban-env-host: \n" + + " curr:\n" + + " package version: 20.2.0\n" + + " git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" + + " ledger protocol version: 21\n" + + " pre-release version: 0\n" + + " rs-stellar-xdr:\n" + + " package version: 20.1.0\n" + + " git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" + + " base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n", + expectedError: nil, + expectedProtocolVersion: 21, + }, + { + name: "core protocol version not found", + commandOutput: "", + expectedError: fmt.Errorf("error parsing protocol version from stellar-core output"), + expectedProtocolVersion: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeExecCmdOut = tt.commandOutput + coreVersion, err := GetCoreProtocolVersion("/usr/bin/stellar-core") + + if tt.expectedError != nil { + require.EqualError(t, err, tt.expectedError.Error()) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedProtocolVersion, coreVersion) + } + }) + } +} diff --git a/services/horizon/internal/ingest/db_integration_test.go b/services/horizon/internal/ingest/db_integration_test.go index 606cd9fb2b..61998649f0 100644 --- a/services/horizon/internal/ingest/db_integration_test.go +++ b/services/horizon/internal/ingest/db_integration_test.go @@ -100,6 +100,8 @@ func (s *DBTestSuite) SetupTest() { s.checkpointHash = xdr.Hash{1, 2, 3} s.ledgerBackend = &ledgerbackend.MockDatabaseBackend{} s.historyAdapter = &mockHistoryArchiveAdapter{} + ledgerbackend.GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + var err error sIface, err := NewSystem(Config{ HistorySession: s.tt.HorizonSession(), diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go index 3c7c587aa2..3efcd73e3b 100644 --- a/services/horizon/internal/ingest/main_test.go +++ b/services/horizon/internal/ingest/main_test.go @@ -96,6 +96,8 @@ func TestNewSystem(t *testing.T) { CheckpointFrequency: 64, } + ledgerbackend.GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } + sIface, err := NewSystem(config) assert.NoError(t, err) system := sIface.(*system) From c43e98114da8a0a727b9a4362b31d96cadb4b88f Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 20 Jun 2024 00:49:17 -0700 Subject: [PATCH 4/4] Addressing review comments --- exp/services/ledgerexporter/internal/app.go | 2 +- .../ledgerexporter/internal/config.go | 9 +++- .../ledgerexporter/internal/config_test.go | 46 +++++++++---------- ingest/ledgerbackend/captive_core_backend.go | 20 ++++++-- .../captive_core_backend_test.go | 39 ++++++++-------- ingest/ledgerbackend/stellar_core_version.go | 13 +++--- .../stellar_core_version_test.go | 4 +- .../internal/ingest/db_integration_test.go | 2 +- services/horizon/internal/ingest/main.go | 27 ++++++----- services/horizon/internal/ingest/main_test.go | 26 +++++++++-- 10 files changed, 114 insertions(+), 74 deletions(-) diff --git a/exp/services/ledgerexporter/internal/app.go b/exp/services/ledgerexporter/internal/app.go index 54c7a72096..40cdf90f0e 100644 --- a/exp/services/ledgerexporter/internal/app.go +++ b/exp/services/ledgerexporter/internal/app.go @@ -104,7 +104,7 @@ func (a *App) init(ctx context.Context, runtimeSettings RuntimeSettings) error { collectors.NewGoCollector(), ) - if a.config, err = NewConfig(runtimeSettings); err != nil { + if a.config, err = NewConfig(runtimeSettings, nil); err != nil { return errors.Wrap(err, "Could not load configuration") } if archive, err = a.config.GenerateHistoryArchive(ctx, logger); err != nil { diff --git a/exp/services/ledgerexporter/internal/config.go b/exp/services/ledgerexporter/internal/config.go index 22920ed483..d5aad53256 100644 --- a/exp/services/ledgerexporter/internal/config.go +++ b/exp/services/ledgerexporter/internal/config.go @@ -71,6 +71,7 @@ type Config struct { CoreVersion string SerializedCaptiveCoreToml []byte + CoreBuildVersionFn ledgerbackend.CoreBuildVersionFunc } // This will generate the config based on settings @@ -78,12 +79,16 @@ type Config struct { // settings - requested settings // // return - *Config or an error if any range validation failed. -func NewConfig(settings RuntimeSettings) (*Config, error) { +func NewConfig(settings RuntimeSettings, getCoreVersionFn ledgerbackend.CoreBuildVersionFunc) (*Config, error) { config := &Config{} config.StartLedger = uint32(settings.StartLedger) config.EndLedger = uint32(settings.EndLedger) config.Mode = settings.Mode + config.CoreBuildVersionFn = ledgerbackend.CoreBuildVersion + if getCoreVersionFn != nil { + config.CoreBuildVersionFn = getCoreVersionFn + } logger.Infof("Requested export mode of %v with start=%d, end=%d", settings.Mode.Name(), config.StartLedger, config.EndLedger) @@ -193,7 +198,7 @@ func (config *Config) GenerateCaptiveCoreConfig(coreBinFromPath string) (ledgerb } func (c *Config) setCoreVersionInfo() (err error) { - c.CoreVersion, err = ledgerbackend.GetCoreBuildVersionFunc(c.StellarCoreConfig.StellarCoreBinaryPath) + c.CoreVersion, err = c.CoreBuildVersionFn(c.StellarCoreConfig.StellarCoreBinaryPath) if err != nil { return fmt.Errorf("failed to set stellar-core version: %w", err) } diff --git a/exp/services/ledgerexporter/internal/config_test.go b/exp/services/ledgerexporter/internal/config_test.go index 5cddd71a86..f782de5ea4 100644 --- a/exp/services/ledgerexporter/internal/config_test.go +++ b/exp/services/ledgerexporter/internal/config_test.go @@ -5,7 +5,6 @@ import ( "fmt" "testing" - "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/support/datastore" @@ -21,7 +20,7 @@ func TestNewConfig(t *testing.T) { mockArchive.On("GetRootHAS").Return(historyarchive.HistoryArchiveState{CurrentLedger: 5}, nil).Once() config, err := NewConfig( - RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: Append}) + RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: Append}, nil) require.NoError(t, err) err = config.ValidateAndSetLedgerRange(ctx, mockArchive) @@ -41,7 +40,7 @@ func TestNewConfig(t *testing.T) { func TestGenerateHistoryArchiveFromPreconfiguredNetwork(t *testing.T) { ctx := context.Background() config, err := NewConfig( - RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_preconfigured.toml", Mode: Append}) + RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_preconfigured.toml", Mode: Append}, nil) require.NoError(t, err) _, err = config.GenerateHistoryArchive(ctx, nil) @@ -51,7 +50,7 @@ func TestGenerateHistoryArchiveFromPreconfiguredNetwork(t *testing.T) { func TestGenerateHistoryArchiveFromManulConfiguredNetwork(t *testing.T) { ctx := context.Background() config, err := NewConfig( - RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_manual.toml", Mode: Append}) + RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_manual.toml", Mode: Append}, nil) require.NoError(t, err) _, err = config.GenerateHistoryArchive(ctx, nil) @@ -60,7 +59,7 @@ func TestGenerateHistoryArchiveFromManulConfiguredNetwork(t *testing.T) { func TestNewConfigUserAgent(t *testing.T) { config, err := NewConfig( - RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/useragent.toml"}) + RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/useragent.toml"}, nil) require.NoError(t, err) require.Equal(t, config.UserAgent, "useragent_x") } @@ -68,20 +67,20 @@ func TestNewConfigUserAgent(t *testing.T) { func TestResumeDisabled(t *testing.T) { // resumable is only enabled when mode is Append config, err := NewConfig( - RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: ScanFill}) + RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: ScanFill}, nil) require.NoError(t, err) require.False(t, config.Resumable()) } func TestInvalidConfigFilePath(t *testing.T) { _, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/notfound.toml"}) + RuntimeSettings{ConfigFilePath: "test/notfound.toml"}, nil) require.ErrorContains(t, err, "config file test/notfound.toml was not found") } func TestNoCaptiveCoreBin(t *testing.T) { cfg, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}) + RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}, nil) require.NoError(t, err) _, err = cfg.GenerateCaptiveCoreConfig("") @@ -90,10 +89,10 @@ func TestNoCaptiveCoreBin(t *testing.T) { func TestDefaultCaptiveCoreBin(t *testing.T) { cfg, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}) + RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}, + func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }) require.NoError(t, err) - ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("/test/default/stellar-core") require.NoError(t, err) require.Equal(t, ccConfig.BinaryPath, "/test/default/stellar-core") @@ -101,20 +100,20 @@ func TestDefaultCaptiveCoreBin(t *testing.T) { func TestInvalidCaptiveCorePreconfiguredNetwork(t *testing.T) { _, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/invalid_preconfigured_network.toml"}) + RuntimeSettings{ConfigFilePath: "test/invalid_preconfigured_network.toml"}, nil) require.ErrorContains(t, err, "invalid captive core config") } func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) { cfg, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/valid_captive_core_preconfigured.toml"}) + RuntimeSettings{ConfigFilePath: "test/valid_captive_core_preconfigured.toml"}, + func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }) require.NoError(t, err) require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs) - ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -129,13 +128,13 @@ func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) { func TestValidCaptiveCoreManualNetwork(t *testing.T) { cfg, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/valid_captive_core_manual.toml"}) + RuntimeSettings{ConfigFilePath: "test/valid_captive_core_manual.toml"}, + func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }) require.NoError(t, err) require.Equal(t, cfg.CoreVersion, "") require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, "test") require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"}) - ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -150,12 +149,12 @@ func TestValidCaptiveCoreManualNetwork(t *testing.T) { func TestValidCaptiveCoreOverridenToml(t *testing.T) { cfg, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override.toml"}) + RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override.toml"}, + func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil }) require.NoError(t, err) require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs) - ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -171,13 +170,13 @@ func TestValidCaptiveCoreOverridenToml(t *testing.T) { func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) { cfg, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override_archives.toml"}) + RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override_archives.toml"}, + func(string) (string, error) { return "v20.2.0-2-g6e73c0a88\n", nil }) require.NoError(t, err) require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase) require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"}) - ledgerbackend.GetCoreBuildVersionFunc = func(string) (string, error) { return "v20.2.0-2-g6e73c0a88\n", nil } ccConfig, err := cfg.GenerateCaptiveCoreConfig("") require.NoError(t, err) @@ -192,7 +191,8 @@ func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) { func TestInvalidCaptiveCoreTomlPath(t *testing.T) { _, err := NewConfig( - RuntimeSettings{ConfigFilePath: "test/invalid_captive_core_toml_path.toml"}) + RuntimeSettings{ConfigFilePath: "test/invalid_captive_core_toml_path.toml"}, + nil) require.ErrorContains(t, err, "Failed to load captive-core-toml-path file") } @@ -291,7 +291,7 @@ func TestValidateStartAndEndLedger(t *testing.T) { mockedHasCtr++ } config, err := NewConfig( - RuntimeSettings{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml", Mode: tt.mode}) + RuntimeSettings{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml", Mode: tt.mode}, nil) require.NoError(t, err) err = config.ValidateAndSetLedgerRange(ctx, mockArchive) if tt.errMsg != "" { @@ -363,7 +363,7 @@ func TestAdjustedLedgerRangeBoundedMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { config, err := NewConfig( - RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: ScanFill}) + RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: ScanFill}, nil) require.NoError(t, err) err = config.ValidateAndSetLedgerRange(ctx, mockArchive) @@ -426,7 +426,7 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { config, err := NewConfig( - RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: Append}) + RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: Append}, nil) require.NoError(t, err) err = config.ValidateAndSetLedgerRange(ctx, mockArchive) require.NoError(t, err) diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go index 75dd8cd06c..9d1f7f7fc0 100644 --- a/ingest/ledgerbackend/captive_core_backend.go +++ b/ingest/ledgerbackend/captive_core_backend.go @@ -20,7 +20,7 @@ import ( "github.com/stellar/go/xdr" ) -var minProtocolVersionSupported uint = 21 +const minProtocolVersionSupported uint = 21 // Ensure CaptiveStellarCore implements LedgerBackend var _ LedgerBackend = (*CaptiveStellarCore)(nil) @@ -154,6 +154,12 @@ type CaptiveCoreConfig struct { // of DATABASE parameter in the captive-core-config-path or if absent, the db will default to sqlite // and the db file will be stored at location derived from StoragePath parameter. UseDB bool + + // CoreProtocolVersionFn is a function that returns the protocol version of the stellar-core binary. + CoreProtocolVersionFn CoreProtocolVersionFunc + + // CoreBuildVersionFn is a function that returns the build version of the stellar-core binary. + CoreBuildVersionFn CoreBuildVersionFunc } // NewCaptive returns a new CaptiveStellarCore instance. @@ -168,7 +174,15 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) { config.Log.SetLevel(logrus.InfoLevel) } - protocolVersion, err := GetCoreProtocolVersionFunc(config.BinaryPath) + if config.CoreProtocolVersionFn == nil { + config.CoreProtocolVersionFn = CoreProtocolVersion + } + + if config.CoreBuildVersionFn == nil { + config.CoreBuildVersionFn = CoreBuildVersion + } + + protocolVersion, err := config.CoreProtocolVersionFn(config.BinaryPath) if err != nil { return nil, fmt.Errorf("error determining stellar-core protocol version: %w", err) } @@ -263,7 +277,7 @@ func (c *CaptiveStellarCore) coreVersionMetric() float64 { func (c *CaptiveStellarCore) setCoreVersion() { var err error - c.captiveCoreVersion, err = GetCoreBuildVersionFunc(c.config.BinaryPath) + c.captiveCoreVersion, err = c.config.CoreBuildVersionFn(c.config.BinaryPath) if err != nil { c.config.Log.Errorf("Failed to set stellar-core version: %s", err) } diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go index 01a58af212..76319c2f77 100644 --- a/ingest/ledgerbackend/captive_core_backend_test.go +++ b/ingest/ledgerbackend/captive_core_backend_test.go @@ -151,15 +151,14 @@ func TestCaptiveNew(t *testing.T) { networkPassphrase := network.PublicNetworkPassphrase historyURLs := []string{server.URL} - GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } - captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ - BinaryPath: executablePath, - NetworkPassphrase: networkPassphrase, - HistoryArchiveURLs: historyURLs, - StoragePath: storagePath, - UserAgent: "uatest", + BinaryPath: executablePath, + NetworkPassphrase: networkPassphrase, + HistoryArchiveURLs: historyURLs, + StoragePath: storagePath, + UserAgent: "uatest", + CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil }, }, ) @@ -185,15 +184,14 @@ func TestCaptiveNewUnsupportedProtocolVersion(t *testing.T) { networkPassphrase := network.PublicNetworkPassphrase historyURLs := []string{server.URL} - GetCoreProtocolVersionFunc = func(string) (uint, error) { return 20, nil } - _, err = NewCaptive( CaptiveCoreConfig{ - BinaryPath: executablePath, - NetworkPassphrase: networkPassphrase, - HistoryArchiveURLs: historyURLs, - StoragePath: storagePath, - UserAgent: "uatest", + BinaryPath: executablePath, + NetworkPassphrase: networkPassphrase, + HistoryArchiveURLs: historyURLs, + StoragePath: storagePath, + UserAgent: "uatest", + CoreProtocolVersionFn: func(string) (uint, error) { return 20, nil }, }, ) @@ -1015,15 +1013,14 @@ func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) { captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{}) assert.NoError(t, err) - GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } - captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ - BinaryPath: executablePath, - NetworkPassphrase: networkPassphrase, - HistoryArchiveURLs: historyURLs, - Toml: captiveCoreToml, - StoragePath: storagePath, + BinaryPath: executablePath, + NetworkPassphrase: networkPassphrase, + HistoryArchiveURLs: historyURLs, + Toml: captiveCoreToml, + StoragePath: storagePath, + CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil }, }, ) assert.NoError(t, err) diff --git a/ingest/ledgerbackend/stellar_core_version.go b/ingest/ledgerbackend/stellar_core_version.go index dac53f3892..d5b085d934 100644 --- a/ingest/ledgerbackend/stellar_core_version.go +++ b/ingest/ledgerbackend/stellar_core_version.go @@ -10,14 +10,15 @@ import ( // By default, it points to exec.Command, overridden for testing purpose var execCommand = exec.Command -var GetCoreProtocolVersionFunc = GetCoreProtocolVersion -var GetCoreBuildVersionFunc = GetCoreBuildVersion -// GetCoreBuildVersion executes the "stellar-core version" command and parses its output to extract +type CoreBuildVersionFunc func(coreBinaryPath string) (string, error) +type CoreProtocolVersionFunc func(coreBinaryPath string) (uint, error) + +// CoreBuildVersion executes the "stellar-core version" command and parses its output to extract // the core version // The output of the "version" command is expected to be a multi-line string where the // first line is the core version in format "vX.Y.Z-*". -func GetCoreBuildVersion(coreBinaryPath string) (string, error) { +func CoreBuildVersion(coreBinaryPath string) (string, error) { versionCmd := execCommand(coreBinaryPath, "version") versionOutput, err := versionCmd.Output() if err != nil { @@ -33,9 +34,9 @@ func GetCoreBuildVersion(coreBinaryPath string) (string, error) { return rows[0], nil } -// GetCoreProtocolVersion retrieves the ledger protocol version from the specified stellar-core binary. +// CoreProtocolVersion retrieves the ledger protocol version from the specified stellar-core binary. // It executes the "stellar-core version" command and parses the output to extract the protocol version. -func GetCoreProtocolVersion(coreBinaryPath string) (uint, error) { +func CoreProtocolVersion(coreBinaryPath string) (uint, error) { if coreBinaryPath == "" { return 0, fmt.Errorf("stellar-core binary path is empty") } diff --git a/ingest/ledgerbackend/stellar_core_version_test.go b/ingest/ledgerbackend/stellar_core_version_test.go index 51bf2947c1..4d58b35875 100644 --- a/ingest/ledgerbackend/stellar_core_version_test.go +++ b/ingest/ledgerbackend/stellar_core_version_test.go @@ -64,7 +64,7 @@ func TestGetCoreBuildVersion(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fakeExecCmdOut = tt.commandOutput - coreVersion, err := GetCoreBuildVersion("") + coreVersion, err := CoreBuildVersion("") if tt.expectedError != nil { require.EqualError(t, err, tt.expectedError.Error()) @@ -110,7 +110,7 @@ func TestGetCoreProtcolVersion(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fakeExecCmdOut = tt.commandOutput - coreVersion, err := GetCoreProtocolVersion("/usr/bin/stellar-core") + coreVersion, err := CoreProtocolVersion("/usr/bin/stellar-core") if tt.expectedError != nil { require.EqualError(t, err, tt.expectedError.Error()) diff --git a/services/horizon/internal/ingest/db_integration_test.go b/services/horizon/internal/ingest/db_integration_test.go index 61998649f0..0ac6e9d796 100644 --- a/services/horizon/internal/ingest/db_integration_test.go +++ b/services/horizon/internal/ingest/db_integration_test.go @@ -100,7 +100,6 @@ func (s *DBTestSuite) SetupTest() { s.checkpointHash = xdr.Hash{1, 2, 3} s.ledgerBackend = &ledgerbackend.MockDatabaseBackend{} s.historyAdapter = &mockHistoryArchiveAdapter{} - ledgerbackend.GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } var err error sIface, err := NewSystem(Config{ @@ -108,6 +107,7 @@ func (s *DBTestSuite) SetupTest() { HistoryArchiveURLs: []string{"http://ignore.test"}, DisableStateVerification: false, CheckpointFrequency: 64, + CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil }, }) s.Assert().NoError(err) s.system = sIface.(*system) diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index 0769faee4f..98dcad34f4 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -109,6 +109,9 @@ type Config struct { MaxLedgerPerFlush uint32 SkipTxmeta bool + + CoreProtocolVersionFn ledgerbackend.CoreProtocolVersionFunc + CoreBuildVersionFn ledgerbackend.CoreBuildVersionFunc } const ( @@ -259,17 +262,19 @@ func NewSystem(config Config) (System, error) { logger := log.WithField("subservice", "stellar-core") ledgerBackend, err := ledgerbackend.NewCaptive( ledgerbackend.CaptiveCoreConfig{ - BinaryPath: config.CaptiveCoreBinaryPath, - StoragePath: config.CaptiveCoreStoragePath, - UseDB: config.CaptiveCoreConfigUseDB, - Toml: config.CaptiveCoreToml, - NetworkPassphrase: config.NetworkPassphrase, - HistoryArchiveURLs: config.HistoryArchiveURLs, - CheckpointFrequency: config.CheckpointFrequency, - LedgerHashStore: ledgerbackend.NewHorizonDBLedgerHashStore(config.HistorySession), - Log: logger, - Context: ctx, - UserAgent: fmt.Sprintf("captivecore horizon/%s golang/%s", apkg.Version(), runtime.Version()), + BinaryPath: config.CaptiveCoreBinaryPath, + StoragePath: config.CaptiveCoreStoragePath, + UseDB: config.CaptiveCoreConfigUseDB, + Toml: config.CaptiveCoreToml, + NetworkPassphrase: config.NetworkPassphrase, + HistoryArchiveURLs: config.HistoryArchiveURLs, + CheckpointFrequency: config.CheckpointFrequency, + LedgerHashStore: ledgerbackend.NewHorizonDBLedgerHashStore(config.HistorySession), + Log: logger, + Context: ctx, + UserAgent: fmt.Sprintf("captivecore horizon/%s golang/%s", apkg.Version(), runtime.Version()), + CoreProtocolVersionFn: config.CoreProtocolVersionFn, + CoreBuildVersionFn: config.CoreBuildVersionFn, }, ) if err != nil { diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go index 3efcd73e3b..40a3d48cce 100644 --- a/services/horizon/internal/ingest/main_test.go +++ b/services/horizon/internal/ingest/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "database/sql" + "reflect" "sync" "testing" "time" @@ -94,22 +95,39 @@ func TestNewSystem(t *testing.T) { DisableStateVerification: true, HistoryArchiveURLs: []string{"https://history.stellar.org/prd/core-live/core_live_001"}, CheckpointFrequency: 64, + CoreProtocolVersionFn: func(string) (uint, error) { return 21, nil }, } - ledgerbackend.GetCoreProtocolVersionFunc = func(string) (uint, error) { return 21, nil } - sIface, err := NewSystem(config) assert.NoError(t, err) system := sIface.(*system) - assert.Equal(t, config, system.config) + CompareConfigs(t, config, system.config) assert.Equal(t, config.DisableStateVerification, system.disableStateVerification) - assert.Equal(t, config, system.runner.(*ProcessorRunner).config) + CompareConfigs(t, config, system.runner.(*ProcessorRunner).config) assert.Equal(t, system.ctx, system.runner.(*ProcessorRunner).ctx) assert.Equal(t, system.maxLedgerPerFlush, MaxLedgersPerFlush) } +// Custom comparator function.This function is needed because structs in Go that contain function fields +// cannot be directly compared using assert.Equal, so here we compare each individual field, skipping the function fields. +func CompareConfigs(t *testing.T, expected, actual Config) bool { + fields := reflect.TypeOf(expected) + for i := 0; i < fields.NumField(); i++ { + field := fields.Field(i) + if field.Name == "CoreProtocolVersionFn" || field.Name == "CoreBuildVersionFn" { + continue + } + expectedValue := reflect.ValueOf(expected).Field(i).Interface() + actualValue := reflect.ValueOf(actual).Field(i).Interface() + if !assert.Equal(t, expectedValue, actualValue, field.Name) { + return false + } + } + return true +} + func TestStateMachineRunReturnsUnexpectedTransaction(t *testing.T) { historyQ := &mockDBQ{} system := &system{