Skip to content

Commit

Permalink
ingest/ledgerbackend: Add version check for protocol 21 (#5346)
Browse files Browse the repository at this point in the history 
* Remove protocol 20 from integration tests

* Implement a version check for protocol 21 when creating a new captive-core instance.

* Add common functions to get the stellar-core build version and protocol version and update ledgerexporter and captivecorebackend to use these functions.
  • Loading branch information
@urvisavla
urvisavla authored Jun 20, 2024
1 parent 100dc4f commit 0b7df85
Show file tree
Hide file tree
Showing 18 changed files with 340 additions and 336 deletions.
7 changes: 2 additions & 5 deletions .github/workflows/horizon.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
os: [ubuntu-20.04, ubuntu-22.04]
go: ["1.21", "1.22"]
pg: [12, 16]
protocol-version: [20, 21]
protocol-version: [21]
runs-on: ${{ matrix.os }}
services:
postgres:
Expand All @@ -36,9 +36,6 @@ jobs:
PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21
PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21
PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
PGHOST: localhost
PGPORT: 5432
PGUSER: postgres
Expand Down Expand Up @@ -101,7 +98,7 @@ jobs:
- name: Calculate the source hash
id: calculate_source_hash
run: |
combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_20_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_19_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV"
- name: Restore Horizon binary and integration tests source hash to cache
Expand Down
2 changes: 1 addition & 1 deletion exp/services/ledgerexporter/internal/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (a *App) init(ctx context.Context, runtimeSettings RuntimeSettings) error {
collectors.NewGoCollector(),
)

if a.config, err = NewConfig(runtimeSettings); err != nil {
if a.config, err = NewConfig(runtimeSettings, nil); err != nil {
return errors.Wrap(err, "Could not load configuration")
}
if archive, err = a.config.GenerateHistoryArchive(ctx, logger); err != nil {
Expand Down
28 changes: 8 additions & 20 deletions exp/services/ledgerexporter/internal/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ import (
_ "embed"
"fmt"
"os"
"os/exec"
"strings"

"github.com/stellar/go/historyarchive"
"github.com/stellar/go/ingest/ledgerbackend"
Expand Down Expand Up @@ -73,19 +71,24 @@ type Config struct {

CoreVersion string
SerializedCaptiveCoreToml []byte
CoreBuildVersionFn ledgerbackend.CoreBuildVersionFunc
}

// This will generate the config based on settings
//
// settings - requested settings
//
// return - *Config or an error if any range validation failed.
func NewConfig(settings RuntimeSettings) (*Config, error) {
func NewConfig(settings RuntimeSettings, getCoreVersionFn ledgerbackend.CoreBuildVersionFunc) (*Config, error) {
config := &Config{}

config.StartLedger = uint32(settings.StartLedger)
config.EndLedger = uint32(settings.EndLedger)
config.Mode = settings.Mode
config.CoreBuildVersionFn = ledgerbackend.CoreBuildVersion
if getCoreVersionFn != nil {
config.CoreBuildVersionFn = getCoreVersionFn
}

logger.Infof("Requested export mode of %v with start=%d, end=%d", settings.Mode.Name(), config.StartLedger, config.EndLedger)

Expand Down Expand Up @@ -194,26 +197,11 @@ func (config *Config) GenerateCaptiveCoreConfig(coreBinFromPath string) (ledgerb
}, nil
}

// By default, it points to exec.Command, overridden for testing purpose
var execCommand = exec.Command

// Executes the "stellar-core version" command and parses its output to extract
// the core version
// The output of the "version" command is expected to be a multi-line string where the
// first line is the core version in format "vX.Y.Z-*".
func (c *Config) setCoreVersionInfo() (err error) {
versionCmd := execCommand(c.StellarCoreConfig.StellarCoreBinaryPath, "version")
versionOutput, err := versionCmd.Output()
c.CoreVersion, err = c.CoreBuildVersionFn(c.StellarCoreConfig.StellarCoreBinaryPath)
if err != nil {
return fmt.Errorf("failed to execute stellar-core version command: %w", err)
}

// Split the output into lines
rows := strings.Split(string(versionOutput), "\n")
if len(rows) == 0 || len(rows[0]) == 0 {
return fmt.Errorf("stellar-core version not found")
return fmt.Errorf("failed to set stellar-core version: %w", err)
}
c.CoreVersion = rows[0]
logger.Infof("stellar-core version: %s", c.CoreVersion)
return nil
}
Expand Down
117 changes: 23 additions & 94 deletions exp/services/ledgerexporter/internal/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package ledgerexporter
import (
"context"
"fmt"
"os"
"os/exec"
"testing"

"github.com/stellar/go/network"
Expand All @@ -13,7 +11,6 @@ import (
"github.com/stretchr/testify/require"

"github.com/stellar/go/historyarchive"
"github.com/stellar/go/support/errors"
)

func TestNewConfig(t *testing.T) {
Expand All @@ -23,7 +20,7 @@ func TestNewConfig(t *testing.T) {
mockArchive.On("GetRootHAS").Return(historyarchive.HistoryArchiveState{CurrentLedger: 5}, nil).Once()

config, err := NewConfig(
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: Append})
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: Append}, nil)

require.NoError(t, err)
err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
Expand All @@ -43,7 +40,7 @@ func TestNewConfig(t *testing.T) {
func TestGenerateHistoryArchiveFromPreconfiguredNetwork(t *testing.T) {
ctx := context.Background()
config, err := NewConfig(
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_preconfigured.toml", Mode: Append})
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_preconfigured.toml", Mode: Append}, nil)
require.NoError(t, err)

_, err = config.GenerateHistoryArchive(ctx, nil)
Expand All @@ -53,7 +50,7 @@ func TestGenerateHistoryArchiveFromPreconfiguredNetwork(t *testing.T) {
func TestGenerateHistoryArchiveFromManulConfiguredNetwork(t *testing.T) {
ctx := context.Background()
config, err := NewConfig(
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_manual.toml", Mode: Append})
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_manual.toml", Mode: Append}, nil)
require.NoError(t, err)

_, err = config.GenerateHistoryArchive(ctx, nil)
Expand All @@ -62,28 +59,28 @@ func TestGenerateHistoryArchiveFromManulConfiguredNetwork(t *testing.T) {

func TestNewConfigUserAgent(t *testing.T) {
config, err := NewConfig(
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/useragent.toml"})
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/useragent.toml"}, nil)
require.NoError(t, err)
require.Equal(t, config.UserAgent, "useragent_x")
}

func TestResumeDisabled(t *testing.T) {
// resumable is only enabled when mode is Append
config, err := NewConfig(
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: ScanFill})
RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: ScanFill}, nil)
require.NoError(t, err)
require.False(t, config.Resumable())
}

func TestInvalidConfigFilePath(t *testing.T) {
_, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/notfound.toml"})
RuntimeSettings{ConfigFilePath: "test/notfound.toml"}, nil)
require.ErrorContains(t, err, "config file test/notfound.toml was not found")
}

func TestNoCaptiveCoreBin(t *testing.T) {
cfg, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"})
RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}, nil)
require.NoError(t, err)

_, err = cfg.GenerateCaptiveCoreConfig("")
Expand All @@ -92,31 +89,31 @@ func TestNoCaptiveCoreBin(t *testing.T) {

func TestDefaultCaptiveCoreBin(t *testing.T) {
cfg, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"})
RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"},
func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
require.NoError(t, err)

cmdOut = "v20.2.0-2-g6e73c0a88\n"
ccConfig, err := cfg.GenerateCaptiveCoreConfig("/test/default/stellar-core")
require.NoError(t, err)
require.Equal(t, ccConfig.BinaryPath, "/test/default/stellar-core")
}

func TestInvalidCaptiveCorePreconfiguredNetwork(t *testing.T) {
_, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/invalid_preconfigured_network.toml"})
RuntimeSettings{ConfigFilePath: "test/invalid_preconfigured_network.toml"}, nil)

require.ErrorContains(t, err, "invalid captive core config")
}

func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) {
cfg, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_preconfigured.toml"})
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_preconfigured.toml"},
func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
require.NoError(t, err)

require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs)

cmdOut = "v20.2.0-2-g6e73c0a88\n"
ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
require.NoError(t, err)

Expand All @@ -131,13 +128,13 @@ func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) {

func TestValidCaptiveCoreManualNetwork(t *testing.T) {
cfg, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_manual.toml"})
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_manual.toml"},
func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
require.NoError(t, err)
require.Equal(t, cfg.CoreVersion, "")
require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, "test")
require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"})

cmdOut = "v20.2.0-2-g6e73c0a88\n"
ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
require.NoError(t, err)

Expand All @@ -152,12 +149,12 @@ func TestValidCaptiveCoreManualNetwork(t *testing.T) {

func TestValidCaptiveCoreOverridenToml(t *testing.T) {
cfg, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override.toml"})
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override.toml"},
func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
require.NoError(t, err)
require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs)

cmdOut = "v20.2.0-2-g6e73c0a88\n"
ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
require.NoError(t, err)

Expand All @@ -173,13 +170,13 @@ func TestValidCaptiveCoreOverridenToml(t *testing.T) {

func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) {
cfg, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override_archives.toml"})
RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override_archives.toml"},
func(string) (string, error) { return "v20.2.0-2-g6e73c0a88\n", nil })
require.NoError(t, err)

require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http://testarchive"})

cmdOut = "v20.2.0-2-g6e73c0a88\n"
ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
require.NoError(t, err)

Expand All @@ -194,7 +191,8 @@ func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) {

func TestInvalidCaptiveCoreTomlPath(t *testing.T) {
_, err := NewConfig(
RuntimeSettings{ConfigFilePath: "test/invalid_captive_core_toml_path.toml"})
RuntimeSettings{ConfigFilePath: "test/invalid_captive_core_toml_path.toml"},
nil)
require.ErrorContains(t, err, "Failed to load captive-core-toml-path file")
}

Expand Down Expand Up @@ -293,7 +291,7 @@ func TestValidateStartAndEndLedger(t *testing.T) {
mockedHasCtr++
}
config, err := NewConfig(
RuntimeSettings{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml", Mode: tt.mode})
RuntimeSettings{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml", Mode: tt.mode}, nil)
require.NoError(t, err)
err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
if tt.errMsg != "" {
Expand Down Expand Up @@ -365,7 +363,7 @@ func TestAdjustedLedgerRangeBoundedMode(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := NewConfig(
RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: ScanFill})
RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: ScanFill}, nil)

require.NoError(t, err)
err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
Expand Down Expand Up @@ -428,7 +426,7 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := NewConfig(
RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: Append})
RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: Append}, nil)
require.NoError(t, err)
err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
require.NoError(t, err)
Expand All @@ -438,72 +436,3 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
}
mockArchive.AssertExpectations(t)
}

var cmdOut = ""

func fakeExecCommand(command string, args ...string) *exec.Cmd {
cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...)
cmd := exec.Command(os.Args[0], cs...)
cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+cmdOut)
return cmd
}

func init() {
execCommand = fakeExecCommand
}

func TestExecCmdHelperProcess(t *testing.T) {
if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" {
return
}
fmt.Fprint(os.Stdout, os.Getenv("CMD_OUT"))
os.Exit(0)
}

func TestSetCoreVersionInfo(t *testing.T) {
execCommand = fakeExecCommand
tests := []struct {
name string
commandOutput string
expectedError error
expectedCoreVer string
}{
{
name: "version found",
commandOutput: "v20.2.0-2-g6e73c0a88\n" +
"rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" +
"soroban-env-host: \n" +
" curr:\n" +
" package version: 20.2.0\n" +
" git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" +
" ledger protocol version: 20\n" +
" pre-release version: 0\n" +
" rs-stellar-xdr:\n" +
" package version: 20.1.0\n" +
" git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" +
" base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n",
expectedError: nil,
expectedCoreVer: "v20.2.0-2-g6e73c0a88",
},
{
name: "core version not found",
commandOutput: "",
expectedError: errors.New("stellar-core version not found"),
expectedCoreVer: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config := Config{}
cmdOut = tt.commandOutput
err := config.setCoreVersionInfo()

if tt.expectedError != nil {
require.EqualError(t, err, tt.expectedError.Error())
} else {
require.NoError(t, err)
require.Equal(t, tt.expectedCoreVer, config.CoreVersion)
}
})
}
}
Loading

0 comments on commit 0b7df85

Please sign in to comment.