Skip to content

Commit

Permalink
exp/services/ledgerexporter: Add metadata to exported files
Browse files Browse the repository at this point in the history
  • Loading branch information
@urvisavla
urvisavla committed May 28, 2024
1 parent afd526d commit 2720f57
Show file tree
Hide file tree
Showing 16 changed files with 618 additions and 101 deletions.
16 changes: 13 additions & 3 deletions exp/services/ledgerexporter/internal/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,16 @@ func NewApp(flags Flags) *App {
return app
}

// Version gets the version of the app from environment variable "LEXIE_VERSION" which is set during the build process.
// If the environment variable is not set, it defaults to "develop".
func Version() string {
version := os.Getenv("LEXIE_VERSION")
if version == "" {
version = "develop"
}
return version
}

func (a *App) init(ctx context.Context) error {
var err error
var archive historyarchive.ArchiveInterface
Expand All @@ -101,7 +111,7 @@ func (a *App) init(ctx context.Context) error {
collectors.NewGoCollector(),
)

if a.config, err = NewConfig(ctx, a.flags); err != nil {
if a.config, err = NewConfig(Version(), a.flags); err != nil {
return errors.Wrap(err, "Could not load configuration")
}
if archive, err = datastore.CreateHistoryArchiveFromNetworkName(ctx, a.config.Network); err != nil {
Expand All @@ -126,7 +136,7 @@ func (a *App) init(ctx context.Context) error {
}

queue := NewUploadQueue(uploadQueueCapacity, registry)
if a.exportManager, err = NewExportManager(a.config.LedgerBatchConfig, a.ledgerBackend, queue, registry); err != nil {
if a.exportManager, err = NewExportManager(a.config, a.ledgerBackend, queue, registry); err != nil {
return err
}
a.uploader = NewUploader(a.dataStore, queue, registry)
Expand Down Expand Up @@ -256,7 +266,7 @@ func (a *App) Run() {
// newLedgerBackend Creates and initializes captive core ledger backend
// Currently, only supports captive-core as ledger backend
func newLedgerBackend(config *Config, prometheusRegistry *prometheus.Registry) (ledgerbackend.LedgerBackend, error) {
captiveConfig, err := config.GenerateCaptiveCoreConfig()
captiveConfig, err := config.generateCaptiveCoreConfig()
if err != nil {
return nil, err
}
Expand Down
68 changes: 63 additions & 5 deletions exp/services/ledgerexporter/internal/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ package ledgerexporter
import (
"context"
_ "embed"
"fmt"
"os/exec"
"regexp"
"strings"

"github.com/stellar/go/historyarchive"
"github.com/stellar/go/ingest/ledgerbackend"
Expand Down Expand Up @@ -45,15 +48,20 @@ type Config struct {
StartLedger uint32
EndLedger uint32
Resume bool

// derived config
Version string // lexie version
CoreVersion string // stellar-core version
ProtocolVersion string // stellar protocol version
}

// This will generate the config based on commandline flags and toml
//
// ctx - the caller context
// version - Ledger Exporter version
// flags - command line flags
//
// return - *Config or an error if any range validation failed.
func NewConfig(ctx context.Context, flags Flags) (*Config, error) {
func NewConfig(version string, flags Flags) (*Config, error) {
config := &Config{}

config.StartLedger = uint32(flags.StartLedger)
Expand All @@ -67,6 +75,7 @@ func NewConfig(ctx context.Context, flags Flags) (*Config, error) {
return nil, err
}
logger.Infof("Config: %v", *config)
config.Version = version

return config, nil
}
Expand Down Expand Up @@ -100,16 +109,17 @@ func (config *Config) ValidateAndSetLedgerRange(ctx context.Context, archive his
return nil
}

func (config *Config) GenerateCaptiveCoreConfig() (ledgerbackend.CaptiveCoreConfig, error) {
func (config *Config) generateCaptiveCoreConfig() (ledgerbackend.CaptiveCoreConfig, error) {
coreConfig := &config.StellarCoreConfig

// Look for stellar-core binary in $PATH, if not supplied
if coreConfig.StellarCoreBinaryPath == "" {
if config.StellarCoreConfig.StellarCoreBinaryPath == "" {
var err error
if coreConfig.StellarCoreBinaryPath, err = exec.LookPath("stellar-core"); err != nil {
if config.StellarCoreConfig.StellarCoreBinaryPath, err = exec.LookPath("stellar-core"); err != nil {
return ledgerbackend.CaptiveCoreConfig{}, errors.Wrap(err, "Failed to find stellar-core binary")
}
}
config.setCoreVersionInfo()

var captiveCoreConfig []byte
// Default network config
Expand Down Expand Up @@ -151,6 +161,54 @@ func (config *Config) GenerateCaptiveCoreConfig() (ledgerbackend.CaptiveCoreConf
}, nil
}

// By default, it points to exec.Command, overridden for testing purpose
var execCommand = exec.Command

// retrieves and sets the core and protocol versions from the stellar-core binary.
// It executes the "stellar-core version" command and parses its output to extract
// the core version and ledger protocol version.
// The output of the "version" command is expected to be a multi-line string where:
// - The first line is the core version in format "vX.Y.Z-*".
// - One of the subsequent lines contains the protocol version in the format "ledger protocol version: X", where X is a number.
func (c *Config) setCoreVersionInfo() (err error) {
versionCmd := execCommand(c.StellarCoreConfig.StellarCoreBinaryPath, "version")
versionOutput, err := versionCmd.Output()
if err != nil {
return fmt.Errorf("failed to execute stellar-core version command: %w", err)
}

// Split the output into lines
rows := strings.Split(string(versionOutput), "\n")
if len(rows) == 0 {
return fmt.Errorf("stellar-core version command output is empty")
}

// Validate and set the core version
coreVersionPattern := `^v\d+\.\d+\.\d+`
if match, _ := regexp.MatchString(coreVersionPattern, rows[0]); !match {
return fmt.Errorf("core version not found in stellar-core version output")
}
c.CoreVersion = rows[0]

// Validate and set the protocol version
if len(rows) < 2 {
return fmt.Errorf("protocol version not found in stellar-core version output")
}

re := regexp.MustCompile(`ledger protocol version: (\d+)`)
for _, row := range rows[1:] {
if matches := re.FindStringSubmatch(row); len(matches) > 1 {
c.ProtocolVersion = matches[1]
break
}
}
if c.ProtocolVersion == "" {
return fmt.Errorf("protocol version not found in stellar-core version output")
}

return nil
}

func (config *Config) processToml(tomlPath string) error {
// Load config TOML file
cfg, err := toml.LoadFile(tomlPath)
Expand Down
104 changes: 95 additions & 9 deletions exp/services/ledgerexporter/internal/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@ package ledgerexporter
import (
"context"
"fmt"
"os"
"os/exec"
"testing"

"github.com/stellar/go/historyarchive"
"github.com/stretchr/testify/require"

"github.com/stellar/go/historyarchive"
"github.com/stellar/go/support/errors"
)

func TestNewConfigResumeEnabled(t *testing.T) {
Expand All @@ -15,7 +19,7 @@ func TestNewConfigResumeEnabled(t *testing.T) {
mockArchive := &historyarchive.MockArchive{}
mockArchive.On("GetRootHAS").Return(historyarchive.HistoryArchiveState{CurrentLedger: 5}, nil).Once()

config, err := NewConfig(ctx,
config, err := NewConfig("v1.0",
Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/test.toml", Resume: true})
config.ValidateAndSetLedgerRange(ctx, mockArchive)
require.NoError(t, err)
Expand All @@ -29,22 +33,20 @@ func TestNewConfigResumeEnabled(t *testing.T) {
}

func TestNewConfigResumeDisabled(t *testing.T) {
ctx := context.Background()

mockArchive := &historyarchive.MockArchive{}
mockArchive.On("GetRootHAS").Return(historyarchive.HistoryArchiveState{CurrentLedger: 5}, nil).Once()

// resume disabled by default
config, err := NewConfig(ctx,
config, err := NewConfig("v1.0",
Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/test.toml"})
require.NoError(t, err)
require.False(t, config.Resume)
}

func TestInvalidTomlConfig(t *testing.T) {
ctx := context.Background()

_, err := NewConfig(ctx,
_, err := NewConfig("v1.0",
Flags{StartLedger: 1, EndLedger: 2, ConfigFilePath: "test/no_network.toml", Resume: true})
require.ErrorContains(t, err, "Invalid TOML config")
}
Expand Down Expand Up @@ -110,7 +112,7 @@ func TestValidateStartAndEndLedger(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := NewConfig(ctx,
config, err := NewConfig("v1.0",
Flags{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml"})
require.NoError(t, err)
err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
Expand Down Expand Up @@ -188,7 +190,7 @@ func TestAdjustedLedgerRangeBoundedMode(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := NewConfig(ctx,
config, err := NewConfig("v1.0",
Flags{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile})
require.NoError(t, err)
err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
Expand Down Expand Up @@ -257,7 +259,7 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config, err := NewConfig(ctx,
config, err := NewConfig("v1.0",
Flags{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile})
require.NoError(t, err)
err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
Expand All @@ -267,3 +269,87 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
})
}
}

var cmdOut = ""

func fakeExecCommand(command string, args ...string) *exec.Cmd {
cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...)
cmd := exec.Command(os.Args[0], cs...)
cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+cmdOut)
return cmd
}

func TestExecCmdHelperProcess(t *testing.T) {
if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" {
return
}
fmt.Fprintf(os.Stdout, os.Getenv("CMD_OUT"))
os.Exit(0)
}

func TestSetCoreVersionInfo(t *testing.T) {
tests := []struct {
name string
commandOutput string
expectedError error
expectedCoreVer string
expectedProtoVer string
}{
{
name: "version found",
commandOutput: "v20.2.0-2-g6e73c0a88\n" +
"rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" +
"soroban-env-host: \n" +
" curr:\n" +
" package version: 20.2.0\n" +
" git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" +
" ledger protocol version: 20\n" +
" pre-release version: 0\n" +
" rs-stellar-xdr:\n" +
" package version: 20.1.0\n" +
" git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" +
" base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n",
expectedError: nil,
expectedCoreVer: "v20.2.0-2-g6e73c0a88",
expectedProtoVer: "20",
},
{
name: "protocol version not found",
commandOutput: "v20.2.0-2-g6e73c0a88\n",
expectedError: errors.New("protocol version not found in stellar-core version output"),
expectedCoreVer: "v20.2.0-2-g6e73c0a88",
expectedProtoVer: "",
},
{
name: "core version invalid format",
commandOutput: "ledger protocol version: 20\\n\" +",
expectedError: errors.New("core version not found in stellar-core version output"),
expectedCoreVer: "",
expectedProtoVer: "",
},
{
name: "core version not found",
commandOutput: "",
expectedError: errors.New("core version not found in stellar-core version output"),
expectedCoreVer: "",
expectedProtoVer: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config := Config{}

cmdOut = tt.commandOutput
execCommand = fakeExecCommand
err := config.setCoreVersionInfo()

if tt.expectedError != nil {
require.EqualError(t, err, tt.expectedError.Error())
} else {
require.NoError(t, err)
require.Equal(t, tt.expectedCoreVer, config.CoreVersion)
require.Equal(t, tt.expectedProtoVer, config.ProtocolVersion)
}
})
}
}
Loading

0 comments on commit 2720f57

Please sign in to comment.