- Several changes to integration.go to pull out common functions from…

… parameters_test.go to configure captive core

- Add flags to capture state for upgrade version in test.
- Add support to skip container deletion for debugging

- Add test for upgrade related changes

services/horizon/internal/integration/change_test.go

@@ -3,18 +3,19 @@ package integration
 import (
 	"context"
 	"github.com/stellar/go/historyarchive"
+	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/services/horizon/internal/test/integration"
 	"github.com/stellar/go/xdr"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
+	"io"
 	"testing"
 	"time"
 )
 
-func TestCoreDump(t *testing.T) {
+func TestProtocolUpgradeChanges(t *testing.T) {
 	tt := assert.New(t)
-	itest := integration.NewTest(t, integration.Config{SkipHorizonStart: true})
+	itest := integration.NewTest(t, integration.Config{SkipHorizonStart: true, SkipProtocolUpgrade: true})
 	archive, err := historyarchive.Connect(
 		integration.HistoryArchiveUrl,
 		historyarchive.ArchiveOptions{
@@ -23,66 +24,90 @@ func TestCoreDump(t *testing.T) {
 		})
 	tt.NoError(err)
 
+	// Manually invoke command to upgrade protocol
+	itest.UpgradeProtocol(itest.Config().ProtocolVersion)
+	upgradedLedgerSeq, _ := itest.GetUpgradeLedgerSeq()
+
 	var latestCheckpoint uint32
-	startTime := time.Now()
 	publishedNextCheckpoint := func() bool {
 		has, requestErr := archive.GetRootHAS()
 		if requestErr != nil {
 			t.Logf("request to fetch checkpoint failed: %v", requestErr)
 			return false
 		}
 		latestCheckpoint = has.CurrentLedger
-		t.Logf("Latest ledger so far: %d", latestCheckpoint)
-		return latestCheckpoint >= uint32(7) // ALLOW for atleast 3 checkpoints
+		return latestCheckpoint >= upgradedLedgerSeq
 	}
-	//time.Sleep(15 * time.Second)
-
 	// Ensure that a checkpoint has been created with the ledgerNumber you want in it
-	tt.Eventually(publishedNextCheckpoint, 45*time.Second, time.Second)
-	endTime := time.Now()
+	tt.Eventually(publishedNextCheckpoint, 15*time.Second, time.Second)
 
-	t.Logf("waited %v seconds to start captive core...", endTime.Sub(startTime).Seconds())
-	t.Log("---------- STARTING CAPTIVE CORE ---------")
+	prevLedgerToUpgrade := upgradedLedgerSeq - 1
+	ledgerSeqToLedgers := getLedgersFromArchive(itest, prevLedgerToUpgrade, upgradedLedgerSeq)
+	prevLedgerChangeMap := changeMap(getChangesFromLedger(itest, ledgerSeqToLedgers[prevLedgerToUpgrade]))
+	upgradedLedgerChangeMap := changeMap(getChangesFromLedger(itest, ledgerSeqToLedgers[upgradedLedgerSeq]))
+
+	tt.Zero(prevLedgerChangeMap[ingest.LedgerEntryChangeReasonUpgrade])
+	tt.NotZero(upgradedLedgerChangeMap[ingest.LedgerEntryChangeReasonUpgrade])
+}
 
-	ledgerSeqToLedgers := getLedgersFromArchive(itest, 2, 7)
-	t.Logf("----- length of hashmap is %v", len(ledgerSeqToLedgers))
-	time.Sleep(45 * time.Second)
+func getChangesFromLedger(itest *integration.Test, ledger xdr.LedgerCloseMeta) []ingest.Change {
+	t := itest.CurrentTest()
+	changeReader, err := ingest.NewLedgerChangeReaderFromLedgerCloseMeta(itest.GetPassPhrase(), ledger)
+	changes := make([]ingest.Change, 0)
+	defer changeReader.Close()
+	if err != nil {
+		t.Fatalf("unable to create ledger change reader: %v", err)
+	}
+	for {
+		change, err := changeReader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("unable to read ledger change: %v", err)
+		}
+		changes = append(changes, change)
+	}
+	return changes
 }
 
 func getLedgersFromArchive(itest *integration.Test, startingLedger uint32, endLedger uint32) map[uint32]xdr.LedgerCloseMeta {
 	t := itest.CurrentTest()
 
-	ccConfig, cleanpupFn, err := itest.CreateCaptiveCoreConfig()
+	ccConfig, cleanupFn, err := itest.CreateCaptiveCoreConfig()
 	if err != nil {
-		panic(err)
+		t.Fatalf("unable to create captive core config: %v", err)
 	}
+	defer cleanupFn()
 
-	defer cleanpupFn()
 	captiveCore, err := ledgerbackend.NewCaptive(*ccConfig)
 	if err != nil {
-		panic(err)
+		t.Fatalf("unable to create captive core: %v", err)
 	}
 	defer captiveCore.Close()
 
 	ctx := context.Background()
-	require.NoError(t, err)
-
 	err = captiveCore.PrepareRange(ctx, ledgerbackend.BoundedRange(startingLedger, endLedger))
 	if err != nil {
 		t.Fatalf("failed to prepare range: %v", err)
 	}
 
-	t.Logf("Ledger Range ----- [%v, %v]", startingLedger, endLedger)
-
 	var seqToLedgersMap = make(map[uint32]xdr.LedgerCloseMeta)
 	for ledgerSeq := startingLedger; ledgerSeq <= endLedger; ledgerSeq++ {
 		ledger, err := captiveCore.GetLedger(ctx, ledgerSeq)
 		if err != nil {
 			t.Fatalf("failed to get ledgerNum: %v, error: %v", ledgerSeq, err)
 		}
 		seqToLedgersMap[ledgerSeq] = ledger
-		itest.CurrentTest().Logf("processed ledgerNum: %v, hash: %v", ledgerSeq, ledger.LedgerHash().HexString())
 	}
 
 	return seqToLedgersMap
 }
+
+func changeMap(changes []ingest.Change) map[ingest.LedgerEntryChangeReason]int {
+	changeMap := make(map[ingest.LedgerEntryChangeReason]int)
+	for _, change := range changes {
+		changeMap[change.Reason]++
+	}
+	return changeMap
+}

services/horizon/internal/integration/parameters_test.go

@@ -41,34 +41,16 @@ var networkParamArgs = map[string]string{
 	horizon.NetworkPassphraseFlagName:   "",
 }
 
-const (
-	SimpleCaptiveCoreToml = `
-		PEER_PORT=11725
-		ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
-
-		UNSAFE_QUORUM=true
-		FAILURE_SAFETY=0
-
-		[[VALIDATORS]]
-		NAME="local_core"
-		HOME_DOMAIN="core.local"
-		PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"
-		ADDRESS="localhost"
-		QUALITY="MEDIUM"`
-
-	StellarCoreURL = "http://localhost:11626"
-)
-
 var (
 	CaptiveCoreConfigErrMsg = "error generating captive core configuration: invalid config: "
 )
 
 // Ensures that BUCKET_DIR_PATH is not an allowed value for Captive Core.
 func TestBucketDirDisallowed(t *testing.T) {
 	config := `BUCKET_DIR_PATH="/tmp"
-		` + SimpleCaptiveCoreToml
+		` + integration.SimpleCaptiveCoreToml
 
-	confName, _, cleanup := createCaptiveCoreConfig(config)
+	confName, _, cleanup := integration.CreateCaptiveCoreConfig(config)
 	defer cleanup()
 	testConfig := integration.GetTestConfig()
 	testConfig.HorizonIngestParameters = map[string]string{
@@ -103,7 +85,7 @@ func TestEnvironmentPreserved(t *testing.T) {
 
 	testConfig := integration.GetTestConfig()
 	testConfig.HorizonEnvironment = map[string]string{
-		"STELLAR_CORE_URL": StellarCoreURL,
+		"STELLAR_CORE_URL": integration.StellarCoreURL,
 	}
 	test := integration.NewTest(t, *testConfig)
 
@@ -112,7 +94,7 @@ func TestEnvironmentPreserved(t *testing.T) {
 	test.WaitForHorizonIngest()
 
 	envValue := os.Getenv("STELLAR_CORE_URL")
-	assert.Equal(t, StellarCoreURL, envValue)
+	assert.Equal(t, integration.StellarCoreURL, envValue)
 
 	test.Shutdown()
 
@@ -252,7 +234,7 @@ func TestNetworkEnvironmentVariable(t *testing.T) {
 
 // Ensures that the filesystem ends up in the correct state with Captive Core.
 func TestCaptiveCoreConfigFilesystemState(t *testing.T) {
-	confName, storagePath, cleanup := createCaptiveCoreConfig(SimpleCaptiveCoreToml)
+	confName, storagePath, cleanup := integration.CreateCaptiveCoreConfig(integration.SimpleCaptiveCoreToml)
 	defer cleanup()
 
 	localParams := integration.MergeMaps(defaultCaptiveCoreParameters, map[string]string{
@@ -671,30 +653,3 @@ func validateCaptiveCoreDiskState(itest *integration.Test, rootDir string) {
 	tt.DirExists(storageDir)
 	tt.FileExists(coreConf)
 }
-
-// createCaptiveCoreConfig will create a temporary TOML config with the
-// specified contents as well as a temporary storage directory. You should
-// `defer` the returned function to clean these up when you're done.
-func createCaptiveCoreConfig(contents string) (string, string, func()) {
-	tomlFile, err := ioutil.TempFile("", "captive-core-test-*.toml")
-	defer tomlFile.Close()
-	if err != nil {
-		panic(err)
-	}
-
-	_, err = tomlFile.WriteString(contents)
-	if err != nil {
-		panic(err)
-	}
-
-	storagePath, err := os.MkdirTemp("", "captive-core-test-*-storage")
-	if err != nil {
-		panic(err)
-	}
-
-	filename := tomlFile.Name()
-	return filename, storagePath, func() {
-		os.Remove(filename)
-		os.RemoveAll(storagePath)
-	}
-}