Merge branch 'master' into new-core-endpoints

.github/workflows/galexie-release.yml

@@ -16,7 +16,7 @@ jobs:
       # this is the multi-arch index sha, get it by 'docker buildx imagetools inspect stellar/quickstart:testing'
       GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE: docker.io/stellar/quickstart:testing@sha256:03c6679f838a92b1eda4cd3a9e2bdee4c3586e278a138a0acf36a9bc99a0041f
       GALEXIE_INTEGRATION_TESTS_QUICKSTART_IMAGE_PULL: "false"
-      STELLAR_CORE_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      STELLAR_CORE_VERSION: 21.3.1-2007.4ede19620.focal
     steps:
       - name: Set VERSION
         run: |

.github/workflows/galexie.yml

@@ -10,7 +10,7 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     env:
-      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.1.0-1921.b3aeb14cc.focal
+      CAPTIVE_CORE_DEBIAN_PKG_VERSION: 21.3.1-2007.4ede19620.focal
       GALEXIE_INTEGRATION_TESTS_ENABLED: "true"
       GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       # this pins to a version of quickstart:testing that has the same version as GALEXIE_INTEGRATION_TESTS_CAPTIVE_CORE_BIN

.github/workflows/horizon.yml

@@ -33,9 +33,9 @@ jobs:
       HORIZON_INTEGRATION_TESTS_ENABLED: true
       HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
       HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true
-      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
-      PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21
-      PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
+      PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.3.1-2007.4ede19620.focal
+      PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21.3.1-2007.4ede19620.focal
+      PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.5.1
       PGHOST: localhost
       PGPORT: 5432
       PGUSER: postgres
@@ -98,7 +98,7 @@ jobs:
       - name: Calculate the source hash
         id: calculate_source_hash
         run: |
-          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
+          combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_21_RPC_DOCKER_IMG }}-${{ env.PROTOCOL_21_CORE_DEBIAN_PKG_VERSION }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
           echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV"
 
       - name: Restore Horizon binary and integration tests source hash to cache
@@ -123,7 +123,7 @@ jobs:
     name: Test (and push) verify-range image
     runs-on: ubuntu-22.04
     env:
-      STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal
+      STELLAR_CORE_VERSION: 21.3.1-2007.4ede19620.focal
       CAPTIVE_CORE_STORAGE_PATH: /tmp
     steps:
       - uses: actions/checkout@v3

ingest/README.md

@@ -23,9 +23,9 @@ From a high level, the ingestion library is broken down into a few modular compo
 
                 [ Ledger Backend ]
                         |
-                        |
-                     Captive  
-                      Core 
+                    |---+---|
+                 Captive Buffered
+                  Core   Storage
 ```
 
 This is described in a little more detail in [`doc.go`](./doc.go), its accompanying examples, the documentation within this package, and the rest of this tutorial.

ingest/ledgerbackend/buffered_storage_backend_test.go

@@ -505,7 +505,9 @@ func TestLedgerBufferBoundedObjectNotFound(t *testing.T) {
 	bsb.ledgerBuffer.wg.Wait()
 
 	_, err := bsb.GetLedger(ctx, 3)
-	assert.EqualError(t, err, "failed getting next ledger batch from queue: ledger object containing sequence 3 is missing: file does not exist")
+	assert.ErrorContains(t, err, "ledger object containing sequence 3 is missing")
+	assert.ErrorContains(t, err, objectName)
+	assert.ErrorContains(t, err, "file does not exist")
 }
 
 func TestLedgerBufferUnboundedObjectNotFound(t *testing.T) {
@@ -571,5 +573,8 @@ func TestLedgerBufferRetryLimit(t *testing.T) {
 	bsb.ledgerBuffer.wg.Wait()
 
 	_, err := bsb.GetLedger(context.Background(), 3)
-	assert.EqualError(t, err, "failed getting next ledger batch from queue: maximum retries exceeded for downloading object containing sequence 3: transient error")
+	assert.ErrorContains(t, err, "failed getting next ledger batch from queue")
+	assert.ErrorContains(t, err, "maximum retries exceeded for downloading object containing sequence 3")
+	assert.ErrorContains(t, err, objectName)
+	assert.ErrorContains(t, err, "transient error")
 }

ingest/ledgerbackend/captive_core_backend.go

@@ -112,9 +112,11 @@ type CaptiveStellarCore struct {
 	lastLedger         *uint32 // end of current segment if offline, nil if online
 	previousLedgerHash *string
 
-	config             CaptiveCoreConfig
-	stellarCoreClient  *stellarcore.Client
-	captiveCoreVersion string // Updates when captive-core restarts
+	config                   CaptiveCoreConfig
+	captiveCoreStartDuration prometheus.Summary
+	captiveCoreNewDBCounter  prometheus.Counter
+	stellarCoreClient        *stellarcore.Client
+	captiveCoreVersion       string // Updates when captive-core restarts
 }
 
 // CaptiveCoreConfig contains all the parameters required to create a CaptiveStellarCore instance
@@ -230,7 +232,7 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) {
 
 	c.stellarCoreRunnerFactory = func() stellarCoreRunnerInterface {
 		c.setCoreVersion()
-		return newStellarCoreRunner(config)
+		return newStellarCoreRunner(config, c.captiveCoreNewDBCounter)
 	}
 
 	if config.Toml != nil && config.Toml.HTTPPort != 0 {
@@ -315,7 +317,27 @@ func (c *CaptiveStellarCore) registerMetrics(registry *prometheus.Registry, name
 			return float64(latest)
 		},
 	)
-	registry.MustRegister(coreSynced, supportedProtocolVersion, latestLedger)
+	c.captiveCoreStartDuration = prometheus.NewSummary(prometheus.SummaryOpts{
+		Namespace:  namespace,
+		Subsystem:  "ingest",
+		Name:       "captive_stellar_core_start_duration_seconds",
+		Help:       "duration of start up time when running captive core on an unbounded range, sliding window = 10m",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+	})
+	c.captiveCoreNewDBCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: "ingest",
+		Name:      "captive_stellar_core_new_db",
+		Help:      "counter for the number of times we start up captive core with a new buckets db, sliding window = 10m",
+	})
+
+	registry.MustRegister(
+		coreSynced,
+		supportedProtocolVersion,
+		latestLedger,
+		c.captiveCoreStartDuration,
+		c.captiveCoreNewDBCounter,
+	)
 }
 
 func (c *CaptiveStellarCore) getLatestCheckpointSequence() (uint32, error) {
@@ -521,19 +543,27 @@ func (c *CaptiveStellarCore) startPreparingRange(ctx context.Context, ledgerRang
 // Please note that using a BoundedRange, currently, requires a full-trust on
 // history archive. This issue is being fixed in Stellar-Core.
 func (c *CaptiveStellarCore) PrepareRange(ctx context.Context, ledgerRange Range) error {
+	startTime := time.Now()
 	if alreadyPrepared, err := c.startPreparingRange(ctx, ledgerRange); err != nil {
 		return errors.Wrap(err, "error starting prepare range")
 	} else if alreadyPrepared {
 		return nil
 	}
 
+	var reportedStartTime bool
 	// the prepared range might be below ledgerRange.from so we
 	// need to seek ahead until we reach ledgerRange.from
 	for seq := c.prepared.from; seq <= ledgerRange.from; seq++ {
 		_, err := c.GetLedger(ctx, seq)
 		if err != nil {
 			return errors.Wrapf(err, "Error fast-forwarding to %d", ledgerRange.from)
 		}
+		if !reportedStartTime {
+			reportedStartTime = true
+			if c.captiveCoreStartDuration != nil && !ledgerRange.bounded {
+				c.captiveCoreStartDuration.Observe(time.Since(startTime).Seconds())
+			}
+		}
 	}
 
 	return nil