Merge branch 'master' into deprecate-cli-flags

.github/workflows/horizon.yml

@@ -35,9 +35,9 @@ jobs:
       HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }}
       PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 19.13.1-1481.3acf6dd26.focal
       PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:19.13.1-1481.3acf6dd26.focal
-      PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:20.0.0-rc1-35
-      PROTOCOL_19_CORE_DEBIAN_PKG_VERSION: 19.12.0-1378.2109a168a.focal
-      PROTOCOL_19_CORE_DOCKER_IMG: stellar/stellar-core:19.12.0-1378.2109a168a.focal
+      PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:20.0.0-rc3-39
+      PROTOCOL_19_CORE_DEBIAN_PKG_VERSION: 19.14.0-1500.5664eff4e.focal
+      PROTOCOL_19_CORE_DOCKER_IMG: stellar/stellar-core:19.14.0-1500.5664eff4e.focal
       PGHOST: localhost
       PGPORT: 5432
       PGUSER: postgres
@@ -133,7 +133,7 @@ jobs:
     name: Test (and push) verify-range image
     runs-on: ubuntu-22.04
     env:
-      STELLAR_CORE_VERSION: 19.12.0-1378.2109a168a.focal
+      STELLAR_CORE_VERSION: 19.14.0-1500.5664eff4e.focal
       CAPTIVE_CORE_STORAGE_PATH: /tmp
     steps:
       - uses: actions/checkout@v3

Makefile

@@ -14,7 +14,7 @@ xdr/Stellar-contract.x \
 xdr/Stellar-internal.x \
 xdr/Stellar-contract-config-setting.x
 
-XDRGEN_COMMIT=80e38ef2a96489f6b501d4db3a350406e5aa3bab
+XDRGEN_COMMIT=a231a92475ac6154c0c2f46dc503809823985060
 XDR_COMMIT=9ac02641139e6717924fdad716f6e958d0168491
 
 .PHONY: xdr xdr-clean xdr-update

exp/tools/dump-ledger-state/Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:20.04
 
-ENV STELLAR_CORE_VERSION=19.12.0-1378.2109a168a.focal
+ENV STELLAR_CORE_VERSION=19.14.0-1500.5664eff4e.focal
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl wget gnupg apt-utils

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.3
 	github.com/spf13/viper v1.3.2
-	github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee
+	github.com/stellar/go-xdr v0.0.0-20230919160922-6c7b68458206
 	github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible
 	github.com/stretchr/testify v1.8.1
 	github.com/tyler-smith/go-bip39 v0.0.0-20180618194314-52158e4697b8

go.sum

@@ -325,8 +325,8 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee h1:fbVs0xmXpBvVS4GBeiRmAE3Le70ofAqFMch1GTiq/e8=
-github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps=
+github.com/stellar/go-xdr v0.0.0-20230919160922-6c7b68458206 h1:UFuvvpbWL8+jqO1QmKYWSVhiMp4MRiIFd8/zQlUINH0=
+github.com/stellar/go-xdr v0.0.0-20230919160922-6c7b68458206/go.mod h1:yoxyU/M8nl9LKeWIoBrbDPQ7Cy+4jxRcWcOayZ4BMps=
 github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible h1:jMXXAcz6xTarGDQ4VtVbtERogcmDQw4RaE85Cr9CgoQ=
 github.com/stellar/throttled v2.2.3-0.20190823235211-89d75816f59d+incompatible/go.mod h1:7CJ23pXirXBJq45DqvO6clzTEGM/l1SfKrgrzLry8b4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

ingest/ledgerbackend/buffered_meta_pipe_reader.go

@@ -97,7 +97,7 @@ func (b *bufferedLedgerMetaReader) readLedgerMetaFromPipe() (*xdr.LedgerCloseMet
 	}
 
 	var xlcm xdr.LedgerCloseMeta
-	_, err = xlcm.DecodeFrom(b.decoder)
+	_, err = xlcm.DecodeFrom(b.decoder, xdr3.DecodeDefaultMaxDepth)
 	if err != nil {
 		return nil, errors.Wrap(err, "unmarshaling framed LedgerCloseMeta")
 	}

ingest/ledgerbackend/toml.go

@@ -70,12 +70,11 @@ type captiveCoreTomlValues struct {
 	BucketDirPath string `toml:"BUCKET_DIR_PATH,omitempty"`
 	// we cannot omitempty because 0 is a valid configuration for HTTP_PORT
 	// and the default is 11626
-	HTTPPort                  uint     `toml:"HTTP_PORT"`
-	PublicHTTPPort            bool     `toml:"PUBLIC_HTTP_PORT,omitempty"`
-	NodeNames                 []string `toml:"NODE_NAMES,omitempty"`
-	NetworkPassphrase         string   `toml:"NETWORK_PASSPHRASE,omitempty"`
-	PeerPort                  uint     `toml:"PEER_PORT,omitempty"`
-	LimitTxQueueSourceAccount bool     `toml:"LIMIT_TX_QUEUE_SOURCE_ACCOUNT,omitempty"`
+	HTTPPort          uint     `toml:"HTTP_PORT"`
+	PublicHTTPPort    bool     `toml:"PUBLIC_HTTP_PORT,omitempty"`
+	NodeNames         []string `toml:"NODE_NAMES,omitempty"`
+	NetworkPassphrase string   `toml:"NETWORK_PASSPHRASE,omitempty"`
+	PeerPort          uint     `toml:"PEER_PORT,omitempty"`
 	// we cannot omitempty because 0 is a valid configuration for FAILURE_SAFETY
 	// and the default is -1
 	FailureSafety                         int                  `toml:"FAILURE_SAFETY"`

services/horizon/docker/verify-range/start

@@ -50,7 +50,8 @@ dump_horizon_db() {
 	echo "dumping history_ledgers"
 	psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -t -A -F"," --variable="FETCH_COUNT=100" -c "select sequence, ledger_hash, previous_ledger_hash, transaction_count, operation_count, closed_at, id, total_coins, fee_pool, base_fee, base_reserve, max_tx_set_size, protocol_version, ledger_header, successful_transaction_count, failed_transaction_count from history_ledgers order by sequence asc" > "${1}_ledgers"
 	echo "dumping history_operations"
-	psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -t -A -F"," --variable="FETCH_COUNT=100" -c "select * from history_operations order by id asc" > "${1}_operations"
+	# skip is_payment column which was only introduced in the most recent horizon v2.27.0
+	psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -t -A -F"," --variable="FETCH_COUNT=100" -c "select id, transaction_id, application_order, type, details, source_account, source_account_muxed from history_operations order by id asc" > "${1}_operations"
 	echo "dumping history_operation_claimable_balances"
 	psql "postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -t -A -F"," --variable="FETCH_COUNT=100" -c "select history_operation_id, history_claimable_balance_id from history_operation_claimable_balances left join history_claimable_balances on history_claimable_balances.id = history_operation_claimable_balances.history_claimable_balance_id order by history_operation_id asc, id asc" > "${1}_operation_claimable_balances"
 	echo "dumping history_operation_participants"

services/horizon/internal/db2/history/claimable_balances.go

@@ -247,21 +247,30 @@ func (q *Q) GetClaimableBalances(ctx context.Context, query ClaimableBalancesQue
 		return nil, errors.Wrap(err, "could not apply query to page")
 	}
 
-	if query.Asset != nil {
-		// when search by asset, profiling has shown best performance to have the LIMIT on inner query
-		sql = sql.Where("cb.asset = ?", query.Asset)
-	}
+	if query.Asset != nil || query.Sponsor != nil {
 
-	if query.Sponsor != nil {
-		sql = sql.Where("cb.sponsor = ?", query.Sponsor.Address())
-	}
+		// JOIN with claimable_balance_claimants table to query by claimants
+		if query.Claimant != nil {
+			sql = sql.Join("claimable_balance_claimants on claimable_balance_claimants.id = cb.id")
+			sql = sql.Where("claimable_balance_claimants.destination = ?", query.Claimant.Address())
+		}
+
+		// Apply filters for asset and sponsor
+		if query.Asset != nil {
+			sql = sql.Where("cb.asset = ?", query.Asset)
+		}
+		if query.Sponsor != nil {
+			sql = sql.Where("cb.sponsor = ?", query.Sponsor.Address())
+		}
+
+	} else if query.Claimant != nil {
+		// If only the claimant is provided without additional filters, a JOIN with claimable_balance_claimants
+		// does not perform efficiently. Instead, use a subquery (with LIMIT) to retrieve claimable balances based on
+		// the claimant's address.
 
-	if query.Claimant != nil {
 		var selectClaimableBalanceClaimants = sq.Select("id").From("claimable_balance_claimants").
-			Where("destination = ?", query.Claimant.Address()).
-			// Given that each destination can be a claimant for each balance maximum once
-			// we can LIMIT the subquery.
-			Limit(query.PageQuery.Limit)
+			Where("destination = ?", query.Claimant.Address()).Limit(query.PageQuery.Limit)
+
 		subSql, err := applyClaimableBalancesQueriesCursor(selectClaimableBalanceClaimants, l, r, query.PageQuery.Order)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not apply subquery to page")

services/horizon/internal/db2/history/claimable_balances_test.go

@@ -233,6 +233,187 @@ func TestFindClaimableBalancesByDestination(t *testing.T) {
 	tt.Assert.Len(cbs, 1)
 }
 
+func insertClaimants(q *Q, tt *test.T, cBalance ClaimableBalance) error {
+	claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(10)
+	for _, claimant := range cBalance.Claimants {
+		claimant := ClaimableBalanceClaimant{
+			BalanceID:          cBalance.BalanceID,
+			Destination:        claimant.Destination,
+			LastModifiedLedger: cBalance.LastModifiedLedger,
+		}
+		err := claimantsInsertBuilder.Add(tt.Ctx, claimant)
+		if err != nil {
+			return err
+		}
+	}
+	return claimantsInsertBuilder.Exec(tt.Ctx)
+}
+
+type claimableBalanceQueryResult struct {
+	Claimants []string
+	Asset     string
+	Sponsor   string
+}
+
+func validateClaimableBalanceQuery(t *test.T, q *Q, query ClaimableBalancesQuery, expectedQueryResult []claimableBalanceQueryResult) {
+	cbs, err := q.GetClaimableBalances(t.Ctx, query)
+	t.Assert.NoError(err)
+	for i, expected := range expectedQueryResult {
+		for j, claimant := range expected.Claimants {
+			t.Assert.Equal(claimant, cbs[i].Claimants[j].Destination)
+		}
+		if expected.Asset != "" {
+			t.Assert.Equal(expected.Asset, cbs[i].Asset.String())
+		}
+		if expected.Sponsor != "" {
+			t.Assert.Equal(expected.Sponsor, cbs[i].Sponsor.String)
+		}
+	}
+}
+
+// TestFindClaimableBalancesByDestinationWithLimit tests querying claimable balances by destination and asset
+func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+
+	test.ResetHorizonDB(t, tt.HorizonDB)
+	q := &Q{tt.HorizonSession()}
+
+	assetIssuer := "GA25GQLHJU3LPEJXEIAXK23AWEA5GWDUGRSHTQHDFT6HXHVMRULSQJUJ"
+	asset1 := xdr.MustNewCreditAsset("ASSET1", assetIssuer)
+	asset2 := xdr.MustNewCreditAsset("ASSET2", assetIssuer)
+
+	sponsor1 := "GA25GQLHJU3LPEJXEIAXK23AWEA5GWDUGRSHTQHDFT6HXHVMRULSQJUJ"
+	sponsor2 := "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H"
+
+	dest1 := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"
+	dest2 := "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H"
+
+	claimants := []Claimant{
+		{
+			Destination: dest1,
+			Predicate: xdr.ClaimPredicate{
+				Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional,
+			},
+		},
+		{
+			Destination: dest2,
+			Predicate: xdr.ClaimPredicate{
+				Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional,
+			},
+		},
+	}
+
+	balanceID1 := xdr.ClaimableBalanceId{
+		Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0,
+		V0:   &xdr.Hash{1, 2, 3},
+	}
+	id, err := xdr.MarshalHex(balanceID1)
+	tt.Assert.NoError(err)
+	cBalance1 := ClaimableBalance{
+		BalanceID:          id,
+		Claimants:          claimants,
+		Asset:              asset1,
+		Sponsor:            null.StringFrom(sponsor1),
+		LastModifiedLedger: 123,
+		Amount:             10,
+	}
+	err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance1})
+	tt.Assert.NoError(err)
+
+	claimants2 := []Claimant{
+		{
+			Destination: dest2,
+			Predicate: xdr.ClaimPredicate{
+				Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional,
+			},
+		},
+	}
+
+	balanceID2 := xdr.ClaimableBalanceId{
+		Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0,
+		V0:   &xdr.Hash{4, 5, 6},
+	}
+	id, err = xdr.MarshalHex(balanceID2)
+	tt.Assert.NoError(err)
+	cBalance2 := ClaimableBalance{
+		BalanceID: id,
+		Claimants: claimants2,
+		Asset:     asset2,
+		Sponsor:   null.StringFrom(sponsor2),
+
+		LastModifiedLedger: 456,
+		Amount:             10,
+	}
+	err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance2})
+	tt.Assert.NoError(err)
+
+	err = insertClaimants(q, tt, cBalance1)
+	tt.Assert.NoError(err)
+
+	err = insertClaimants(q, tt, cBalance2)
+	tt.Assert.NoError(err)
+
+	pageQuery := db2.MustPageQuery("", false, "", 1)
+
+	// no claimant parameter, no filters
+	query := ClaimableBalancesQuery{
+		PageQuery: pageQuery,
+	}
+	validateClaimableBalanceQuery(tt, q, query, []claimableBalanceQueryResult{
+		{Claimants: []string{dest1, dest2}},
+	})
+
+	// invalid claimant parameter
+	query = ClaimableBalancesQuery{
+		PageQuery: pageQuery,
+		Claimant:  xdr.MustAddressPtr("GA25GQLHJU3LPEJXEIAXK23AWEA5GWDUGRSHTQHDFT6HXHVMRULSQJUJ"),
+		Asset:     &asset2,
+		Sponsor:   xdr.MustAddressPtr(sponsor1),
+	}
+	validateClaimableBalanceQuery(tt, q, query, []claimableBalanceQueryResult{})
+
+	// claimant parameter, no filters
+	query = ClaimableBalancesQuery{
+		PageQuery: pageQuery,
+		Claimant:  xdr.MustAddressPtr(dest1),
+	}
+	validateClaimableBalanceQuery(tt, q, query, []claimableBalanceQueryResult{
+		{Claimants: []string{dest1, dest2}},
+	})
+
+	// claimant parameter, asset filter
+	query = ClaimableBalancesQuery{
+		PageQuery: pageQuery,
+		Claimant:  xdr.MustAddressPtr(dest2),
+		Asset:     &asset1,
+	}
+	validateClaimableBalanceQuery(tt, q, query, []claimableBalanceQueryResult{
+		{Claimants: []string{dest1, dest2}, Asset: asset1.String()},
+	})
+
+	// claimant parameter, sponsor filter
+	query = ClaimableBalancesQuery{
+		PageQuery: pageQuery,
+		Claimant:  xdr.MustAddressPtr(dest2),
+		Sponsor:   xdr.MustAddressPtr(sponsor1),
+	}
+	validateClaimableBalanceQuery(tt, q, query, []claimableBalanceQueryResult{
+		{Claimants: []string{dest1, dest2}, Sponsor: sponsor1},
+	})
+
+	//claimant parameter, asset filter, sponsor filter
+	query = ClaimableBalancesQuery{
+		PageQuery: pageQuery,
+		Claimant:  xdr.MustAddressPtr(dest2),
+		Asset:     &asset2,
+		Sponsor:   xdr.MustAddressPtr(sponsor2),
+	}
+	validateClaimableBalanceQuery(tt, q, query, []claimableBalanceQueryResult{
+		{Claimants: []string{dest2}, Asset: asset2.String(), Sponsor: sponsor2},
+	})
+}
+
 func TestUpdateClaimableBalance(t *testing.T) {
 	tt := test.Start(t)
 	defer tt.Finish()

services/horizon/internal/httpx/server.go

@@ -15,7 +15,6 @@ import (
 	"github.com/stellar/go/services/horizon/internal/ledger"
 	hProblem "github.com/stellar/go/services/horizon/internal/render/problem"
 	"github.com/stellar/go/services/horizon/internal/render/sse"
-	"github.com/stellar/go/services/horizon/internal/txsub/sequence"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/log"
 	"github.com/stellar/go/support/render/problem"
@@ -49,7 +48,6 @@ func init() {
 	// register problems
 	problem.SetLogFilter(problem.LogUnknownErrors)
 	problem.RegisterError(sql.ErrNoRows, problem.NotFound)
-	problem.RegisterError(sequence.ErrNoMoreRoom, hProblem.ServerOverCapacity)
 	problem.RegisterError(db2.ErrInvalidCursor, problem.BadRequest)
 	problem.RegisterError(db2.ErrInvalidLimit, problem.BadRequest)
 	problem.RegisterError(db2.ErrInvalidOrder, problem.BadRequest)

services/horizon/internal/init.go

@@ -14,7 +14,6 @@ import (
 	"github.com/stellar/go/services/horizon/internal/paths"
 	"github.com/stellar/go/services/horizon/internal/simplepath"
 	"github.com/stellar/go/services/horizon/internal/txsub"
-	"github.com/stellar/go/services/horizon/internal/txsub/sequence"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/log"
 )
@@ -239,9 +238,8 @@ func initWebMetrics(app *App) {
 
 func initSubmissionSystem(app *App) {
 	app.submitter = &txsub.System{
-		Pending:         txsub.NewDefaultSubmissionList(),
-		Submitter:       txsub.NewDefaultSubmitter(http.DefaultClient, app.config.StellarCoreURL),
-		SubmissionQueue: sequence.NewManager(),
+		Pending:   txsub.NewDefaultSubmissionList(),
+		Submitter: txsub.NewDefaultSubmitter(http.DefaultClient, app.config.StellarCoreURL),
 		DB: func(ctx context.Context) txsub.HorizonDB {
 			return &history.Q{SessionInterface: app.HorizonSession()}
 		},