diff --git a/CHANGELOG.md b/CHANGELOG.md index 9292db39..02e11dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### Added +- Add `Cursor` in `GetEventsResponse`. This tells the client until what ledger events are being queried. e.g.: `startLEdger` (inclusive) - `endLedger` (exclusive) +- Limitation: getEvents are capped by 10K `LedgerScanLimit` which means you can query events for 10K ledger at maximum for a given request. - Add `EndLedger` in `GetEventsRequest`. This provides finer control and clarity on the range of ledgers being queried. - Disk-Based Event Storage: Events are now stored on disk instead of in memory. For context, storing approximately 3 million events will require around 1.5 GB of disk space. This change enhances the scalability and can now support a larger retention window (~7 days) for events. diff --git a/Makefile b/Makefile index 59e4f671..632e6165 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,10 @@ endif # (libpreflight.a is put at target/release-with-panic-unwind/ when not cross compiling) CARGO_BUILD_TARGET ?= $(shell rustc -vV | sed -n 's|host: ||p') +SOROBAN_RPC_BINARY := soroban-rpc +STELLAR_RPC_BINARY := stellar-rpc + + # update the Cargo.lock every time the Cargo.toml changes. Cargo.lock: Cargo.toml cargo update --workspace @@ -77,11 +81,19 @@ clean: cargo clean go clean ./... +# DEPRECATED - please use build-stellar-rpc instead # the build-soroban-rpc build target is an optimized build target used by -# https://github.com/stellar/pipelines/stellar-horizon/Jenkinsfile-soroban-rpc-package-builder +# https://github.com/stellar/pipelines/blob/master/soroban-rpc/Jenkinsfile-soroban-rpc-package-builder # as part of the package building. build-soroban-rpc: build-libs - go build -ldflags="${GOLDFLAGS}" ${MACOS_MIN_VER} -o soroban-rpc -trimpath -v ./cmd/soroban-rpc + go build -ldflags="${GOLDFLAGS}" ${MACOS_MIN_VER} -o ${SOROBAN_RPC_BINARY} -trimpath -v ./cmd/soroban-rpc + +# the build-stellar-rpc build target is an optimized build target used by +# https://github.com/stellar/pipelines/blob/master/soroban-rpc/Jenkinsfile-soroban-rpc-package-builder +# as part of the package building. +build-stellar-rpc: build-libs + go build -ldflags="${GOLDFLAGS}" ${MACOS_MIN_VER} -o ${STELLAR_RPC_BINARY} -trimpath -v ./cmd/soroban-rpc + go-check-branch: golangci-lint run ./... --new-from-rev $$(git rev-parse origin/main) diff --git a/cmd/soroban-rpc/docker/Dockerfile b/cmd/soroban-rpc/docker/Dockerfile index cbe03a87..a9608256 100644 --- a/cmd/soroban-rpc/docker/Dockerfile +++ b/cmd/soroban-rpc/docker/Dockerfile @@ -1,6 +1,7 @@ FROM golang:1.22-bullseye as build ARG RUST_TOOLCHAIN_VERSION=stable ARG REPOSITORY_VERSION +ARG BINARY_NAME=soroban-rpc WORKDIR /go/src/github.com/stellar/soroban-rpc @@ -18,11 +19,14 @@ RUN apt-get clean RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain $RUST_TOOLCHAIN_VERSION -RUN make REPOSITORY_VERSION=${REPOSITORY_VERSION} build-soroban-rpc -RUN mv soroban-rpc /bin/soroban-rpc +RUN make REPOSITORY_VERSION=${REPOSITORY_VERSION} build-${BINARY_NAME} + +# Move the binary to a common location +RUN mv ${BINARY_NAME} /bin/${BINARY_NAME} FROM ubuntu:22.04 ARG STELLAR_CORE_VERSION +ARG BINARY_NAME=soroban-rpc ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*} ENV STELLAR_CORE_BINARY_PATH /usr/bin/stellar-core ENV DEBIAN_FRONTEND=noninteractive @@ -35,5 +39,8 @@ RUN echo "deb https://apt.stellar.org focal unstable" >/etc/apt/sources.list.d/S RUN apt-get update && apt-get install -y stellar-core=${STELLAR_CORE_VERSION} RUN apt-get clean -COPY --from=build /bin/soroban-rpc /app/ -ENTRYPOINT ["/app/soroban-rpc"] +# Copy the binary from the build stage +COPY --from=build /bin/${BINARY_NAME} /app/${BINARY_NAME} + +# Set the entrypoint to the specific binary +ENTRYPOINT ["/app/${BINARY_NAME}"] \ No newline at end of file diff --git a/cmd/soroban-rpc/docker/Makefile b/cmd/soroban-rpc/docker/Makefile index f3f39994..1d02a5cd 100644 --- a/cmd/soroban-rpc/docker/Makefile +++ b/cmd/soroban-rpc/docker/Makefile @@ -22,12 +22,22 @@ ifndef STELLAR_CORE_VERSION $(error STELLAR_CORE_VERSION environment variable must be set. For example 19.10.1-1310.6649f5173.focal~soroban) endif -TAG ?= stellar/stellar-soroban-rpc:$(SOROBAN_RPC_VERSION_PACKAGE_VERSION) +# Set default value for BINARY_NAME if not provided +BINARY_NAME ?= soroban-rpc + +# Set the TAG based on the value of BINARY_NAME +ifeq ($(BINARY_NAME),stellar-rpc) + TAG := stellar/stellar-rpc:$(SOROBAN_RPC_VERSION_PACKAGE_VERSION) +else + TAG := stellar/stellar-soroban-rpc:$(SOROBAN_RPC_VERSION_PACKAGE_VERSION) +endif + docker-build: $(SUDO) docker build --pull --platform linux/amd64 $(DOCKER_OPTS) \ --label org.opencontainers.image.created="$(BUILD_DATE)" \ --build-arg STELLAR_CORE_VERSION=$(STELLAR_CORE_VERSION) --build-arg SOROBAN_RPC_VERSION=$(SOROBAN_RPC_VERSION_PACKAGE_VERSION) \ + --build-arg BINARY_NAME=$(BINARY_NAME) \ -t $(TAG) -f Dockerfile.release . docker-push: diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index af836455..ef2d5cbf 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -25,6 +25,7 @@ const ( var ErrNoTransaction = errors.New("no transaction with this hash exists") type Transaction struct { + TransactionHash string Result []byte // XDR encoded xdr.TransactionResult Meta []byte // XDR encoded xdr.TransactionMeta Envelope []byte // XDR encoded xdr.TransactionEnvelope @@ -223,6 +224,7 @@ func ParseTransaction(lcm xdr.LedgerCloseMeta, ingestTx ingest.LedgerTransaction Sequence: lcm.LedgerSequence(), CloseTime: lcm.LedgerCloseTime(), } + tx.TransactionHash = ingestTx.Result.TransactionHash.HexString() if tx.Result, err = ingestTx.Result.Result.MarshalBinary(); err != nil { return tx, fmt.Errorf("couldn't encode transaction Result: %w", err) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 48290fb7..8312b530 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math" "strings" "time" @@ -85,11 +86,13 @@ func (e eventTypeSet) matches(event xdr.ContractEvent) bool { } type EventInfo struct { - EventType string `json:"type"` - Ledger int32 `json:"ledger"` - LedgerClosedAt string `json:"ledgerClosedAt"` - ContractID string `json:"contractId"` - ID string `json:"id"` + EventType string `json:"type"` + Ledger int32 `json:"ledger"` + LedgerClosedAt string `json:"ledgerClosedAt"` + ContractID string `json:"contractId"` + ID string `json:"id"` + + // Deprecated: PagingToken field is deprecated, please use Cursor at top level for pagination PagingToken string `json:"pagingToken"` InSuccessfulContractCall bool `json:"inSuccessfulContractCall"` TransactionHash string `json:"txHash"` @@ -336,6 +339,8 @@ type PaginationOptions struct { type GetEventsResponse struct { Events []EventInfo `json:"events"` LatestLedger uint32 `json:"latestLedger"` + // Cursor represents last populated event ID if total events reach the limit or end of the search window + Cursor string `json:"cursor"` } type eventsRPCHandler struct { @@ -439,7 +444,10 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques limit = request.Pagination.Limit } } - endLedger := request.StartLedger + LedgerScanLimit + endLedger := start.Ledger + LedgerScanLimit + + // endLedger should not exceed ledger retention window + endLedger = min(ledgerRange.LastLedger.Sequence+1, endLedger) if request.EndLedger != 0 { endLedger = min(request.EndLedger, endLedger) @@ -509,9 +517,21 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques results = append(results, info) } + var cursor string + if uint(len(results)) == limit { + lastEvent := results[len(results)-1] + cursor = lastEvent.ID + } else { + // cursor represents end of the search window if events does not reach limit + // here endLedger is always exclusive when fetching events + // so search window is max Cursor value with endLedger - 1 + cursor = db.Cursor{Ledger: endLedger - 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + } + return GetEventsResponse{ LatestLedger: ledgerRange.LastLedger.Sequence, Events: results, + Cursor: cursor, }, nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index a7373287..cd06b0e3 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math" "path" "strconv" "strings" @@ -655,7 +656,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("filtering by contract id", func(t *testing.T) { @@ -801,7 +803,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(4).HexString(), }, } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) results, err = handler.getEvents(ctx, GetEventsRequest{ StartLedger: 1, @@ -835,7 +839,7 @@ func TestGetEvents(t *testing.T) { expected[0].ValueJSON = valueJs expected[0].TopicJSON = topicsJs - require.Equal(t, GetEventsResponse{expected, 1}, results) + require.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("filtering by both contract id and topic", func(t *testing.T) { @@ -946,7 +950,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(3).HexString(), }, } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("filtering by event type", func(t *testing.T) { @@ -1021,7 +1027,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(0).HexString(), }, } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("with limit", func(t *testing.T) { @@ -1092,7 +1100,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + cursor := expected[len(expected)-1].ID + + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("with cursor", func(t *testing.T) { @@ -1192,7 +1202,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - assert.Equal(t, GetEventsResponse{expected, 5}, results) + cursor := expected[len(expected)-1].ID + assert.Equal(t, GetEventsResponse{expected, 5, cursor}, results) results, err = handler.getEvents(context.TODO(), GetEventsRequest{ Pagination: &PaginationOptions{ @@ -1201,7 +1212,14 @@ func TestGetEvents(t *testing.T) { }, }) require.NoError(t, err) - assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5}, results) + + latestLedger := 5 + endLedger := min(5+LedgerScanLimit, latestLedger+1) + + // Note: endLedger is always exclusive when fetching events + // so search window is always max Cursor value with endLedger - 1 + cursor = db.Cursor{Ledger: uint32(endLedger - 1), Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5, cursor}, results) }) } diff --git a/cmd/soroban-rpc/internal/methods/get_transactions.go b/cmd/soroban-rpc/internal/methods/get_transactions.go index 8c3f10a0..beb0affe 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions.go @@ -58,6 +58,9 @@ func (req GetTransactionsRequest) isValid(maxLimit uint, ledgerRange ledgerbucke type TransactionInfo struct { // Status is one of: TransactionSuccess, TransactionFailed, TransactionNotFound. Status string `json:"status"` + // TransactionHash is the hex encoded hash of the transaction. Note that for fee-bump transaction + // this will be the hash of the fee-bump transaction instead of the inner transaction hash. + TransactionHash string `json:"txHash"` // ApplicationOrder is the index of the transaction among all the transactions // for that ledger. ApplicationOrder int32 `json:"applicationOrder"` @@ -194,6 +197,7 @@ func (h transactionsRPCHandler) processTransactionsInLedger( } txInfo := TransactionInfo{ + TransactionHash: tx.TransactionHash, ApplicationOrder: tx.ApplicationOrder, FeeBump: tx.FeeBump, Ledger: tx.Ledger.Sequence, diff --git a/cmd/soroban-rpc/internal/methods/get_transactions_test.go b/cmd/soroban-rpc/internal/methods/get_transactions_test.go index 5a90202f..24e3f429 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions_test.go @@ -20,6 +20,19 @@ const ( NetworkPassphrase string = "passphrase" ) +var expectedTransactionInfo = TransactionInfo{ + Status: "SUCCESS", + TransactionHash: "b0d0b35dcaed0152d62fbbaa28ed3fa4991c87e7e169a8fca2687b17ee26ca2d", + ApplicationOrder: 1, + FeeBump: false, + Ledger: 1, + LedgerCloseTime: 125, + EnvelopeXDR: "AAAAAgAAAQCAAAAAAAAAAD8MNL+TrQ2ZcdBMzJD3BVEcg4qtlzSkovsNegP8f+iaAAAAAQAAAAD///+dAAAAAAAAAAAAAAAAAAAAAAAAAAA=", //nolint:lll + ResultMetaXDR: "AAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAA", + ResultXDR: "AAAAAAAAAGQAAAAAAAAAAAAAAAA=", + DiagnosticEventsXDR: []string{}, +} + // createTestLedger Creates a test ledger with 2 transactions func createTestLedger(sequence uint32) xdr.LedgerCloseMeta { sequence -= 100 @@ -70,6 +83,9 @@ func TestGetTransactions_DefaultLimit(t *testing.T) { // assert transactions result assert.Len(t, response.Transactions, 10) + + // assert the transaction structure. We will match only 1 tx for sanity purposes. + assert.Equal(t, expectedTransactionInfo, response.Transactions[0]) } func TestGetTransactions_DefaultLimitExceedsLatestLedger(t *testing.T) { @@ -104,6 +120,9 @@ func TestGetTransactions_DefaultLimitExceedsLatestLedger(t *testing.T) { // assert transactions result assert.Len(t, response.Transactions, 6) + + // assert the transaction structure. We will match only 1 tx for sanity purposes. + assert.Equal(t, expectedTransactionInfo, response.Transactions[0]) } func TestGetTransactions_CustomLimit(t *testing.T) { @@ -143,6 +162,9 @@ func TestGetTransactions_CustomLimit(t *testing.T) { assert.Len(t, response.Transactions, 2) assert.Equal(t, uint32(1), response.Transactions[0].Ledger) assert.Equal(t, uint32(1), response.Transactions[1].Ledger) + + // assert the transaction structure. We will match only 1 tx for sanity purposes. + assert.Equal(t, expectedTransactionInfo, response.Transactions[0]) } func TestGetTransactions_CustomLimitAndCursor(t *testing.T) {