From af319d455dbd57f8e97a5102efae01a728804ce5 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 9 Sep 2024 13:00:40 -0700 Subject: [PATCH 1/4] Add endLedger in GetEventResponse --- CHANGELOG.md | 2 ++ .../internal/methods/get_events.go | 3 +++ .../internal/methods/get_events_test.go | 26 +++++++++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5aae50c..d665be73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Added +- Add `EndLedger` in `GetEventsResponse`. This tells the client until what ledger events are being queried. e.g.: `startLEdger` (inclusive) - `endLedger` (exclusive) +- Limitation: getEvents are capped by 10K `LedgerScanLimit` which means you can query events for 10K ledger at maximum for a given request. - Add `EndLedger` in `GetEventsRequest`. This provides finer control and clarity on the range of ledgers being queried. - Disk-Based Event Storage: Events are now stored on disk instead of in memory. For context, storing approximately 3 million events will require around 1.5 GB of disk space. This change enhances the scalability and can now support a larger retention window (~7 days) for events. diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 36aefcdb..ef82b869 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -334,6 +334,8 @@ type PaginationOptions struct { type GetEventsResponse struct { Events []EventInfo `json:"events"` LatestLedger uint32 `json:"latestLedger"` + // All the events until the endLedger (exclusive) is reached + EndLedger uint32 `json:"endLedger"` } type eventsRPCHandler struct { @@ -510,6 +512,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques return GetEventsResponse{ LatestLedger: ledgerRange.LastLedger.Sequence, Events: results, + EndLedger: endLedger, }, nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 44692304..930dc953 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -655,7 +655,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + endLedger := uint32(1 + LedgerScanLimit) + assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) }) t.Run("filtering by contract id", func(t *testing.T) { @@ -801,7 +802,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(4).HexString(), }, } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + endLedger := uint32(1 + LedgerScanLimit) + + assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) results, err = handler.getEvents(ctx, GetEventsRequest{ StartLedger: 1, @@ -835,7 +838,7 @@ func TestGetEvents(t *testing.T) { expected[0].ValueJSON = valueJs expected[0].TopicJSON = topicsJs - require.Equal(t, GetEventsResponse{expected, 1}, results) + require.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) }) t.Run("filtering by both contract id and topic", func(t *testing.T) { @@ -946,7 +949,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(3).HexString(), }, } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + endLedger := uint32(1 + LedgerScanLimit) + + assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) }) t.Run("filtering by event type", func(t *testing.T) { @@ -1021,7 +1026,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(0).HexString(), }, } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + endLedger := uint32(1 + LedgerScanLimit) + assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) }) t.Run("with limit", func(t *testing.T) { @@ -1092,7 +1098,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - assert.Equal(t, GetEventsResponse{expected, 1}, results) + endLedger := uint32(1 + LedgerScanLimit) + + assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) }) t.Run("with cursor", func(t *testing.T) { @@ -1192,7 +1200,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - assert.Equal(t, GetEventsResponse{expected, 5}, results) + endLedger := uint32(LedgerScanLimit) + assert.Equal(t, GetEventsResponse{expected, 5, endLedger}, results) results, err = handler.getEvents(context.TODO(), GetEventsRequest{ Pagination: &PaginationOptions{ @@ -1201,7 +1210,8 @@ func TestGetEvents(t *testing.T) { }, }) assert.NoError(t, err) - assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5}, results) + + assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5, endLedger}, results) }) } From 880ecfa30167afe0a69fe17c4cce13d2dad367fa Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 11 Sep 2024 12:33:11 -0700 Subject: [PATCH 2/4] Add Cursor to response and deprecate PagingToken --- .../internal/methods/get_events.go | 35 ++++++++++++++----- .../internal/methods/get_events_test.go | 35 +++++++++++-------- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index ef82b869..8cf445a8 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math" "strings" "time" @@ -85,11 +86,13 @@ func (e eventTypeSet) matches(event xdr.ContractEvent) bool { } type EventInfo struct { - EventType string `json:"type"` - Ledger int32 `json:"ledger"` - LedgerClosedAt string `json:"ledgerClosedAt"` - ContractID string `json:"contractId"` - ID string `json:"id"` + EventType string `json:"type"` + Ledger int32 `json:"ledger"` + LedgerClosedAt string `json:"ledgerClosedAt"` + ContractID string `json:"contractId"` + ID string `json:"id"` + + // Deprecated: PagingToken field is deprecated, please use Cursor at top level for pagination PagingToken string `json:"pagingToken"` InSuccessfulContractCall bool `json:"inSuccessfulContractCall"` TransactionHash string `json:"txHash"` @@ -334,8 +337,8 @@ type PaginationOptions struct { type GetEventsResponse struct { Events []EventInfo `json:"events"` LatestLedger uint32 `json:"latestLedger"` - // All the events until the endLedger (exclusive) is reached - EndLedger uint32 `json:"endLedger"` + // Cursor represent last populated event ID or end of the search window if no events are found + Cursor string `json:"cursor"` } type eventsRPCHandler struct { @@ -439,7 +442,10 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques limit = request.Pagination.Limit } } - endLedger := request.StartLedger + LedgerScanLimit + endLedger := start.Ledger + LedgerScanLimit + + // endLedger should not exceed ledger retention window + endLedger = min(ledgerRange.LastLedger.Sequence+1, endLedger) if request.EndLedger != 0 { endLedger = min(request.EndLedger, endLedger) @@ -509,10 +515,21 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques results = append(results, info) } + var cursor string + if len(results) > 0 { + lastEvent := results[len(results)-1] + cursor = lastEvent.ID + } else { + // cursor represents end of the search window if no events are found + // here endLedger is always exclusive when fetching events + // so search window is max Cursor value with endLedger - 1 + cursor = db.Cursor{Ledger: endLedger - 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + } + return GetEventsResponse{ LatestLedger: ledgerRange.LastLedger.Sequence, Events: results, - EndLedger: endLedger, + Cursor: cursor, }, nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 930dc953..efef6381 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math" "path" "strconv" "strings" @@ -655,8 +656,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - endLedger := uint32(1 + LedgerScanLimit) - assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) + cursor := expected[len(expected)-1].ID + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("filtering by contract id", func(t *testing.T) { @@ -802,9 +803,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(4).HexString(), }, } - endLedger := uint32(1 + LedgerScanLimit) + cursor := expected[len(expected)-1].ID - assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) results, err = handler.getEvents(ctx, GetEventsRequest{ StartLedger: 1, @@ -838,7 +839,7 @@ func TestGetEvents(t *testing.T) { expected[0].ValueJSON = valueJs expected[0].TopicJSON = topicsJs - require.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) + require.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("filtering by both contract id and topic", func(t *testing.T) { @@ -949,9 +950,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(3).HexString(), }, } - endLedger := uint32(1 + LedgerScanLimit) + cursor := expected[len(expected)-1].ID - assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("filtering by event type", func(t *testing.T) { @@ -1026,8 +1027,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(0).HexString(), }, } - endLedger := uint32(1 + LedgerScanLimit) - assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) + cursor := expected[len(expected)-1].ID + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("with limit", func(t *testing.T) { @@ -1098,9 +1099,9 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - endLedger := uint32(1 + LedgerScanLimit) + cursor := expected[len(expected)-1].ID - assert.Equal(t, GetEventsResponse{expected, 1, endLedger}, results) + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) t.Run("with cursor", func(t *testing.T) { @@ -1200,8 +1201,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - endLedger := uint32(LedgerScanLimit) - assert.Equal(t, GetEventsResponse{expected, 5, endLedger}, results) + cursor := expected[len(expected)-1].ID + assert.Equal(t, GetEventsResponse{expected, 5, cursor}, results) results, err = handler.getEvents(context.TODO(), GetEventsRequest{ Pagination: &PaginationOptions{ @@ -1211,7 +1212,13 @@ func TestGetEvents(t *testing.T) { }) assert.NoError(t, err) - assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5, endLedger}, results) + latestLedger := 5 + endLedger := min(5+LedgerScanLimit, latestLedger+1) + + // Note: endLedger is always exclusive when fetching events + // so search window is always max Cursor value with endLedger - 1 + cursor = db.Cursor{Ledger: uint32(endLedger - 1), Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + assert.Equal(t, GetEventsResponse{[]EventInfo{}, 5, cursor}, results) }) } From 31d4fca2e4f87eda757f71dd9e6005b4ac297b39 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 18 Sep 2024 12:47:27 -0700 Subject: [PATCH 3/4] Update cursor logic and tests --- cmd/soroban-rpc/internal/methods/get_events.go | 4 ++-- cmd/soroban-rpc/internal/methods/get_events_test.go | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index a109d6b0..19b6e711 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -518,11 +518,11 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } var cursor string - if len(results) > 0 { + if uint(len(results)) == limit { lastEvent := results[len(results)-1] cursor = lastEvent.ID } else { - // cursor represents end of the search window if no events are found + // cursor represents end of the search window if events does not reach limit // here endLedger is always exclusive when fetching events // so search window is max Cursor value with endLedger - 1 cursor = db.Cursor{Ledger: endLedger - 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index e125eb37..cd06b0e3 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -656,7 +656,7 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(i).HexString(), }) } - cursor := expected[len(expected)-1].ID + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) @@ -803,7 +803,7 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(4).HexString(), }, } - cursor := expected[len(expected)-1].ID + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) @@ -950,7 +950,7 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(3).HexString(), }, } - cursor := expected[len(expected)-1].ID + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) @@ -1027,7 +1027,8 @@ func TestGetEvents(t *testing.T) { TransactionHash: ledgerCloseMeta.TransactionHash(0).HexString(), }, } - cursor := expected[len(expected)-1].ID + cursor := db.Cursor{Ledger: 1, Tx: math.MaxUint32, Event: math.MaxUint32 - 1}.String() + assert.Equal(t, GetEventsResponse{expected, 1, cursor}, results) }) From 236ff7662c63b03bc3154a55fb2e73906b6e7ca5 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 19 Sep 2024 15:41:53 -0700 Subject: [PATCH 4/4] update CHANGELOG.md --- CHANGELOG.md | 2 +- cmd/soroban-rpc/internal/methods/get_events.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3da90c94..02e11dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ ### Added -- Add `EndLedger` in `GetEventsResponse`. This tells the client until what ledger events are being queried. e.g.: `startLEdger` (inclusive) - `endLedger` (exclusive) +- Add `Cursor` in `GetEventsResponse`. This tells the client until what ledger events are being queried. e.g.: `startLEdger` (inclusive) - `endLedger` (exclusive) - Limitation: getEvents are capped by 10K `LedgerScanLimit` which means you can query events for 10K ledger at maximum for a given request. - Add `EndLedger` in `GetEventsRequest`. This provides finer control and clarity on the range of ledgers being queried. - Disk-Based Event Storage: Events are now stored on disk instead of in memory. For context, storing approximately 3 million events will require around 1.5 GB of disk space. diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 19b6e711..8312b530 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -339,7 +339,7 @@ type PaginationOptions struct { type GetEventsResponse struct { Events []EventInfo `json:"events"` LatestLedger uint32 `json:"latestLedger"` - // Cursor represent last populated event ID or end of the search window if no events are found + // Cursor represents last populated event ID if total events reach the limit or end of the search window Cursor string `json:"cursor"` }