Merge branch 'main' into get-transactions

.github/workflows/e2e.yml

@@ -37,7 +37,7 @@ jobs:
       # resolution options, using npm release or a gh ref:
       #
       # option #1, set the version of stellar-sdk based on a npm release version
-      SYSTEM_TEST_JS_STELLAR_SDK_NPM_VERSION: 11.1.0
+      SYSTEM_TEST_JS_STELLAR_SDK_NPM_VERSION: 11.3.0
       # option #2, set the version of stellar-sdk used as a ref to a gh repo if
       # a value is set on SYSTEM_TEST_JS_STELLAR_SDK_GH_REPO, it takes
       # precedence over any SYSTEM_TEST_JS_STELLAR_SDK_NPM_VERSION

.github/workflows/publish.yml

@@ -35,7 +35,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
       VERSION: '${{ github.event.release.name }}'
-      NAME: 'soroban-rpc-${{ github.event.release.name }}-${{ matrix.target }}'
+      NAME: 'stellar-rpc-client-${{ github.event.release.name }}-${{ matrix.target }}'
     steps:
     - uses: actions/checkout@v3
     - run: rustup update
@@ -49,8 +49,8 @@ jobs:
         CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
       run: |
         cd target/package
-        tar xvfz soroban-rpc-$VERSION.crate
-        cd soroban-rpc-$VERSION
+        tar xvfz stellar-rpc-client-$VERSION.crate
+        cd stellar-rpc-client-$VERSION
         cargo build --target-dir=../.. --release --target ${{ matrix.target }}
     - name: Compress 
       run: |

.github/workflows/rust.yml

@@ -109,7 +109,7 @@ jobs:
           cargo-hack-feature-options: --features opt --ignore-unknown-features
     uses: stellar/actions/.github/workflows/rust-publish-dry-run-v2.yml@main
     with:
-      crates: soroban-rpc
+      crates: stellar-rpc-client
       runs-on: ${{ matrix.os }}
       target: ${{ matrix.target }}
       cargo-hack-feature-options: ${{ matrix.cargo-hack-feature-options }}

Cargo.toml

@@ -8,7 +8,7 @@ default-members = ["cmd/crates/stellar-rpc-client"]
 #exclude = ["cmd/crates/soroban-test/tests/fixtures/hello"]
 
 [workspace.package]
-version = "20.3.3"
+version = "20.3.5"
 rust-version = "1.74.0"
 
 [workspace.dependencies.soroban-env-host]
@@ -61,7 +61,7 @@ version = "=20.3.2"
 # rev = "4aef54ff9295c2fca4c5b9fbd2c92d0ff99f67de"
 
 [workspace.dependencies.stellar-rpc-client]
-version = "20.3.3"
+version = "20.3.5"
 path = "cmd/crates/stellar-rpc-client"
 
 [workspace.dependencies.stellar-xdr]

cmd/crates/stellar-rpc-client/src/lib.rs

@@ -1,7 +1,7 @@
 use http::{uri::Authority, Uri};
 use itertools::Itertools;
 use jsonrpsee_core::params::ObjectParams;
-use jsonrpsee_core::{self, client::ClientT, rpc_params};
+use jsonrpsee_core::{self, client::ClientT};
 use jsonrpsee_http_client::{HeaderMap, HttpClient, HttpClientBuilder};
 use serde_aux::prelude::{
     deserialize_default_from_null, deserialize_number_from_string,
@@ -279,6 +279,21 @@ pub struct SimulateHostFunctionResult {
     pub xdr: xdr::ScVal,
 }
 
+#[derive(serde::Deserialize, serde::Serialize, Debug, Clone, PartialEq)]
+#[serde(tag = "type")]
+pub enum LedgerEntryChange {
+    #[serde(rename = "created")]
+    Created { key: String, after: String },
+    #[serde(rename = "deleted")]
+    Deleted { key: String, before: String },
+    #[serde(rename = "updated")]
+    Updated {
+        key: String,
+        before: String,
+        after: String,
+    },
+}
+
 #[derive(serde::Deserialize, serde::Serialize, Debug, Default, Clone)]
 pub struct SimulateTransactionResponse {
     #[serde(
@@ -305,6 +320,12 @@ pub struct SimulateTransactionResponse {
         default
     )]
     pub restore_preamble: Option<RestorePreamble>,
+    #[serde(
+        rename = "stateChanges",
+        skip_serializing_if = "Option::is_none",
+        default
+    )]
+    pub state_changes: Option<Vec<LedgerEntryChange>>,
     #[serde(rename = "latestLedger")]
     pub latest_ledger: u32,
     #[serde(skip_serializing_if = "Option::is_none", default)]
@@ -637,7 +658,10 @@ impl Client {
     /// # Errors
     pub async fn get_network(&self) -> Result<GetNetworkResponse, Error> {
         tracing::trace!("Getting network");
-        Ok(self.client()?.request("getNetwork", rpc_params![]).await?)
+        Ok(self
+            .client()?
+            .request("getNetwork", ObjectParams::new())
+            .await?)
     }
 
     ///
@@ -646,7 +670,7 @@ impl Client {
         tracing::trace!("Getting latest ledger");
         Ok(self
             .client()?
-            .request("getLatestLedger", rpc_params![])
+            .request("getLatestLedger", ObjectParams::new())
             .await?)
     }
 
@@ -691,16 +715,15 @@ soroban config identity fund {address} --helper-url <url>"#
     ) -> Result<GetTransactionResponse, Error> {
         let client = self.client()?;
         tracing::trace!("Sending:\n{tx:#?}");
+        let mut oparams = ObjectParams::new();
+        oparams.insert("transaction", tx.to_xdr_base64(Limits::none())?)?;
         let SendTransactionResponse {
             hash,
             error_result_xdr,
             status,
             ..
         } = client
-            .request(
-                "sendTransaction",
-                rpc_params![tx.to_xdr_base64(Limits::none())?],
-            )
+            .request("sendTransaction", oparams)
             .await
             .map_err(|err| {
                 Error::TransactionSubmissionFailed(format!("No status yet:\n {err:#?}"))
@@ -761,11 +784,11 @@ soroban config identity fund {address} --helper-url <url>"#
     ) -> Result<SimulateTransactionResponse, Error> {
         tracing::trace!("Simulating:\n{tx:#?}");
         let base64_tx = tx.to_xdr_base64(Limits::none())?;
-        let mut builder = ObjectParams::new();
-        builder.insert("transaction", base64_tx)?;
+        let mut oparams = ObjectParams::new();
+        oparams.insert("transaction", base64_tx)?;
         let response: SimulateTransactionResponse = self
             .client()?
-            .request("simulateTransaction", builder)
+            .request("simulateTransaction", oparams)
             .await?;
         tracing::trace!("Simulation response:\n {response:#?}");
         match response.error {
@@ -835,10 +858,9 @@ soroban config identity fund {address} --helper-url <url>"#
     ///
     /// # Errors
     pub async fn get_transaction(&self, tx_id: &str) -> Result<GetTransactionResponseRaw, Error> {
-        Ok(self
-            .client()?
-            .request("getTransaction", rpc_params![tx_id])
-            .await?)
+        let mut oparams = ObjectParams::new();
+        oparams.insert("hash", tx_id)?;
+        Ok(self.client()?.request("getTransaction", oparams).await?)
     }
 
     ///
@@ -855,10 +877,9 @@ soroban config identity fund {address} --helper-url <url>"#
             }
             base64_keys.push(k.to_xdr_base64(Limits::none())?);
         }
-        Ok(self
-            .client()?
-            .request("getLedgerEntries", rpc_params![base64_keys])
-            .await?)
+        let mut oparams = ObjectParams::new();
+        oparams.insert("keys", base64_keys)?;
+        Ok(self.client()?.request("getLedgerEntries", oparams).await?)
     }
 
     ///
@@ -1070,10 +1091,20 @@ mod tests {
  "minResourceFee": "100000000",
  "cost": { "cpuInsns": "1000", "memBytes": "1000" },
  "transactionData": "",
- "latestLedger": 1234
-        }"#;
+ "latestLedger": 1234,
+ "stateChanges": [{
+    "type": "created",
+    "key": "AAAAAAAAAABuaCbVXZ2DlXWarV6UxwbW3GNJgpn3ASChIFp5bxSIWg==",
+    "before": null,
+    "after": "AAAAZAAAAAAAAAAAbmgm1V2dg5V1mq1elMcG1txjSYKZ9wEgoSBaeW8UiFoAAAAAAAAAZAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+  }]
+  }"#;
 
         let resp: SimulateTransactionResponse = serde_json::from_str(s).unwrap();
+        assert_eq!(
+            resp.state_changes.unwrap()[0],
+            LedgerEntryChange::Created { key: "AAAAAAAAAABuaCbVXZ2DlXWarV6UxwbW3GNJgpn3ASChIFp5bxSIWg==".to_string(), after: "AAAAZAAAAAAAAAAAbmgm1V2dg5V1mq1elMcG1txjSYKZ9wEgoSBaeW8UiFoAAAAAAAAAZAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=".to_string() },
+        );
         assert_eq!(resp.min_resource_fee, 100_000_000);
     }
 

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -34,10 +34,11 @@ import (
 )
 
 const (
-	prometheusNamespace          = "soroban_rpc"
-	maxLedgerEntryWriteBatchSize = 150
-	defaultReadTimeout           = 5 * time.Second
-	defaultShutdownGracePeriod   = 10 * time.Second
+	prometheusNamespace                   = "soroban_rpc"
+	maxLedgerEntryWriteBatchSize          = 150
+	defaultReadTimeout                    = 5 * time.Second
+	defaultShutdownGracePeriod            = 10 * time.Second
+	inMemoryInitializationLedgerLogPeriod = 1_000_000
 )
 
 type Daemon struct {
@@ -137,6 +138,11 @@ func MustNew(cfg *config.Config) *Daemon {
 		logger.UseJSONFormatter()
 	}
 
+	logger.WithFields(supportlog.F{
+		"version": config.Version,
+		"commit":  config.CommitHash,
+	}).Info("starting Soroban RPC")
+
 	core, err := newCaptiveCore(cfg, logger)
 	if err != nil {
 		logger.WithError(err).Fatal("could not create captive core")
@@ -196,7 +202,20 @@ func MustNew(cfg *config.Config) *Daemon {
 	// NOTE: We could optimize this to avoid unnecessary ingestion calls
 	//       (the range of txmetads can be larger than the store retention windows)
 	//       but it's probably not worth the pain.
+	var initialSeq uint32
+	var currentSeq uint32
 	err = db.NewLedgerReader(dbConn).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error {
+		currentSeq = txmeta.LedgerSequence()
+		if initialSeq == 0 {
+			initialSeq = currentSeq
+			logger.WithFields(supportlog.F{
+				"seq": currentSeq,
+			}).Info("initializing in-memory store")
+		} else if (currentSeq-initialSeq)%inMemoryInitializationLedgerLogPeriod == 0 {
+			logger.WithFields(supportlog.F{
+				"seq": currentSeq,
+			}).Debug("still initializing in-memory store")
+		}
 		if err := eventStore.IngestEvents(txmeta); err != nil {
 			logger.WithError(err).Fatal("could not initialize event memory store")
 		}
@@ -205,6 +224,11 @@ func MustNew(cfg *config.Config) *Daemon {
 		}
 		return nil
 	})
+	if currentSeq != 0 {
+		logger.WithFields(supportlog.F{
+			"seq": currentSeq,
+		}).Info("finished initializing in-memory store")
+	}
 	if err != nil {
 		logger.WithError(err).Fatal("could not obtain txmeta cache from the database")
 	}
@@ -285,10 +309,8 @@ func MustNew(cfg *config.Config) *Daemon {
 
 func (d *Daemon) Run() {
 	d.logger.WithFields(supportlog.F{
-		"version": config.Version,
-		"commit":  config.CommitHash,
-		"addr":    d.server.Addr,
-	}).Info("starting Soroban JSON RPC server")
+		"addr": d.server.Addr,
+	}).Info("starting HTTP server")
 
 	panicGroup := util.UnrecoverablePanicGroup.Log(d.logger)
 	panicGroup.Go(func() {

cmd/soroban-rpc/internal/events/events.go

@@ -265,3 +265,10 @@ func readEvents(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (
 	}
 	return events, err
 }
+
+// GetLedgerRange returns the first and latest ledger available in the store.
+func (m *MemoryStore) GetLedgerRange() ledgerbucketwindow.LedgerRange {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+	return m.eventsByLedger.GetLedgerRange()
+}

cmd/soroban-rpc/internal/jsonrpc.go

@@ -134,6 +134,15 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler {
 			Logger: func(text string) { params.Logger.Debug(text) },
 		},
 	}
+
+	// Get the largest history window
+	var ledgerRangeGetter methods.LedgerRangeGetter = params.EventStore
+	var retentionWindow = cfg.EventLedgerRetentionWindow
+	if cfg.TransactionLedgerRetentionWindow > cfg.EventLedgerRetentionWindow {
+		retentionWindow = cfg.TransactionLedgerRetentionWindow
+		ledgerRangeGetter = params.TransactionStore
+	}
+
 	handlers := []struct {
 		methodName           string
 		underlyingHandler    jrpc2.Handler
@@ -143,7 +152,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler {
 	}{
 		{
 			methodName:           "getHealth",
-			underlyingHandler:    methods.NewHealthCheck(params.TransactionStore, cfg.MaxHealthyLedgerLatency),
+			underlyingHandler:    methods.NewHealthCheck(retentionWindow, ledgerRangeGetter, cfg.MaxHealthyLedgerLatency),
 			longName:             "get_health",
 			queueLimit:           cfg.RequestBacklogGetHealthQueueLimit,
 			requestDurationLimit: cfg.MaxGetHealthExecutionDuration,

cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go

@@ -65,6 +65,35 @@ func (w *LedgerBucketWindow[T]) Len() uint32 {
 	return uint32(len(w.buckets))
 }
 
+type LedgerInfo struct {
+	Sequence  uint32
+	CloseTime int64
+}
+
+type LedgerRange struct {
+	FirstLedger LedgerInfo
+	LastLedger  LedgerInfo
+}
+
+func (w *LedgerBucketWindow[T]) GetLedgerRange() LedgerRange {
+	length := w.Len()
+	if length == 0 {
+		return LedgerRange{}
+	}
+	firstBucket := w.Get(0)
+	lastBucket := w.Get(length - 1)
+	return LedgerRange{
+		FirstLedger: LedgerInfo{
+			Sequence:  firstBucket.LedgerSeq,
+			CloseTime: firstBucket.LedgerCloseTimestamp,
+		},
+		LastLedger: LedgerInfo{
+			Sequence:  lastBucket.LedgerSeq,
+			CloseTime: lastBucket.LedgerCloseTimestamp,
+		},
+	}
+}
+
 // Get obtains a bucket from the window
 func (w *LedgerBucketWindow[T]) Get(i uint32) *LedgerBucket[T] {
 	length := w.Len()

cmd/soroban-rpc/internal/methods/get_events.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/creachadair/jrpc2"
-	"github.com/creachadair/jrpc2/handler"
 
 	"github.com/stellar/go/strkey"
 	"github.com/stellar/go/support/errors"
@@ -429,7 +428,7 @@ func NewGetEventsHandler(eventsStore *events.MemoryStore, maxLimit, defaultLimit
 		maxLimit:     maxLimit,
 		defaultLimit: defaultLimit,
 	}
-	return handler.New(func(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) {
+	return NewHandler(func(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) {
 		return eventsHandler.getEvents(request)
 	})
 }