Update CLI and soroban-rpc to the latest XDR changes (#846)

.github/workflows/soroban-rpc.yml

@@ -110,7 +110,7 @@ jobs:
     env:
       SOROBAN_RPC_INTEGRATION_TESTS_ENABLED: true
       SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
-      PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 19.12.1-1419.0ad2053d5.focal~soroban
+      PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 19.12.1-1425.df613c240.focal~soroban
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3

Cargo.toml

@@ -16,18 +16,18 @@ version = "0.9.4"
 [workspace.dependencies.soroban-env-host]
 version = "0.0.17"
 git = "https://github.com/stellar/rs-soroban-env"
-rev = "63cf7fe3d5ffc60db57fba97e9fc9c5778cd559c"
+rev = "c5607a2e9e296b2636b46dc910387aa3446b3e29"
 
 [workspace.dependencies.soroban-spec]
 version = "0.9.1"
 git = "https://github.com/stellar/rs-soroban-sdk"
-rev = "b1cec10b79cee3c1ecf63079b1088ce97b32c68a"
+rev = "5943aa66baf85ebb70d946c37e297abc80d13a78"
 # path = "../rs-soroban-sdk/soroban-spec"
 
 [workspace.dependencies.soroban-spec-rust]
 version = "0.9.1"
 git = "https://github.com/stellar/rs-soroban-sdk"
-rev = "b1cec10b79cee3c1ecf63079b1088ce97b32c68a"
+rev = "5943aa66baf85ebb70d946c37e297abc80d13a78"
 # path = "../rs-soroban-sdk/soroban-spec-rust"
 
 [workspace.dependencies.soroban-spec-json]
@@ -45,12 +45,12 @@ path = "./cmd/crates/soroban-spec-tools"
 [workspace.dependencies.soroban-sdk]
 version = "0.9.1"
 git = "https://github.com/stellar/rs-soroban-sdk"
-rev = "b1cec10b79cee3c1ecf63079b1088ce97b32c68a"
+rev = "5943aa66baf85ebb70d946c37e297abc80d13a78"
 
 [workspace.dependencies.soroban-ledger-snapshot]
 version = "0.9.1"
 git = "https://github.com/stellar/rs-soroban-sdk"
-rev = "b1cec10b79cee3c1ecf63079b1088ce97b32c68a"
+rev = "5943aa66baf85ebb70d946c37e297abc80d13a78"
 
 [workspace.dependencies.soroban-cli]
 version = "0.9.4"
@@ -59,7 +59,7 @@ path = "cmd/soroban-cli"
 [workspace.dependencies.stellar-xdr]
 version = "0.0.17"
 git = "https://github.com/stellar/rs-stellar-xdr"
-rev = "4eaf2388c1de6fc295ed5f7df8174c199923df5b"
+rev = "d6e02584ac9f4046bf38eaf445ced0d4f33631fd"
 default-features = false
 
 [workspace.dependencies]

cmd/soroban-cli/src/commands/contract/bump.rs

@@ -164,7 +164,7 @@ impl Cmd {
                     instructions: 0,
                     read_bytes: 0,
                     write_bytes: 0,
-                    extended_meta_data_size_bytes: 0,
+                    contract_events_size_bytes: 0,
                 },
                 refundable_fee: 0,
             }),

cmd/soroban-cli/src/commands/contract/invoke.rs

@@ -8,6 +8,8 @@ use std::{fmt::Debug, fs, io, rc::Rc};
 
 use clap::{arg, command, value_parser, Parser};
 use heck::ToKebabCase;
+use soroban_env_host::e2e_invoke::get_ledger_changes;
+use soroban_env_host::xdr::ReadXdr;
 use soroban_env_host::{
     budget::Budget,
     events::HostEvent,
@@ -403,7 +405,8 @@ impl Cmd {
                 root_invocation: payload.invocation,
             })
             .collect();
-        let (storage, budget, events, expiration_ledger_bumps) = h.try_finish().map_err(|h| h.1)?;
+        let budget = h.budget_cloned();
+        let (storage, events) = h.try_finish()?;
         let footprint = &create_ledger_footprint(&storage.footprint);
         log_events(
             footprint,
@@ -412,6 +415,14 @@ impl Cmd {
             Some(&budget),
         );
 
+        let ledger_changes = get_ledger_changes(&budget, &storage, &state)?;
+        let mut expiration_ledger_bumps: HashMap<LedgerKey, u32> = HashMap::new();
+        for ledger_entry_change in ledger_changes {
+            if let Some(exp_change) = ledger_entry_change.expiration_change {
+                let key = xdr::LedgerKey::from_xdr(ledger_entry_change.encoded_key)?;
+                expiration_ledger_bumps.insert(key, exp_change.new_expiration_ledger);
+            }
+        }
         utils::bump_ledger_entry_expirations(&mut state.ledger_entries, &expiration_ledger_bumps);
 
         self.config.set_state(&mut state)?;

cmd/soroban-cli/src/commands/contract/restore.rs

@@ -171,7 +171,7 @@ impl Cmd {
                     instructions: 0,
                     read_bytes: 0,
                     write_bytes: 0,
-                    extended_meta_data_size_bytes: 0,
+                    contract_events_size_bytes: 0,
                 },
                 refundable_fee: 0,
             }),

cmd/soroban-cli/src/commands/events.rs

@@ -344,6 +344,7 @@ mod tests {
             max_entry_expiration: 6,
             min_persistent_entry_expiration: 7,
             min_temp_entry_expiration: 8,
+            autobump_ledgers: 16,
         };
 
         events_file.commit(&events, &ledger_info, &temp).unwrap();

cmd/soroban-cli/src/rpc/transaction.rs

@@ -110,7 +110,7 @@ mod tests {
                 instructions: 0,
                 read_bytes: 5,
                 write_bytes: 0,
-                extended_meta_data_size_bytes: 0,
+                contract_events_size_bytes: 0,
             },
             refundable_fee: 0,
             ext: ExtensionPoint::V0,

cmd/soroban-cli/src/utils.rs

@@ -1,11 +1,11 @@
+use std::hash::BuildHasher;
 use std::{collections::HashMap, io::ErrorKind, path::Path};
 
 use ed25519_dalek::Signer;
 use sha2::{Digest, Sha256};
 
 use soroban_env_host::{
     budget::Budget,
-    expiration_ledger_bumps::ExpirationLedgerBumps,
     storage::{AccessType, Footprint, Storage},
     xdr::{
         AccountEntry, AccountEntryExt, AccountId, Asset, ContractCodeEntry, ContractCodeEntryBody,
@@ -134,15 +134,10 @@ pub fn add_contract_to_ledger_entries(
     entries.push((Box::new(contract_key), Box::new(contract_entry)));
 }
 
-pub fn bump_ledger_entry_expirations(
+pub fn bump_ledger_entry_expirations<S: BuildHasher>(
     entries: &mut [(Box<LedgerKey>, Box<LedgerEntry>)],
-    bumps: &ExpirationLedgerBumps,
+    lookup: &HashMap<LedgerKey, u32, S>,
 ) {
-    // let lookup: HashMap<LedgerKey, u32> = bumps
-    let lookup = bumps
-        .iter()
-        .map(|b| (b.key.as_ref().clone(), b.min_expiration))
-        .collect::<HashMap<_, _>>();
     for (k, e) in entries.iter_mut() {
         if let Some(min_expiration) = lookup.get(k.as_ref()) {
             if let LedgerEntryData::ContractData(entry) = &mut e.data {

cmd/soroban-rpc/internal/db/ledgerentry.go

@@ -253,7 +253,7 @@ func (l *ledgerEntryReadTx) GetLedgerEntry(key xdr.LedgerKey, includeExpired boo
 			if err != nil {
 				return false, xdr.LedgerEntry{}, err
 			}
-			if expirationLedgerSeq <= xdr.Uint32(latestClosedLedger) {
+			if expirationLedgerSeq < xdr.Uint32(latestClosedLedger) {
 				return false, xdr.LedgerEntry{}, nil
 			}
 		}

cmd/soroban-rpc/internal/db/ledgerentry_test.go

@@ -357,8 +357,9 @@ func TestGetLedgerEntryHidesExpiredContractDataEntries(t *testing.T) {
 	}{
 		{21, true},
 		{22, true},
-		{23, false},
+		{23, true},
 		{24, false},
+		{25, false},
 	} {
 		// ffwd to the ledger sequence
 		tx, err := NewReadWriter(db, 0, 15).NewTx(context.Background())
@@ -403,8 +404,9 @@ func TestGetLedgerEntryHidesExpiredContractCodeEntries(t *testing.T) {
 	}{
 		{21, true},
 		{22, true},
-		{23, false},
+		{23, true},
 		{24, false},
+		{25, false},
 	} {
 		// ffwd to the ledger sequence
 		tx, err := NewReadWriter(db, 0, 15).NewTx(context.Background())

cmd/soroban-rpc/internal/ingest/service.go

@@ -218,7 +218,7 @@ func (s *Service) fillEntriesFromCheckpoint(ctx context.Context, archive history
 
 func (s *Service) ingest(ctx context.Context, sequence uint32) error {
 	startTime := time.Now()
-	s.logger.Infof("Applying txmeta for ledger %d", sequence)
+	s.logger.Infof("Ingesting ledger %d", sequence)
 	ledgerCloseMeta, err := s.ledgerBackend.GetLedger(ctx, sequence)
 	if err != nil {
 		return err
@@ -251,6 +251,7 @@ func (s *Service) ingest(ctx context.Context, sequence uint32) error {
 	if err := tx.Commit(sequence); err != nil {
 		return err
 	}
+	s.logger.Debugf("Ingested ledger %d", sequence)
 
 	s.ingestionDurationMetric.
 		With(prometheus.Labels{"type": "total"}).Observe(time.Since(startTime).Seconds())

cmd/soroban-rpc/internal/preflight/preflight.go

@@ -151,11 +151,17 @@ func getFootprintExpirationPreflight(params PreflightParameters) (Preflight, err
 	handle := cgo.NewHandle(snapshotSourceHandle{params.LedgerEntryReadTx, params.Logger})
 	defer handle.Delete()
 
+	latestLedger, err := params.LedgerEntryReadTx.GetLatestLedgerSequence()
+	if err != nil {
+		return Preflight{}, err
+	}
+
 	res := C.preflight_footprint_expiration_op(
 		C.uintptr_t(handle),
 		C.uint64_t(params.BucketListSize),
 		opBodyCString,
 		footprintCString,
+		C.uint32_t(latestLedger),
 	)
 
 	C.free(unsafe.Pointer(opBodyCString))
@@ -188,26 +194,26 @@ func getInvokeHostFunctionPreflight(params PreflightParameters) (Preflight, erro
 	if err != nil {
 		return Preflight{}, err
 	}
-	minTempEntryExpiration := uint32(0)
-	minPersistentEntryExpiration := uint32(0)
-	maxEntryExpiration := uint32(0)
-	if hasConfig {
-		setting := stateExpirationConfig.Data.MustConfigSetting().MustStateExpirationSettings()
-		minTempEntryExpiration = uint32(setting.MinTempEntryExpiration)
-		minPersistentEntryExpiration = uint32(setting.MinPersistentEntryExpiration)
-		maxEntryExpiration = uint32(setting.MaxEntryExpiration)
+	if !hasConfig {
+		return Preflight{}, errors.New("state expiration config setting missing in ledger storage")
 	}
 
+	stateExpiration := stateExpirationConfig.Data.MustConfigSetting().MustStateExpirationSettings()
+	// It's of utmost importance to simulate the transactions like we were on the next ledger.
+	// Otherwise, users would need to wait for an extra ledger to close in order to observe the effects of the latest ledger
+	// transaction submission.
+	sequenceNumber := latestLedger + 1
 	li := C.CLedgerInfo{
 		network_passphrase: C.CString(params.NetworkPassphrase),
-		sequence_number:    C.uint(latestLedger),
+		sequence_number:    C.uint32_t(sequenceNumber),
 		protocol_version:   20,
 		timestamp:          C.uint64_t(time.Now().Unix()),
 		// Current base reserve is 0.5XLM (in stroops)
 		base_reserve:                    5_000_000,
-		min_temp_entry_expiration:       C.uint(minTempEntryExpiration),
-		min_persistent_entry_expiration: C.uint(minPersistentEntryExpiration),
-		max_entry_expiration:            C.uint(maxEntryExpiration),
+		min_temp_entry_expiration:       C.uint(stateExpiration.MinTempEntryExpiration),
+		min_persistent_entry_expiration: C.uint(stateExpiration.MinPersistentEntryExpiration),
+		max_entry_expiration:            C.uint(stateExpiration.MaxEntryExpiration),
+		auto_bump_ledgers:               C.uint(stateExpiration.AutoBumpLedgers),
 	}
 
 	sourceAccountCString := C.CString(sourceAccountB64)

cmd/soroban-rpc/internal/preflight/preflight_test.go

@@ -137,10 +137,10 @@ var mockLedgerEntries = []xdr.LedgerEntry{
 		Data: xdr.LedgerEntryData{
 			Type: xdr.LedgerEntryTypeConfigSetting,
 			ConfigSetting: &xdr.ConfigSettingEntry{
-				ConfigSettingId: xdr.ConfigSettingIdConfigSettingContractMetaDataV0,
-				ContractMetaData: &xdr.ConfigSettingContractMetaDataV0{
-					TxMaxExtendedMetaDataSizeBytes: 100,
-					FeeExtendedMetaData1Kb:         100,
+				ConfigSettingId: xdr.ConfigSettingIdConfigSettingContractEventsV0,
+				ContractEvents: &xdr.ConfigSettingContractEventsV0{
+					TxMaxContractEventsSizeBytes: 10000,
+					FeeContractEvents1Kb:         1,
 				},
 			},
 		},
@@ -152,9 +152,9 @@ var mockLedgerEntries = []xdr.LedgerEntry{
 			ConfigSetting: &xdr.ConfigSettingEntry{
 				ConfigSettingId: xdr.ConfigSettingIdConfigSettingContractBandwidthV0,
 				ContractBandwidth: &xdr.ConfigSettingContractBandwidthV0{
-					LedgerMaxPropagateSizeBytes: 100,
-					TxMaxSizeBytes:              100,
-					FeePropagateData1Kb:         100,
+					LedgerMaxTxsSizeBytes: 100000,
+					TxMaxSizeBytes:        1000,
+					FeeTxSize1Kb:          1,
 				},
 			},
 		},

cmd/soroban-rpc/internal/test/docker-compose.yml

@@ -15,7 +15,7 @@ services:
     # Note: Please keep the image pinned to an immutable tag matching the Captive Core version.
     #       This avoid implicit updates which break compatibility between
     #       the Core container and captive core.
-    image: ${CORE_IMAGE:-2opremio/stellar-core:19.12.1-1419.0ad2053d5.focal-soroban}
+    image: ${CORE_IMAGE:-2opremio/stellar-core:19.12.1-1425.df613c240.focal-soroban}
     depends_on:
       - core-postgres
     restart: on-failure