Merge branch 'main' into flatten-getEvents

.cargo-husky/hooks/pre-commit

@@ -4,6 +4,3 @@ set -e
 
 echo '+cargo fmt --check'
 cargo fmt --check || (cargo fmt && exit 1)
-
-echo '+cargo run --bin doc-gen --features clap-markdown'
-cargo run --bin doc-gen --features clap-markdown

.cargo-husky/hooks/pre-push

@@ -13,4 +13,8 @@ echo '+cargo clippy -- -Dwarnings -Dclippy::all -Dclippy::pedantic'
 cargo clippy --all -- -Dwarnings
 
 echo '+cargo test --all'
-cargo test --all || (echo "might need to rebuild make build-snapshot` && exit 1)
+cargo build
+cargo test --all || (echo "might need to rebuild make build-snapshot" && exit 1)
+
+echo '+cargo run --bin doc-gen --features clap-markdown'
+cargo run --bin doc-gen --features clap-markdown

.github/actions/setup-integration-tests/action.yml

@@ -0,0 +1,61 @@
+name: 'Set up integration tests'
+description: 'Set up Go & Rust, build artifacts, work around cache issues and Ubuntu quirks'
+inputs:
+  go-version:
+    required: true
+runs:
+  using: "composite"
+  steps:
+  - uses: ./.github/actions/setup-go
+    with:
+      go-version: ${{ matrix.go }}
+  - uses: stellar/actions/rust-cache@main
+  - name: Build soroban contract fixtures
+    shell: bash
+    run: |
+      rustup update
+      rustup target add wasm32-unknown-unknown
+      make build_rust
+      make build-test-wasms
+
+  - name: Install Captive Core
+    shell: bash
+    run: |
+      # Workaround for https://github.com/actions/virtual-environments/issues/5245,
+      # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpful
+      # message about why the installation fails)
+      sudo apt-get remove -y libc++1-10 libc++abi1-10 || true
+
+      sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+      sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+      sudo apt-get update && sudo apt-get install -y stellar-core="$PROTOCOL_20_CORE_DEBIAN_PKG_VERSION"
+      echo "Using stellar core version $(stellar-core version)"
+
+  # Docker-compose's remote contexts on Ubuntu 20 started failing with an OpenSSL versioning error.
+  # See https://stackoverflow.com/questions/66579446/error-executing-docker-compose-building-webserver-unable-to-prepare-context-un
+  - name: Work around Docker Compose problem
+    shell: bash
+    run: |
+      sudo apt-get update
+      sudo apt-get install -y ca-certificates curl gnupg
+
+      # Install docker apt repo
+      sudo install -m 0755 -d /etc/apt/keyrings
+      curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+      sudo chmod a+r /etc/apt/keyrings/docker.gpg
+      echo \
+        "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+        "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+        sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+      # Install docker-compose v2 from apt repo
+      sudo apt-get update
+      sudo apt-get remove -y moby-compose
+      sudo apt-get install -y docker-compose-plugin
+
+      echo "Docker Compose Version:"
+      docker-compose version
+
+  - name: Build libpreflight
+    shell: bash
+    run: make build-libpreflight

.github/workflows/soroban-rpc.yml

@@ -102,15 +102,17 @@ jobs:
 
   integration:
     name: Integration tests
+    continue-on-error: true
     strategy:
       matrix:
         os: [ubuntu-20.04]
         go: [1.20.1]
+        test: ['.*CLI.*', '^Test(([^C])|(C[^L])|(CL[^I])).*$']
+    runs-on: ${{ matrix.os }}
     env:
       SOROBAN_RPC_INTEGRATION_TESTS_ENABLED: true
       SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core
       PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 19.13.1-1481.3acf6dd26.focal
-    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -119,57 +121,9 @@ jobs:
           # We need to full history for git-restore-mtime to know what modification dates to use.
           # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing).
           fetch-depth: "0"
-
-      - uses: ./.github/actions/setup-go
+      - uses: ./.github/actions/setup-integration-tests
         with:
           go-version: ${{ matrix.go }}
-      - uses: stellar/actions/rust-cache@main
-      - name: Build soroban contract fixtures
-        run: |
-          rustup update
-          rustup target add wasm32-unknown-unknown
-          make build_rust
-          make build-test-wasms
-
-      - name: Install Captive Core
-        run: |
-          # Workaround for https://github.com/actions/virtual-environments/issues/5245,
-          # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpful
-          # message about why the installation fails)
-          sudo apt-get remove -y libc++1-10 libc++abi1-10 || true
-
-          sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
-          sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
-          sudo apt-get update && sudo apt-get install -y stellar-core="$PROTOCOL_20_CORE_DEBIAN_PKG_VERSION"
-          echo "Using stellar core version $(stellar-core version)"
-
-      # Docker-compose's remote contexts on Ubuntu 20 started failing with an OpenSSL versioning error.
-      # See https://stackoverflow.com/questions/66579446/error-executing-docker-compose-building-webserver-unable-to-prepare-context-un
-      - name: Work around Docker Compose problem
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y ca-certificates curl gnupg
-
-          # Install docker apt repo
-          sudo install -m 0755 -d /etc/apt/keyrings
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-          sudo chmod a+r /etc/apt/keyrings/docker.gpg
-          echo \
-            "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-            "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
-            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-          # Install docker-compose v2 from apt repo
-          sudo apt-get update
-          sudo apt-get remove -y moby-compose
-          sudo apt-get install -y docker-compose-plugin
-
-          echo "Docker Compose Version:"
-          docker-compose version
-
-      - name: Build libpreflight
-        run: make build-libpreflight
-
       - name: Run Soroban RPC Integration Tests
         run: |
-          go test -race -timeout 25m -v ./cmd/soroban-rpc/internal/test/...
+          go test -race -run '${{ matrix.test }}' -timeout 60m -v ./cmd/soroban-rpc/internal/test/...

.gitignore

@@ -1,5 +1,7 @@
 target/
+captive-core/
 .soroban/
 !test.toml
+*.sqlite
 
 cmd/crates/soroban-spec-typescript/fixtures/ts/package-lock.json

cmd/crates/soroban-test/tests/fixtures/test-wasms/udt/src/lib.rs

@@ -34,6 +34,7 @@ pub struct Contract;
 
 #[contractimpl]
 impl Contract {
+    #[allow(clippy::unnecessary_fold)]
     pub fn add(a: UdtEnum, b: UdtEnum) -> i64 {
         let a = match a {
             UdtEnum::UdtA => 0,
@@ -45,7 +46,7 @@ impl Contract {
             UdtEnum::UdtA => 0,
             UdtEnum::UdtB(udt) => udt.a + udt.b,
             UdtEnum::UdtC(val) => val as i64,
-            UdtEnum::UdtD(tup) => tup.0 + tup.1.iter().fold(0i64, |sum, i| sum + i),
+            UdtEnum::UdtD(tup) => tup.0 + tup.1.iter().sum::<i64>(),
         };
         a + b
     }

cmd/soroban-rpc/README.md

@@ -0,0 +1,58 @@
+# Soroban-RPC
+
+Soroban-RPC allows you to communicate directly with Soroban via a JSON RPC interface.
+
+For example, you can build an application and have it send a transaction, get ledger and event data or simulate transactions.
+
+## Dependencies
+  - [Git](https://git-scm.com/downloads)
+  - [Go](https://golang.org/doc/install)
+  - [Rust](https://www.rust-lang.org/tools/install)
+  - [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html)
+
+## Building Stellar-Core
+Soroban-RPC requires an instance of stellar-core binary on the same host. This is referred to as the `Captive Core`. 
+Since, we are building RPC from source, we recommend considering two approaches to get the stellar-core binary:
+- If saving time is top priority and your development machine is on a linux debian OS, then consider installing the 
+testnet release candidates from the [testing repository.](https://apt.stellar.org/pool/unstable/s/stellar-core/)
+- The recommended option is to compile the core source directly on your machine:
+    - Clone the stellar-core repo:
+        ```bash
+        git clone https://github.com/stellar/stellar-core.git
+        cd stellar-core
+        ```
+    - Fetch the tags and checkout the testnet release tag:
+        ```bash
+        git fetch --tags
+        git checkout tags/v20.0.0-rc.2.1 -b soroban-testnet-release
+        ```
+    - Follow the build steps listed in [INSTALL.md](https://github.com/stellar/stellar-core/blob/master/INSTALL.md) file for the instructions on building the local binary
+
+## Building Soroban-RPC
+- Similar to stellar-core, we will clone the soroban-tools repo and checkout the testnet release tag:
+```bash
+git clone https://github.com/stellar/soroban-tools.git
+cd soroban-tools
+git fetch --tags
+git checkout tags/v20.0.0-rc4 -b soroban-testnet-release
+```
+- Build soroban-rpc target:
+```bash
+make build-soroban-rpc
+```
+This will install and build the required dependencies and generate a `soroban-rpc` binary in the working directory.
+
+## Configuring and Running RPC Server
+- Both stellar-core and soroban-rpc require configuration files to run. 
+  - For production, we specifically recommend running Soroban RPC with a TOML configuration file rather than CLI flags. 
+  - There is a new subcommand `gen-config-file` which takes all the same arguments as the root command (or no arguments at all), 
+  and outputs the resulting config toml file to stdout.
+      ```bash
+      ./soroban-rpc gen-config-file
+      ```
+  - Paste the output to a file and save it as `.toml` file in any directory. 
+  - Make sure to update the config values to testnet specific ones. You can refer to [Configuring](https://docs.google.com/document/d/1SIbrFWFgju5RAsi6stDyEtgTa78VEt8f3HhqCLoySx4/edit#heading=h.80d1jdtd7ktj) section in the Runbook for specific config settings.
+- If everything is set up correctly, then you can run the RPC server with the following command:
+```bash
+./soroban-rpc --config-path <PATH_TO_THE_RPC_CONFIG_FILE>
+```

cmd/soroban-rpc/lib/preflight/src/fees.rs

@@ -27,12 +27,14 @@ use std::convert::{TryFrom, TryInto};
 /// Serialize XDR size for any `ExpirationEntry` ledger entry.
 const EXPIRATION_ENTRY_SIZE: u32 = 48;
 
+// TODO: this should perhaps be an new type
+#[allow(clippy::too_many_arguments)]
 pub(crate) fn compute_host_function_transaction_data_and_min_fee(
     op: &InvokeHostFunctionOp,
     pre_storage: &LedgerStorage,
     post_storage: &Storage,
     budget: &Budget,
-    events: &Vec<DiagnosticEvent>,
+    events: &[DiagnosticEvent],
     invocation_result: &ScVal,
     bucket_list_size: u64,
     current_ledger_seq: u32,
@@ -127,8 +129,9 @@ fn estimate_max_transaction_size_for_operation(
     Ok(u32::try_from(envelope_size)?)
 }
 
+#[allow(clippy::cast_possible_truncation)]
 fn calculate_host_function_soroban_resources(
-    ledger_changes: &Vec<LedgerEntryChange>,
+    ledger_changes: &[LedgerEntryChange],
     footprint: &Footprint,
     budget: &Budget,
 ) -> Result<SorobanResources> {
@@ -162,6 +165,7 @@ fn calculate_host_function_soroban_resources(
     })
 }
 
+#[allow(clippy::cast_possible_wrap)]
 fn get_fee_configurations(
     ledger_storage: &LedgerStorage,
     bucket_list_size: u64,
@@ -232,18 +236,19 @@ fn get_fee_configurations(
 }
 
 // Calculate the implicit ExpirationEntry bytes that will be read for expirable LedgerEntries
-fn calculate_expiration_entry_bytes(ledger_entries: &Vec<LedgerKey>) -> Result<u32> {
-    Ok(ledger_entries
+fn calculate_expiration_entry_bytes(ledger_entries: &[LedgerKey]) -> u32 {
+    ledger_entries
         .iter()
         .map(|lk| match lk {
             LedgerKey::ContractData(_) | LedgerKey::ContractCode(_) => EXPIRATION_ENTRY_SIZE,
             _ => 0,
         })
-        .sum())
+        .sum()
 }
 
+#[allow(clippy::cast_possible_truncation)]
 fn calculate_unmodified_ledger_entry_bytes(
-    ledger_entries: &Vec<LedgerKey>,
+    ledger_entries: &[LedgerKey],
     pre_storage: &LedgerStorage,
     include_expired: bool,
 ) -> Result<u32> {
@@ -258,7 +263,7 @@ fn calculate_unmodified_ledger_entry_bytes(
     Ok(res as u32)
 }
 
-fn calculate_contract_events_size_bytes(events: &Vec<DiagnosticEvent>) -> Result<u32> {
+fn calculate_contract_events_size_bytes(events: &[DiagnosticEvent]) -> Result<u32> {
     let mut res: u32 = 0;
     for e in events {
         if e.event.type_ != ContractEventType::Contract
@@ -302,7 +307,7 @@ fn finalize_transaction_data_and_min_fee(
             .context("failed to obtain configuration settings from the network")?;
     let (non_refundable_fee, refundable_fee) =
         compute_transaction_resource_fee(transaction_resources, &fee_configuration);
-    let rent_fee = compute_rent_fee(&rent_changes, &rent_fee_configuration, current_ledger_seq);
+    let rent_fee = compute_rent_fee(rent_changes, &rent_fee_configuration, current_ledger_seq);
     let transaction_data = SorobanTransactionData {
         resources: soroban_resources,
         refundable_fee: refundable_fee + rent_fee,
@@ -330,10 +335,10 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee(
     )
     .context("cannot compute bump rent changes")?;
 
-    let expiration_bytes: u32 = calculate_expiration_entry_bytes(footprint.read_only.as_vec())?;
+    let expiration_bytes = calculate_expiration_entry_bytes(footprint.read_only.as_slice());
 
     let unmodified_entry_bytes = calculate_unmodified_ledger_entry_bytes(
-        footprint.read_only.as_vec(),
+        footprint.read_only.as_slice(),
         ledger_storage,
         false,
     )
@@ -363,7 +368,7 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee(
         contract_events_size_bytes: 0,
     };
     finalize_transaction_data_and_min_fee(
-        &ledger_storage,
+        ledger_storage,
         &transaction_resources,
         soroban_resources,
         &rent_changes,
@@ -372,6 +377,7 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee(
     )
 }
 
+#[allow(clippy::cast_possible_truncation)]
 fn compute_bump_footprint_rent_changes(
     footprint: &LedgerFootprint,
     ledger_storage: &LedgerStorage,
@@ -380,7 +386,7 @@ fn compute_bump_footprint_rent_changes(
 ) -> Result<Vec<LedgerEntryRentChange>> {
     let mut rent_changes: Vec<LedgerEntryRentChange> =
         Vec::with_capacity(footprint.read_only.len());
-    for key in (&footprint).read_only.as_vec() {
+    for key in footprint.read_only.as_slice() {
         let unmodified_entry_and_expiration = ledger_storage
             .get(key, false)
             .with_context(|| format!("cannot find bump footprint ledger entry with key {key:?}"))?;
@@ -426,7 +432,7 @@ pub(crate) fn compute_restore_footprint_transaction_data_and_min_fee(
     )
     .context("cannot compute restore rent changes")?;
 
-    let expiration_bytes: u32 = calculate_expiration_entry_bytes(footprint.read_write.as_vec())?;
+    let expiration_bytes = calculate_expiration_entry_bytes(footprint.read_write.as_vec());
     let write_bytes = calculate_unmodified_ledger_entry_bytes(
         footprint.read_write.as_vec(),
         ledger_storage,
@@ -465,6 +471,7 @@ pub(crate) fn compute_restore_footprint_transaction_data_and_min_fee(
     )
 }
 
+#[allow(clippy::cast_possible_truncation)]
 fn compute_restore_footprint_rent_changes(
     footprint: &LedgerFootprint,
     ledger_storage: &LedgerStorage,