diff --git a/.cargo-husky/hooks/pre-commit b/.cargo-husky/hooks/pre-commit index d08d9f9ff..9ffe923d6 100755 --- a/.cargo-husky/hooks/pre-commit +++ b/.cargo-husky/hooks/pre-commit @@ -4,6 +4,3 @@ set -e echo '+cargo fmt --check' cargo fmt --check || (cargo fmt && exit 1) - -echo '+cargo run --bin doc-gen --features clap-markdown' -cargo run --bin doc-gen --features clap-markdown diff --git a/.cargo-husky/hooks/pre-push b/.cargo-husky/hooks/pre-push index 48102f655..9181ab3be 100755 --- a/.cargo-husky/hooks/pre-push +++ b/.cargo-husky/hooks/pre-push @@ -13,4 +13,8 @@ echo '+cargo clippy -- -Dwarnings -Dclippy::all -Dclippy::pedantic' cargo clippy --all -- -Dwarnings echo '+cargo test --all' -cargo test --all || (echo "might need to rebuild make build-snapshot` && exit 1) +cargo build +cargo test --all || (echo "might need to rebuild make build-snapshot" && exit 1) + +echo '+cargo run --bin doc-gen --features clap-markdown' +cargo run --bin doc-gen --features clap-markdown diff --git a/.github/actions/setup-integration-tests/action.yml b/.github/actions/setup-integration-tests/action.yml new file mode 100644 index 000000000..938acb6d4 --- /dev/null +++ b/.github/actions/setup-integration-tests/action.yml @@ -0,0 +1,61 @@ +name: 'Set up integration tests' +description: 'Set up Go & Rust, build artifacts, work around cache issues and Ubuntu quirks' +inputs: + go-version: + required: true +runs: + using: "composite" + steps: + - uses: ./.github/actions/setup-go + with: + go-version: ${{ matrix.go }} + - uses: stellar/actions/rust-cache@main + - name: Build soroban contract fixtures + shell: bash + run: | + rustup update + rustup target add wasm32-unknown-unknown + make build_rust + make build-test-wasms + + - name: Install Captive Core + shell: bash + run: | + # Workaround for https://github.com/actions/virtual-environments/issues/5245, + # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpful + # message about why the installation fails) + sudo apt-get remove -y libc++1-10 libc++abi1-10 || true + + sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add - + sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list' + sudo apt-get update && sudo apt-get install -y stellar-core="$PROTOCOL_20_CORE_DEBIAN_PKG_VERSION" + echo "Using stellar core version $(stellar-core version)" + + # Docker-compose's remote contexts on Ubuntu 20 started failing with an OpenSSL versioning error. + # See https://stackoverflow.com/questions/66579446/error-executing-docker-compose-building-webserver-unable-to-prepare-context-un + - name: Work around Docker Compose problem + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y ca-certificates curl gnupg + + # Install docker apt repo + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + # Install docker-compose v2 from apt repo + sudo apt-get update + sudo apt-get remove -y moby-compose + sudo apt-get install -y docker-compose-plugin + + echo "Docker Compose Version:" + docker-compose version + + - name: Build libpreflight + shell: bash + run: make build-libpreflight diff --git a/.github/workflows/soroban-rpc.yml b/.github/workflows/soroban-rpc.yml index 95daa3e1e..99dc6fc40 100644 --- a/.github/workflows/soroban-rpc.yml +++ b/.github/workflows/soroban-rpc.yml @@ -102,15 +102,17 @@ jobs: integration: name: Integration tests + continue-on-error: true strategy: matrix: os: [ubuntu-20.04] go: [1.20.1] + test: ['.*CLI.*', '^Test(([^C])|(C[^L])|(CL[^I])).*$'] + runs-on: ${{ matrix.os }} env: SOROBAN_RPC_INTEGRATION_TESTS_ENABLED: true SOROBAN_RPC_INTEGRATION_TESTS_CAPTIVE_CORE_BIN: /usr/bin/stellar-core PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 19.13.1-1481.3acf6dd26.focal - runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 with: @@ -119,57 +121,9 @@ jobs: # We need to full history for git-restore-mtime to know what modification dates to use. # Otherwise, the Go test cache will fail (due to the modification time of fixtures changing). fetch-depth: "0" - - - uses: ./.github/actions/setup-go + - uses: ./.github/actions/setup-integration-tests with: go-version: ${{ matrix.go }} - - uses: stellar/actions/rust-cache@main - - name: Build soroban contract fixtures - run: | - rustup update - rustup target add wasm32-unknown-unknown - make build_rust - make build-test-wasms - - - name: Install Captive Core - run: | - # Workaround for https://github.com/actions/virtual-environments/issues/5245, - # libc++1-8 won't be installed if another version is installed (but apt won't give you a helpful - # message about why the installation fails) - sudo apt-get remove -y libc++1-10 libc++abi1-10 || true - - sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add - - sudo bash -c 'echo "deb https://apt.stellar.org focal unstable" > /etc/apt/sources.list.d/SDF-unstable.list' - sudo apt-get update && sudo apt-get install -y stellar-core="$PROTOCOL_20_CORE_DEBIAN_PKG_VERSION" - echo "Using stellar core version $(stellar-core version)" - - # Docker-compose's remote contexts on Ubuntu 20 started failing with an OpenSSL versioning error. - # See https://stackoverflow.com/questions/66579446/error-executing-docker-compose-building-webserver-unable-to-prepare-context-un - - name: Work around Docker Compose problem - run: | - sudo apt-get update - sudo apt-get install -y ca-certificates curl gnupg - - # Install docker apt repo - sudo install -m 0755 -d /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - sudo chmod a+r /etc/apt/keyrings/docker.gpg - echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - - # Install docker-compose v2 from apt repo - sudo apt-get update - sudo apt-get remove -y moby-compose - sudo apt-get install -y docker-compose-plugin - - echo "Docker Compose Version:" - docker-compose version - - - name: Build libpreflight - run: make build-libpreflight - - name: Run Soroban RPC Integration Tests run: | - go test -race -timeout 25m -v ./cmd/soroban-rpc/internal/test/... + go test -race -run '${{ matrix.test }}' -timeout 60m -v ./cmd/soroban-rpc/internal/test/... diff --git a/.gitignore b/.gitignore index 3c608cc13..04525f117 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ target/ +captive-core/ .soroban/ !test.toml +*.sqlite cmd/crates/soroban-spec-typescript/fixtures/ts/package-lock.json diff --git a/cmd/crates/soroban-test/tests/fixtures/test-wasms/udt/src/lib.rs b/cmd/crates/soroban-test/tests/fixtures/test-wasms/udt/src/lib.rs index ee0cc1375..695d8a7a3 100644 --- a/cmd/crates/soroban-test/tests/fixtures/test-wasms/udt/src/lib.rs +++ b/cmd/crates/soroban-test/tests/fixtures/test-wasms/udt/src/lib.rs @@ -34,6 +34,7 @@ pub struct Contract; #[contractimpl] impl Contract { + #[allow(clippy::unnecessary_fold)] pub fn add(a: UdtEnum, b: UdtEnum) -> i64 { let a = match a { UdtEnum::UdtA => 0, @@ -45,7 +46,7 @@ impl Contract { UdtEnum::UdtA => 0, UdtEnum::UdtB(udt) => udt.a + udt.b, UdtEnum::UdtC(val) => val as i64, - UdtEnum::UdtD(tup) => tup.0 + tup.1.iter().fold(0i64, |sum, i| sum + i), + UdtEnum::UdtD(tup) => tup.0 + tup.1.iter().sum::(), }; a + b } diff --git a/cmd/soroban-rpc/README.md b/cmd/soroban-rpc/README.md new file mode 100644 index 000000000..da2baf4e0 --- /dev/null +++ b/cmd/soroban-rpc/README.md @@ -0,0 +1,58 @@ +# Soroban-RPC + +Soroban-RPC allows you to communicate directly with Soroban via a JSON RPC interface. + +For example, you can build an application and have it send a transaction, get ledger and event data or simulate transactions. + +## Dependencies + - [Git](https://git-scm.com/downloads) + - [Go](https://golang.org/doc/install) + - [Rust](https://www.rust-lang.org/tools/install) + - [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) + +## Building Stellar-Core +Soroban-RPC requires an instance of stellar-core binary on the same host. This is referred to as the `Captive Core`. +Since, we are building RPC from source, we recommend considering two approaches to get the stellar-core binary: +- If saving time is top priority and your development machine is on a linux debian OS, then consider installing the +testnet release candidates from the [testing repository.](https://apt.stellar.org/pool/unstable/s/stellar-core/) +- The recommended option is to compile the core source directly on your machine: + - Clone the stellar-core repo: + ```bash + git clone https://github.com/stellar/stellar-core.git + cd stellar-core + ``` + - Fetch the tags and checkout the testnet release tag: + ```bash + git fetch --tags + git checkout tags/v20.0.0-rc.2.1 -b soroban-testnet-release + ``` + - Follow the build steps listed in [INSTALL.md](https://github.com/stellar/stellar-core/blob/master/INSTALL.md) file for the instructions on building the local binary + +## Building Soroban-RPC +- Similar to stellar-core, we will clone the soroban-tools repo and checkout the testnet release tag: +```bash +git clone https://github.com/stellar/soroban-tools.git +cd soroban-tools +git fetch --tags +git checkout tags/v20.0.0-rc4 -b soroban-testnet-release +``` +- Build soroban-rpc target: +```bash +make build-soroban-rpc +``` +This will install and build the required dependencies and generate a `soroban-rpc` binary in the working directory. + +## Configuring and Running RPC Server +- Both stellar-core and soroban-rpc require configuration files to run. + - For production, we specifically recommend running Soroban RPC with a TOML configuration file rather than CLI flags. + - There is a new subcommand `gen-config-file` which takes all the same arguments as the root command (or no arguments at all), + and outputs the resulting config toml file to stdout. + ```bash + ./soroban-rpc gen-config-file + ``` + - Paste the output to a file and save it as `.toml` file in any directory. + - Make sure to update the config values to testnet specific ones. You can refer to [Configuring](https://docs.google.com/document/d/1SIbrFWFgju5RAsi6stDyEtgTa78VEt8f3HhqCLoySx4/edit#heading=h.80d1jdtd7ktj) section in the Runbook for specific config settings. +- If everything is set up correctly, then you can run the RPC server with the following command: +```bash +./soroban-rpc --config-path +``` \ No newline at end of file diff --git a/cmd/soroban-rpc/lib/preflight/src/fees.rs b/cmd/soroban-rpc/lib/preflight/src/fees.rs index 86d3a9b44..d30497700 100644 --- a/cmd/soroban-rpc/lib/preflight/src/fees.rs +++ b/cmd/soroban-rpc/lib/preflight/src/fees.rs @@ -27,12 +27,14 @@ use std::convert::{TryFrom, TryInto}; /// Serialize XDR size for any `ExpirationEntry` ledger entry. const EXPIRATION_ENTRY_SIZE: u32 = 48; +// TODO: this should perhaps be an new type +#[allow(clippy::too_many_arguments)] pub(crate) fn compute_host_function_transaction_data_and_min_fee( op: &InvokeHostFunctionOp, pre_storage: &LedgerStorage, post_storage: &Storage, budget: &Budget, - events: &Vec, + events: &[DiagnosticEvent], invocation_result: &ScVal, bucket_list_size: u64, current_ledger_seq: u32, @@ -127,8 +129,9 @@ fn estimate_max_transaction_size_for_operation( Ok(u32::try_from(envelope_size)?) } +#[allow(clippy::cast_possible_truncation)] fn calculate_host_function_soroban_resources( - ledger_changes: &Vec, + ledger_changes: &[LedgerEntryChange], footprint: &Footprint, budget: &Budget, ) -> Result { @@ -162,6 +165,7 @@ fn calculate_host_function_soroban_resources( }) } +#[allow(clippy::cast_possible_wrap)] fn get_fee_configurations( ledger_storage: &LedgerStorage, bucket_list_size: u64, @@ -232,18 +236,19 @@ fn get_fee_configurations( } // Calculate the implicit ExpirationEntry bytes that will be read for expirable LedgerEntries -fn calculate_expiration_entry_bytes(ledger_entries: &Vec) -> Result { - Ok(ledger_entries +fn calculate_expiration_entry_bytes(ledger_entries: &[LedgerKey]) -> u32 { + ledger_entries .iter() .map(|lk| match lk { LedgerKey::ContractData(_) | LedgerKey::ContractCode(_) => EXPIRATION_ENTRY_SIZE, _ => 0, }) - .sum()) + .sum() } +#[allow(clippy::cast_possible_truncation)] fn calculate_unmodified_ledger_entry_bytes( - ledger_entries: &Vec, + ledger_entries: &[LedgerKey], pre_storage: &LedgerStorage, include_expired: bool, ) -> Result { @@ -258,7 +263,7 @@ fn calculate_unmodified_ledger_entry_bytes( Ok(res as u32) } -fn calculate_contract_events_size_bytes(events: &Vec) -> Result { +fn calculate_contract_events_size_bytes(events: &[DiagnosticEvent]) -> Result { let mut res: u32 = 0; for e in events { if e.event.type_ != ContractEventType::Contract @@ -302,7 +307,7 @@ fn finalize_transaction_data_and_min_fee( .context("failed to obtain configuration settings from the network")?; let (non_refundable_fee, refundable_fee) = compute_transaction_resource_fee(transaction_resources, &fee_configuration); - let rent_fee = compute_rent_fee(&rent_changes, &rent_fee_configuration, current_ledger_seq); + let rent_fee = compute_rent_fee(rent_changes, &rent_fee_configuration, current_ledger_seq); let transaction_data = SorobanTransactionData { resources: soroban_resources, refundable_fee: refundable_fee + rent_fee, @@ -330,10 +335,10 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee( ) .context("cannot compute bump rent changes")?; - let expiration_bytes: u32 = calculate_expiration_entry_bytes(footprint.read_only.as_vec())?; + let expiration_bytes = calculate_expiration_entry_bytes(footprint.read_only.as_slice()); let unmodified_entry_bytes = calculate_unmodified_ledger_entry_bytes( - footprint.read_only.as_vec(), + footprint.read_only.as_slice(), ledger_storage, false, ) @@ -363,7 +368,7 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee( contract_events_size_bytes: 0, }; finalize_transaction_data_and_min_fee( - &ledger_storage, + ledger_storage, &transaction_resources, soroban_resources, &rent_changes, @@ -372,6 +377,7 @@ pub(crate) fn compute_bump_footprint_exp_transaction_data_and_min_fee( ) } +#[allow(clippy::cast_possible_truncation)] fn compute_bump_footprint_rent_changes( footprint: &LedgerFootprint, ledger_storage: &LedgerStorage, @@ -380,7 +386,7 @@ fn compute_bump_footprint_rent_changes( ) -> Result> { let mut rent_changes: Vec = Vec::with_capacity(footprint.read_only.len()); - for key in (&footprint).read_only.as_vec() { + for key in footprint.read_only.as_slice() { let unmodified_entry_and_expiration = ledger_storage .get(key, false) .with_context(|| format!("cannot find bump footprint ledger entry with key {key:?}"))?; @@ -426,7 +432,7 @@ pub(crate) fn compute_restore_footprint_transaction_data_and_min_fee( ) .context("cannot compute restore rent changes")?; - let expiration_bytes: u32 = calculate_expiration_entry_bytes(footprint.read_write.as_vec())?; + let expiration_bytes = calculate_expiration_entry_bytes(footprint.read_write.as_vec()); let write_bytes = calculate_unmodified_ledger_entry_bytes( footprint.read_write.as_vec(), ledger_storage, @@ -465,6 +471,7 @@ pub(crate) fn compute_restore_footprint_transaction_data_and_min_fee( ) } +#[allow(clippy::cast_possible_truncation)] fn compute_restore_footprint_rent_changes( footprint: &LedgerFootprint, ledger_storage: &LedgerStorage, diff --git a/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs b/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs index b000994c9..4eac57269 100644 --- a/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs +++ b/cmd/soroban-rpc/lib/preflight/src/ledger_storage.rs @@ -143,7 +143,7 @@ impl LedgerStorage { if res.xdr.is_null() { return Err(Error::NotFound); } - let v = from_c_xdr(res.clone()); + let v = from_c_xdr(res); unsafe { FreeGoXDR(res) }; Ok(v) } @@ -250,7 +250,7 @@ impl SnapshotSource for LedgerStorage { } let entry_and_expiration = ::get(self, key, false).map_err(HostError::from)?; - return Ok((entry_and_expiration.0.into(), entry_and_expiration.1)); + Ok((entry_and_expiration.0.into(), entry_and_expiration.1)) } fn has(&self, key: &Rc) -> Result { @@ -260,6 +260,6 @@ impl SnapshotSource for LedgerStorage { return Ok(false); } } - return result.map(|_| true); + result.map(|_| true) } } diff --git a/cmd/soroban-rpc/lib/preflight/src/lib.rs b/cmd/soroban-rpc/lib/preflight/src/lib.rs index 4df90fb64..18a1976a8 100644 --- a/cmd/soroban-rpc/lib/preflight/src/lib.rs +++ b/cmd/soroban-rpc/lib/preflight/src/lib.rs @@ -257,7 +257,7 @@ fn option_xdr_to_c(v: Option) -> CXDR { fn xdr_vec_to_c(v: Vec) -> CXDRVector { let c_v = v.into_iter().map(xdr_to_c).collect(); let (array, len) = vec_to_c_array(c_v); - return CXDRVector { array, len }; + CXDRVector { array, len } } fn string_to_c(str: String) -> *mut libc::c_char { diff --git a/cmd/soroban-rpc/lib/preflight/src/preflight.rs b/cmd/soroban-rpc/lib/preflight/src/preflight.rs index 6071f977f..c6c5ac0e4 100644 --- a/cmd/soroban-rpc/lib/preflight/src/preflight.rs +++ b/cmd/soroban-rpc/lib/preflight/src/preflight.rs @@ -209,7 +209,7 @@ fn host_events_to_diagnostic_events(events: &Events) -> Vec { } res } - +#[allow(clippy::cast_sign_loss)] fn get_budget_from_network_config_params(ledger_storage: &LedgerStorage) -> Result { let ConfigSettingEntry::ContractComputeV0(compute) = ledger_storage.get_configuration_setting(ConfigSettingId::ContractComputeV0)? @@ -228,10 +228,9 @@ fn get_budget_from_network_config_params(ledger_storage: &LedgerStorage) -> Resu else { bail!("unexpected config setting entry for CostParamsMemoryBytes key"); }; - let budget = Budget::try_from_configs( compute.tx_max_instructions as u64, - compute.tx_memory_limit as u64, + u64::from(compute.tx_memory_limit), cost_params_cpu, cost_params_memory, ) diff --git a/cmd/soroban-rpc/lib/preflight/src/state_expiration.rs b/cmd/soroban-rpc/lib/preflight/src/state_expiration.rs index d0c9bbff1..c5a485f1f 100644 --- a/cmd/soroban-rpc/lib/preflight/src/state_expiration.rs +++ b/cmd/soroban-rpc/lib/preflight/src/state_expiration.rs @@ -66,5 +66,5 @@ pub(crate) fn get_restored_ledger_sequence( current_ledger_seq: u32, min_persistent_entry_expiration: u32, ) -> u32 { - return current_ledger_seq + min_persistent_entry_expiration - 1; + current_ledger_seq + min_persistent_entry_expiration - 1 }