From 1a6cd7f43f5a862efb50f5af210d5634181b58bf Mon Sep 17 00:00:00 2001 From: Dana Date: Sat, 9 Dec 2023 20:38:35 -0700 Subject: [PATCH] Step Finance Patch Merge pull request #8 from step-finance/sf-3750-explicit-data-includes SF-3750: Explicit Data Includes --- .circleci/config.yml | 25 +++++++ _step-finance/Dockerfile | 14 ++++ _step-finance/Dockerfile.dockerignore | 2 + core/src/banking_stage/committer.rs | 10 ++- core/src/banking_stage/consumer.rs | 4 +- ledger/src/blockstore_processor.rs | 11 ++- rpc-client/src/mock_sender.rs | 2 + rpc/src/transaction_status_service.rs | 13 +++- runtime/src/bank.rs | 96 +++++++++++++++++++++++++-- runtime/src/bank/tests.rs | 2 +- runtime/src/lib.rs | 1 + runtime/src/program_inclusions.rs | 73 ++++++++++++++++++++ runtime/src/runtime_config.rs | 5 ++ storage-bigtable/src/lib.rs | 2 + storage-proto/src/convert.rs | 3 + storage-proto/src/lib.rs | 3 + test-validator/src/lib.rs | 8 ++- transaction-status/src/lib.rs | 52 ++++++++++++++- validator/src/main.rs | 6 ++ 19 files changed, 314 insertions(+), 18 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 _step-finance/Dockerfile create mode 100644 _step-finance/Dockerfile.dockerignore create mode 100644 runtime/src/program_inclusions.rs diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..f4bc0528e1 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,25 @@ +version: 2.1 + +orbs: + aws-ecr: circleci/aws-ecr@8.1.3 + +executors: + custom: + machine: + image: ubuntu-2204:2022.04.2 + resource_class: xlarge + +workflows: + run-pipeline: + jobs: + - aws-ecr/build-and-push-image: + executor: custom + dockerfile: _step-finance/Dockerfile + repo: step-solana + region: eu-north-1 + tag: ${CIRCLE_BRANCH},${CIRCLE_BRANCH}-<< pipeline.number >>,${CIRCLE_BRANCH}-<< pipeline.number >>-${CIRCLE_SHA1} + context: aws-account + filters: + branches: + only: + - step-release diff --git a/_step-finance/Dockerfile b/_step-finance/Dockerfile new file mode 100644 index 0000000000..98d870aae5 --- /dev/null +++ b/_step-finance/Dockerfile @@ -0,0 +1,14 @@ +FROM rust:1.69-bullseye as builder +RUN apt-get update +RUN apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang cmake make libprotobuf-dev protobuf-compiler +WORKDIR /solana-build +COPY . . +RUN cargo build --release + +FROM debian:bullseye-20230522 as runtime +RUN apt-get update +RUN apt-get install -y bzip2 +COPY --from=builder --chmod=700 /solana-build/target/release /usr/local/bin + +WORKDIR /solana +ENTRYPOINT ["solana-test-validator"] diff --git a/_step-finance/Dockerfile.dockerignore b/_step-finance/Dockerfile.dockerignore new file mode 100644 index 0000000000..fdab7b8560 --- /dev/null +++ b/_step-finance/Dockerfile.dockerignore @@ -0,0 +1,2 @@ +target/ +_step-finance/ diff --git a/core/src/banking_stage/committer.rs b/core/src/banking_stage/committer.rs index 4b057e4628..51e461dbcb 100644 --- a/core/src/banking_stage/committer.rs +++ b/core/src/banking_stage/committer.rs @@ -8,11 +8,12 @@ use { solana_runtime::{ accounts::TransactionLoadResult, bank::{ - Bank, CommitTransactionCounts, TransactionBalancesSet, TransactionExecutionResult, - TransactionResults, + Bank, CommitTransactionCounts, TransactionBalancesSet, TransactionDatumSet, + TransactionExecutionResult, TransactionResults, }, bank_utils, prioritization_fee_cache::PrioritizationFeeCache, + program_inclusions::PreOrPostDatum, transaction_batch::TransactionBatch, vote_sender_types::ReplayVoteSender, }, @@ -32,6 +33,7 @@ pub enum CommitTransactionDetails { #[derive(Default)] pub(super) struct PreBalanceInfo { pub native: Vec>, + pub datum: Vec>>>, pub token: Vec>, pub mint_decimals: HashMap, } @@ -141,7 +143,8 @@ impl Committer { ) { if let Some(transaction_status_sender) = &self.transaction_status_sender { let txs = batch.sanitized_transactions().to_vec(); - let post_balances = bank.collect_balances(batch); + let (post_balances, post_datum) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum); let post_token_balances = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals); let mut transaction_index = starting_transaction_index.unwrap_or_default(); @@ -166,6 +169,7 @@ impl Committer { std::mem::take(&mut pre_balance_info.native), post_balances, ), + TransactionDatumSet::new(std::mem::take(&mut pre_balance_info.datum), post_datum), TransactionTokenBalancesSet::new( std::mem::take(&mut pre_balance_info.token), post_token_balances, diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index 42989d8b8e..7b408ab54d 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -21,6 +21,7 @@ use { solana_program_runtime::timings::ExecuteTimings, solana_runtime::{ bank::{Bank, LoadAndExecuteTransactionsOutput, TransactionCheckResult}, + program_inclusions::PreOrPostDatum, transaction_batch::TransactionBatch, transaction_error_metrics::TransactionErrorMetrics, }, @@ -537,7 +538,8 @@ impl Consumer { // If the extra meta-data services are enabled for RPC, collect the // pre-balances for native and token programs. if transaction_status_sender_enabled { - pre_balance_info.native = bank.collect_balances(batch); + (pre_balance_info.native, pre_balance_info.datum) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum); pre_balance_info.token = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals) } diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 42588b3a9d..c81528c19d 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -28,8 +28,8 @@ use { accounts_index::AccountSecondaryIndexes, accounts_update_notifier_interface::AccountsUpdateNotifier, bank::{ - Bank, TransactionBalancesSet, TransactionExecutionDetails, TransactionExecutionResult, - TransactionResults, + Bank, TransactionBalancesSet, TransactionDatumSet, TransactionExecutionDetails, + TransactionExecutionResult, TransactionResults, }, bank_forks::BankForks, bank_utils, @@ -155,7 +155,7 @@ fn execute_batch( vec![] }; - let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions( + let (tx_results, balances, datum) = batch.bank().load_execute_and_commit_transactions( batch, MAX_PROCESSING_AGE, transaction_status_sender.is_some(), @@ -201,6 +201,7 @@ fn execute_batch( transactions, execution_results, balances, + datum, token_balances, rent_debits, transaction_indexes.to_vec(), @@ -1744,6 +1745,7 @@ pub struct TransactionStatusBatch { pub transactions: Vec, pub execution_results: Vec>, pub balances: TransactionBalancesSet, + pub datum: TransactionDatumSet, pub token_balances: TransactionTokenBalancesSet, pub rent_debits: Vec, pub transaction_indexes: Vec, @@ -1761,6 +1763,7 @@ impl TransactionStatusSender { transactions: Vec, execution_results: Vec, balances: TransactionBalancesSet, + datum: TransactionDatumSet, token_balances: TransactionTokenBalancesSet, rent_debits: Vec, transaction_indexes: Vec, @@ -1780,6 +1783,7 @@ impl TransactionStatusSender { }) .collect(), balances, + datum, token_balances, rent_debits, transaction_indexes, @@ -3863,6 +3867,7 @@ pub mod tests { .. }, _balances, + _datums, ) = batch.bank().load_execute_and_commit_transactions( &batch, MAX_PROCESSING_AGE, diff --git a/rpc-client/src/mock_sender.rs b/rpc-client/src/mock_sender.rs index 654f45d029..d7514be2e7 100644 --- a/rpc-client/src/mock_sender.rs +++ b/rpc-client/src/mock_sender.rs @@ -226,6 +226,8 @@ impl RpcSender for MockSender { fee: 0, pre_balances: vec![499999999999999950, 50, 1], post_balances: vec![499999999999999950, 50, 1], + pre_datum: None, + post_datum: None, inner_instructions: OptionSerializer::None, log_messages: OptionSerializer::None, pre_token_balances: OptionSerializer::None, diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index e47673123e..18765ef3cb 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -70,6 +70,7 @@ impl TransactionStatusService { bank, transactions, execution_results, + datum, balances, token_balances, rent_debits, @@ -81,6 +82,8 @@ impl TransactionStatusService { execution_result, pre_balances, post_balances, + pre_datum, + post_datum, pre_token_balances, post_token_balances, rent_debits, @@ -90,6 +93,8 @@ impl TransactionStatusService { execution_results, balances.pre_balances, balances.post_balances, + datum.pre_datum, + datum.post_datum, token_balances.pre_token_balances, token_balances.post_token_balances, rent_debits, @@ -159,6 +164,8 @@ impl TransactionStatusService { fee, pre_balances, post_balances, + pre_datum: Some(pre_datum), + post_datum: Some(post_datum), inner_instructions, log_messages, pre_token_balances, @@ -228,7 +235,7 @@ pub(crate) mod tests { solana_account_decoder::parse_token::token_amount_to_ui_amount, solana_ledger::{genesis_utils::create_genesis_config, get_tmp_ledger_path}, solana_runtime::{ - bank::{Bank, NonceFull, NoncePartial, TransactionBalancesSet}, + bank::{Bank, NonceFull, NoncePartial, TransactionBalancesSet, TransactionDatumSet}, rent_debits::RentDebits, }, solana_sdk::{ @@ -424,6 +431,10 @@ pub(crate) mod tests { let transaction_index: usize = bank.transaction_count().try_into().unwrap(); let transaction_status_batch = TransactionStatusBatch { bank, + datum: TransactionDatumSet { + post_datum: vec![vec![Some(vec![0x69])]], + pre_datum: vec![vec![Some(vec![0x04, 0x20])]], + }, transactions: vec![transaction], execution_results: vec![transaction_result], balances, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 3cc991c952..5da4c98874 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -63,6 +63,7 @@ use { inline_spl_token, message_processor::MessageProcessor, partitioned_rewards::PartitionedEpochRewardsConfig, + program_inclusions::{PreOrPostDatum, ProgramDatumInclusions}, rent_collector::{CollectedInfo, RentCollector}, rent_debits::RentDebits, runtime_config::RuntimeConfig, @@ -423,6 +424,22 @@ impl TransactionBalancesSet { } pub type TransactionBalances = Vec>; +pub struct TransactionDatumSet { + pub pre_datum: TransactionDatum, + pub post_datum: TransactionDatum, +} + +impl TransactionDatumSet { + pub fn new(pre_datum: TransactionDatum, post_datum: TransactionDatum) -> Self { + assert_eq!(pre_datum.len(), post_datum.len()); + Self { + pre_datum, + post_datum, + } + } +} +pub type TransactionDatum = Vec>>>; + /// An ordered list of compiled instructions that were invoked during a /// transaction instruction pub type InnerInstructions = Vec; @@ -800,6 +817,7 @@ impl PartialEq for Bank { loaded_programs_cache: _, check_program_modification_slot: _, epoch_reward_status: _, + program_datum_inclusions: _, // Ignore new fields explicitly if they do not impact PartialEq. // Adding ".." will remove compile-time checks that if a new field // is added to the struct, this PartialEq is accordingly updated. @@ -1072,6 +1090,8 @@ pub struct Bank { bank_freeze_or_destruction_incremented: AtomicBool, epoch_reward_status: EpochRewardStatus, + /// programs that will have their account data sent to geyser + pub program_datum_inclusions: Arc, } struct VoteWithStakeDelegations { @@ -1293,6 +1313,7 @@ impl Bank { loaded_programs_cache: Arc::>::default(), check_program_modification_slot: false, epoch_reward_status: EpochRewardStatus::default(), + program_datum_inclusions: Arc::::default(), }; bank.bank_created(); @@ -1364,6 +1385,7 @@ impl Bank { exit, ); let mut bank = Self::default_with_accounts(accounts); + bank.program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); bank.ancestors = Ancestors::from(vec![bank.slot()]); bank.transaction_debug_keys = debug_keys; bank.runtime_config = runtime_config; @@ -1606,6 +1628,7 @@ impl Bank { loaded_programs_cache: parent.loaded_programs_cache.clone(), check_program_modification_slot: false, epoch_reward_status: parent.epoch_reward_status.clone(), + program_datum_inclusions: parent.program_datum_inclusions.clone(), }; let (_, ancestors_time_us) = measure_us!({ @@ -1886,6 +1909,7 @@ impl Bank { T::default() } let feature_set = new(); + let program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); let mut bank = Self { incremental_snapshot_persistence: fields.incremental_snapshot_persistence, bank_freeze_or_destruction_incremented: AtomicBool::default(), @@ -1948,6 +1972,7 @@ impl Bank { loaded_programs_cache: Arc::>::default(), check_program_modification_slot: false, epoch_reward_status: EpochRewardStatus::default(), + program_datum_inclusions, }; bank.bank_created(); @@ -3924,6 +3949,27 @@ impl Bank { balances } + pub fn collect_balances_and_datum( + &self, + batch: &TransactionBatch, + pre_or_post: PreOrPostDatum, + ) -> (TransactionBalances, TransactionDatum) { + let mut balances: TransactionBalances = vec![]; + let mut datum: TransactionDatum = vec![]; + for transaction in batch.sanitized_transactions() { + let mut transaction_balances: Vec = vec![]; + let mut transaction_datum: Vec>> = vec![]; + for account_key in transaction.message().account_keys().iter() { + let (balance, data) = self.get_balance_and_data(account_key, &pre_or_post); + transaction_balances.push(balance); + transaction_datum.push(data); + } + balances.push(transaction_balances); + datum.push(transaction_datum); + } + (balances, datum) + } + fn program_modification_slot(&self, pubkey: &Pubkey) -> Result { let program = self .get_account_with_fixed_root(pubkey) @@ -5817,11 +5863,15 @@ impl Bank { enable_return_data_recording: bool, timings: &mut ExecuteTimings, log_messages_bytes_limit: Option, - ) -> (TransactionResults, TransactionBalancesSet) { - let pre_balances = if collect_balances { - self.collect_balances(batch) + ) -> ( + TransactionResults, + TransactionBalancesSet, + TransactionDatumSet, + ) { + let (pre_balances, pre_datum) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum) } else { - vec![] + (vec![], vec![]) }; let LoadAndExecuteTransactionsOutput { @@ -5861,14 +5911,15 @@ impl Bank { }, timings, ); - let post_balances = if collect_balances { - self.collect_balances(batch) + let (post_balances, post_datum) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum) } else { - vec![] + (vec![], vec![]) }; ( results, TransactionBalancesSet::new(pre_balances, post_balances), + TransactionDatumSet::new(pre_datum, post_datum), ) } @@ -5987,6 +6038,23 @@ impl Bank { pub fn read_balance(account: &AccountSharedData) -> u64 { account.lamports() } + + pub fn read_data( + &self, + account: &AccountSharedData, + pre_or_post: &PreOrPostDatum, + ) -> Option> { + let data = account.data(); + let owner = account.owner(); + let inclusion = self.program_datum_inclusions.get(owner)?; + let include_data = inclusion.can_include_datum(pre_or_post, &data); + + if !include_data { + None + } else { + Some(data.to_vec()) + } + } /// Each program would need to be able to introspect its own state /// this is hard-coded to the Budget language pub fn get_balance(&self, pubkey: &Pubkey) -> u64 { @@ -5994,6 +6062,20 @@ impl Bank { .map(|x| Self::read_balance(&x)) .unwrap_or(0) } + pub fn get_balance_and_data( + &self, + pubkey: &Pubkey, + pre_or_post: &PreOrPostDatum, + ) -> (u64, Option>) { + self.get_account(pubkey) + .map(|x| { + ( + Self::read_balance(&x), + Self::read_data(self, &x, pre_or_post), + ) + }) + .unwrap_or((0, None)) + } /// Compute all the parents of the bank in order pub fn parents(&self) -> Vec> { diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 7595783fc1..7f2194501f 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -6136,7 +6136,7 @@ fn test_pre_post_transaction_balances() { let txs = vec![tx0, tx1, tx2]; let lock_result = bank0.prepare_batch_for_tests(txs); - let (transaction_results, transaction_balances_set) = bank0 + let (transaction_results, transaction_balances_set, _) = bank0 .load_execute_and_commit_transactions( &lock_result, MAX_PROCESSING_AGE, diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 06259db705..6a98616a1a 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -52,6 +52,7 @@ pub mod non_circulating_supply; pub mod partitioned_rewards; pub mod prioritization_fee; pub mod prioritization_fee_cache; +pub mod program_inclusions; mod pubkey_bins; mod read_only_accounts_cache; pub mod rent_collector; diff --git a/runtime/src/program_inclusions.rs b/runtime/src/program_inclusions.rs new file mode 100644 index 0000000000..e4c6c21bde --- /dev/null +++ b/runtime/src/program_inclusions.rs @@ -0,0 +1,73 @@ +use std::{collections::HashMap, fs, path::PathBuf}; + +use solana_sdk::pubkey::Pubkey; + +pub enum PreOrPostDatum { + PreDatum, + PostDatum, +} + +pub type ProgramDatumInclusions = HashMap; +pub type InclusionsFromConfig = HashMap; + +#[derive(Deserialize, Debug, Clone, Default)] +#[serde(rename_all = "camelCase")] +pub struct DatumInclusion { + #[serde(default)] + pub pre: bool, + #[serde(default)] + pub post: bool, + #[serde(default)] + pub length_exclusions: Vec, +} + +impl DatumInclusion { + pub fn can_include_datum(&self, pre_or_post: &PreOrPostDatum, data: &[u8]) -> bool { + let allow_pre_post = match pre_or_post { + PreOrPostDatum::PreDatum => self.pre, + PreOrPostDatum::PostDatum => self.post, + }; + + if !allow_pre_post { + return false; + } + + !self.length_exclusions.contains(&data.len()) + } +} + +#[derive(serde::Deserialize)] +struct GeyserConfigFileWithInclusions { + #[serde(rename = "datumProgramInclusions")] + pub datum_program_inclusions: InclusionsFromConfig, +} + +pub fn load_datum_program_inclusions(paths: &Option>) -> ProgramDatumInclusions { + let mut datum_program_inclusions: ProgramDatumInclusions = HashMap::new(); + if let Some(paths) = paths { + for path in paths { + let file = fs::read(path); + if !file.is_ok() { + eprintln!("Unable to read JSON file: {:?} Skipping...", path); + continue; + } + let json = serde_json::from_slice::(&file.unwrap()); + if let Err(e) = json { + eprintln!("Unable to parse JSON file {:?}: {:?} Skipping...", path, e); + continue; + } + let inclusions_map = json.unwrap().datum_program_inclusions; + for (pubkey, inc) in inclusions_map.iter() { + let pk_parsed = pubkey.parse::().expect( + format!( + "Bad pubkey provided to datumProgramInclusions in geyser config file {:?}", + path + ) + .as_str(), + ); + datum_program_inclusions.insert(pk_parsed, inc.clone()); + } + } + } + datum_program_inclusions +} diff --git a/runtime/src/runtime_config.rs b/runtime/src/runtime_config.rs index 2439dd85c2..a2bdbff936 100644 --- a/runtime/src/runtime_config.rs +++ b/runtime/src/runtime_config.rs @@ -1,9 +1,14 @@ +use std::sync::Arc; + use solana_program_runtime::compute_budget::ComputeBudget; +use crate::program_inclusions::ProgramDatumInclusions; + /// Encapsulates flags that can be used to tweak the runtime behavior. #[derive(AbiExample, Debug, Default, Clone)] pub struct RuntimeConfig { pub compute_budget: Option, pub log_messages_bytes_limit: Option, pub transaction_account_lock_limit: Option, + pub program_datum_inclusions: Arc, } diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index aa891fe0cd..634544ff34 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -243,6 +243,8 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/storage-proto/src/convert.rs b/storage-proto/src/convert.rs index b166136d30..d0d79c5d2a 100644 --- a/storage-proto/src/convert.rs +++ b/storage-proto/src/convert.rs @@ -369,6 +369,7 @@ impl From for generated::TransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; let err = match status { Ok(()) => None, @@ -522,6 +523,8 @@ impl TryFrom for TransactionStatusMeta { fee, pre_balances, post_balances, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances, diff --git a/storage-proto/src/lib.rs b/storage-proto/src/lib.rs index 0832690f1e..b706986a53 100644 --- a/storage-proto/src/lib.rs +++ b/storage-proto/src/lib.rs @@ -200,6 +200,8 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances: pre_token_balances @@ -231,6 +233,7 @@ impl TryFrom for StoredTransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; if !loaded_addresses.is_empty() { diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index ef3faca4e9..ff10dc0358 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -30,7 +30,8 @@ use { solana_runtime::{ accounts_db::AccountsDbConfig, accounts_index::AccountsIndexConfig, bank_forks::BankForks, genesis_utils::create_genesis_config_with_leader_ex, - hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, runtime_config::RuntimeConfig, + hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + program_inclusions::load_datum_program_inclusions, runtime_config::RuntimeConfig, snapshot_config::SnapshotConfig, snapshot_utils::create_accounts_run_and_snapshot_dirs, }, solana_sdk::{ @@ -905,6 +906,10 @@ impl TestValidator { ..AccountsDbConfig::default() }); + let program_datum_inclusions = Arc::new(load_datum_program_inclusions( + &config.geyser_plugin_config_files, + )); + let runtime_config = RuntimeConfig { compute_budget: config .compute_unit_limit @@ -914,6 +919,7 @@ impl TestValidator { }), log_messages_bytes_limit: config.log_messages_bytes_limit, transaction_account_lock_limit: config.transaction_account_lock_limit, + program_datum_inclusions, }; let mut validator_config = ValidatorConfig { diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 7c9877a26b..776dca3f42 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -331,6 +331,11 @@ pub struct TransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>>, + pub post_datum: Option>>>, pub inner_instructions: Option>, pub log_messages: Option>, pub pre_token_balances: Option>, @@ -348,6 +353,8 @@ impl Default for TransactionStatusMeta { fee: 0, pre_balances: vec![], post_balances: vec![], + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, @@ -369,6 +376,11 @@ pub struct UiTransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>, + pub post_datum: Option>>, #[serde( default = "OptionSerializer::none", skip_serializing_if = "OptionSerializer::should_skip" @@ -445,6 +457,18 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| { @@ -478,6 +502,18 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: OptionSerializer::Skip, log_messages: OptionSerializer::Skip, pre_token_balances: meta @@ -508,6 +544,18 @@ impl From for UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| ixs.into_iter().map(Into::into).collect()) @@ -1015,7 +1063,7 @@ impl ConfirmedTransactionWithStatusMeta { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EncodedConfirmedTransactionWithStatusMeta { pub slot: Slot, @@ -1486,6 +1534,8 @@ mod test { fee: 1234, pre_balances: vec![1, 2, 3], post_balances: vec![4, 5, 6], + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/validator/src/main.rs b/validator/src/main.rs index f888adae28..e8a7510210 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -41,6 +41,7 @@ use { AccountsIndexConfig, IndexLimitMb, }, partitioned_rewards::TestPartitionedEpochRewards, + program_inclusions::load_datum_program_inclusions, runtime_config::RuntimeConfig, snapshot_config::{SnapshotConfig, SnapshotUsage}, snapshot_utils::{ @@ -1211,6 +1212,10 @@ pub fn main() { }; let starting_with_geyser_plugins: bool = on_start_geyser_plugin_config_files.is_some(); + let program_datum_inclusions = Arc::new(load_datum_program_inclusions( + &on_start_geyser_plugin_config_files, + )); + let rpc_bigtable_config = if matches.is_present("enable_rpc_bigtable_ledger_storage") || matches.is_present("enable_bigtable_ledger_upload") { @@ -1385,6 +1390,7 @@ pub fn main() { accounts_shrink_ratio, runtime_config: RuntimeConfig { log_messages_bytes_limit: value_of(&matches, "log_messages_bytes_limit"), + program_datum_inclusions, ..RuntimeConfig::default() }, staked_nodes_overrides: staked_nodes_overrides.clone(),