From d5d6563260786c7a5d8afa0c30f188eb0e544eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Mei=C3=9Fner?= Date: Tue, 8 Oct 2024 14:10:13 +0000 Subject: [PATCH 1/4] Adds ABIv2 de/serialization interface. --- ledger-tool/src/program.rs | 3 ++- program-runtime/src/serialization.rs | 7 +++++++ program-test/src/lib.rs | 1 + programs/bpf_loader/benches/serialization.rs | 18 ++++++++++++------ programs/bpf_loader/src/lib.rs | 2 ++ programs/sbf/benches/bpf_loader.rs | 4 +++- 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/ledger-tool/src/program.rs b/ledger-tool/src/program.rs index f11ddf0e1e8602..208fb8ad911a3e 100644 --- a/ledger-tool/src/program.rs +++ b/ledger-tool/src/program.rs @@ -544,7 +544,8 @@ pub fn program(ledger_path: &Path, matches: &ArgMatches<'_>) { .transaction_context .get_current_instruction_context() .unwrap(), - true, // copy_account_data + true, // copy_account_data + false, // is_abi_v2 ) .unwrap(); diff --git a/program-runtime/src/serialization.rs b/program-runtime/src/serialization.rs index 7d1f9085bb5cbb..ea61faafbe0fd9 100644 --- a/program-runtime/src/serialization.rs +++ b/program-runtime/src/serialization.rs @@ -189,6 +189,7 @@ pub fn serialize_parameters( transaction_context: &TransactionContext, instruction_context: &InstructionContext, copy_account_data: bool, + is_abi_v2: bool, ) -> Result< ( AlignedMemory, @@ -252,6 +253,7 @@ pub fn deserialize_parameters( transaction_context: &TransactionContext, instruction_context: &InstructionContext, copy_account_data: bool, + is_abi_v2: bool, buffer: &[u8], accounts_metadata: &[SerializedAccountMetadata], ) -> Result<(), InstructionError> { @@ -729,6 +731,7 @@ mod tests { invoke_context.transaction_context, instruction_context, copy_account_data, + false, ); assert_eq!( serialization_result.as_ref().err(), @@ -883,6 +886,7 @@ mod tests { invoke_context.transaction_context, instruction_context, copy_account_data, + false, // is_abi_v2 ) .unwrap(); @@ -942,6 +946,7 @@ mod tests { invoke_context.transaction_context, instruction_context, copy_account_data, + false, // is_abi_v2 serialized.as_slice(), &accounts_metadata, ) @@ -974,6 +979,7 @@ mod tests { invoke_context.transaction_context, instruction_context, copy_account_data, + false, // is_abi_v2 ) .unwrap(); let mut serialized_regions = concat_regions(®ions); @@ -1012,6 +1018,7 @@ mod tests { invoke_context.transaction_context, instruction_context, copy_account_data, + false, // is_abi_v2 serialized.as_slice(), &account_lengths, ) diff --git a/program-test/src/lib.rs b/program-test/src/lib.rs index 1894117dcbf454..a4611174580a72 100644 --- a/program-test/src/lib.rs +++ b/program-test/src/lib.rs @@ -131,6 +131,7 @@ pub fn invoke_builtin_function( transaction_context, instruction_context, true, // copy_account_data // There is no VM so direct mapping can not be implemented here + false, // is_abi_v2 )?; // Deserialize data back into instruction params diff --git a/programs/bpf_loader/benches/serialization.rs b/programs/bpf_loader/benches/serialization.rs index a399400b15522a..43239803be120d 100644 --- a/programs/bpf_loader/benches/serialization.rs +++ b/programs/bpf_loader/benches/serialization.rs @@ -121,7 +121,8 @@ fn bench_serialize_unaligned(c: &mut Criterion) { c.bench_function("serialize_unaligned", |b| { b.iter(|| { - let _ = serialize_parameters(&transaction_context, instruction_context, false).unwrap(); + let _ = serialize_parameters(&transaction_context, instruction_context, false, false) + .unwrap(); }); }); } @@ -133,7 +134,8 @@ fn bench_serialize_unaligned_copy_account_data(c: &mut Criterion) { .unwrap(); c.bench_function("serialize_unaligned_copy_account_data", |b| { b.iter(|| { - let _ = serialize_parameters(&transaction_context, instruction_context, true).unwrap(); + let _ = serialize_parameters(&transaction_context, instruction_context, true, false) + .unwrap(); }); }); } @@ -146,7 +148,8 @@ fn bench_serialize_aligned(c: &mut Criterion) { c.bench_function("serialize_aligned", |b| { b.iter(|| { - let _ = serialize_parameters(&transaction_context, instruction_context, false).unwrap(); + let _ = serialize_parameters(&transaction_context, instruction_context, false, false) + .unwrap(); }); }); } @@ -159,7 +162,8 @@ fn bench_serialize_aligned_copy_account_data(c: &mut Criterion) { c.bench_function("serialize_aligned_copy_account_data", |b| { b.iter(|| { - let _ = serialize_parameters(&transaction_context, instruction_context, true).unwrap(); + let _ = serialize_parameters(&transaction_context, instruction_context, true, false) + .unwrap(); }); }); } @@ -172,7 +176,8 @@ fn bench_serialize_unaligned_max_accounts(c: &mut Criterion) { c.bench_function("serialize_unaligned_max_accounts", |b| { b.iter(|| { - let _ = serialize_parameters(&transaction_context, instruction_context, false).unwrap(); + let _ = serialize_parameters(&transaction_context, instruction_context, false, false) + .unwrap(); }); }); } @@ -185,7 +190,8 @@ fn bench_serialize_aligned_max_accounts(c: &mut Criterion) { c.bench_function("serialize_aligned_max_accounts", |b| { b.iter(|| { - let _ = serialize_parameters(&transaction_context, instruction_context, false).unwrap(); + let _ = serialize_parameters(&transaction_context, instruction_context, false, false) + .unwrap(); }); }); } diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs index af8957bbb65873..d5d67be212930a 100644 --- a/programs/bpf_loader/src/lib.rs +++ b/programs/bpf_loader/src/lib.rs @@ -1588,6 +1588,7 @@ fn execute<'a, 'b: 'a>( invoke_context.transaction_context, instruction_context, !direct_mapping, + false, )?; serialize_time.stop(); @@ -1734,6 +1735,7 @@ fn execute<'a, 'b: 'a>( .transaction_context .get_current_instruction_context()?, copy_account_data, + false, parameter_bytes, &invoke_context.get_syscall_context()?.accounts_metadata, ) diff --git a/programs/sbf/benches/bpf_loader.rs b/programs/sbf/benches/bpf_loader.rs index 3850e4f5484465..c00db0bca31034 100644 --- a/programs/sbf/benches/bpf_loader.rs +++ b/programs/sbf/benches/bpf_loader.rs @@ -253,7 +253,8 @@ fn bench_create_vm(bencher: &mut Bencher) { .transaction_context .get_current_instruction_context() .unwrap(), - !direct_mapping, // copy_account_data, + !direct_mapping, // copy_account_data + false, // is_abi_v2 ) .unwrap(); @@ -288,6 +289,7 @@ fn bench_instruction_count_tuner(_bencher: &mut Bencher) { .get_current_instruction_context() .unwrap(), !direct_mapping, // copy_account_data + false, // is_abi_v2 ) .unwrap(); From da13e9a8b4437a3884944d12e48e2c47367ecee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Mei=C3=9Fner?= Date: Tue, 25 Feb 2025 15:00:39 +0000 Subject: [PATCH 2/4] Adds vaddr parameter to push_account_data_region. --- program-runtime/src/serialization.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/program-runtime/src/serialization.rs b/program-runtime/src/serialization.rs index ea61faafbe0fd9..4807d869eda9e2 100644 --- a/program-runtime/src/serialization.rs +++ b/program-runtime/src/serialization.rs @@ -99,7 +99,8 @@ impl Serializer { } else { self.push_region(true); let vaddr = self.vaddr; - self.push_account_data_region(account)?; + self.push_account_data_region(vaddr, account)?; + self.vaddr += account.get_data().len() as u64; vaddr }; @@ -128,19 +129,17 @@ impl Serializer { fn push_account_data_region( &mut self, + vaddr: u64, account: &mut BorrowedAccount<'_>, ) -> Result<(), InstructionError> { if !account.get_data().is_empty() { let region = match account_data_region_memory_state(account) { - MemoryState::Readable => MemoryRegion::new_readonly(account.get_data(), self.vaddr), - MemoryState::Writable => { - MemoryRegion::new_writable(account.get_data_mut()?, self.vaddr) - } + MemoryState::Readable => MemoryRegion::new_readonly(account.get_data(), vaddr), + MemoryState::Writable => MemoryRegion::new_writable(account.get_data_mut()?, vaddr), MemoryState::Cow(index_in_transaction) => { - MemoryRegion::new_cow(account.get_data(), self.vaddr, index_in_transaction) + MemoryRegion::new_cow(account.get_data(), vaddr, index_in_transaction) } }; - self.vaddr += region.len; self.regions.push(region); } From 7ef0de5423f8b324adcffe800c8214bafa873a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Mei=C3=9Fner?= Date: Wed, 26 Feb 2025 11:16:13 +0000 Subject: [PATCH 3/4] Adds writable parameter to finish(). --- program-runtime/src/serialization.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/program-runtime/src/serialization.rs b/program-runtime/src/serialization.rs index 4807d869eda9e2..2f9fc155c9ffe0 100644 --- a/program-runtime/src/serialization.rs +++ b/program-runtime/src/serialization.rs @@ -164,8 +164,8 @@ impl Serializer { self.vaddr += range.len() as u64; } - fn finish(mut self) -> (AlignedMemory, Vec) { - self.push_region(true); + fn finish(mut self, writable: bool) -> (AlignedMemory, Vec) { + self.push_region(writable); debug_assert_eq!(self.region_start, self.buffer.len()); (self.buffer, self.regions) } @@ -354,7 +354,7 @@ fn serialize_parameters_unaligned( s.write_all(instruction_data); s.write_all(program_id.as_ref()); - let (mem, regions) = s.finish(); + let (mem, regions) = s.finish(true); Ok((mem, regions, accounts_metadata)) } @@ -495,7 +495,7 @@ fn serialize_parameters_aligned( s.write_all(instruction_data); s.write_all(program_id.as_ref()); - let (mem, regions) = s.finish(); + let (mem, regions) = s.finish(true); Ok((mem, regions, accounts_metadata)) } From 2cac7b3e9987b24912b68868e6019518d29a12a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20Mei=C3=9Fner?= Date: Tue, 25 Feb 2025 17:34:46 +0000 Subject: [PATCH 4/4] WIP: Adds serialize_parameters_v2(). --- program-runtime/src/serialization.rs | 234 ++++++++++++++++++++++----- 1 file changed, 190 insertions(+), 44 deletions(-) diff --git a/program-runtime/src/serialization.rs b/program-runtime/src/serialization.rs index 2f9fc155c9ffe0..007a25cacfdd38 100644 --- a/program-runtime/src/serialization.rs +++ b/program-runtime/src/serialization.rs @@ -7,7 +7,7 @@ use { solana_pubkey::Pubkey, solana_sbpf::{ aligned_memory::{AlignedMemory, Pod}, - ebpf::{HOST_ALIGN, MM_INPUT_START}, + ebpf::{HOST_ALIGN, MM_INPUT_START, MM_REGION_SIZE}, memory_region::{MemoryRegion, MemoryState}, }, solana_sdk_ids::bpf_loader_deprecated, @@ -202,49 +202,54 @@ pub fn serialize_parameters( return Err(InstructionError::MaxAccountsExceeded); } - let (program_id, is_loader_deprecated) = { - let program_account = - instruction_context.try_borrow_last_program_account(transaction_context)?; - ( - *program_account.get_key(), - *program_account.get_owner() == bpf_loader_deprecated::id(), - ) - }; - - let accounts = (0..instruction_context.get_number_of_instruction_accounts()) - .map(|instruction_account_index| { - if let Some(index) = instruction_context - .is_instruction_account_duplicate(instruction_account_index) - .unwrap() - { - SerializeAccount::Duplicate(index) - } else { - let account = instruction_context - .try_borrow_instruction_account(transaction_context, instruction_account_index) - .unwrap(); - SerializeAccount::Account(instruction_account_index, account) - } - }) - // fun fact: jemalloc is good at caching tiny allocations like this one, - // so collecting here is actually faster than passing the iterator - // around, since the iterator does the work to produce its items each - // time it's iterated on. - .collect::>(); - - if is_loader_deprecated { - serialize_parameters_unaligned( - accounts, - instruction_context.get_instruction_data(), - &program_id, - copy_account_data, - ) + if is_abi_v2 { + abiv2_serialize_instruction(transaction_context, instruction_context) } else { - serialize_parameters_aligned( - accounts, - instruction_context.get_instruction_data(), - &program_id, - copy_account_data, - ) + let (program_id, is_loader_deprecated) = { + let program_account = + instruction_context.try_borrow_last_program_account(transaction_context)?; + ( + *program_account.get_key(), + *program_account.get_owner() == bpf_loader_deprecated::id(), + ) + }; + let accounts = (0..instruction_context.get_number_of_instruction_accounts()) + .map(|instruction_account_index| { + if let Some(index) = instruction_context + .is_instruction_account_duplicate(instruction_account_index) + .unwrap() + { + SerializeAccount::Duplicate(index) + } else { + let account = instruction_context + .try_borrow_instruction_account( + transaction_context, + instruction_account_index, + ) + .unwrap(); + SerializeAccount::Account(instruction_account_index, account) + } + }) + // fun fact: jemalloc is good at caching tiny allocations like this one, + // so collecting here is actually faster than passing the iterator + // around, since the iterator does the work to produce its items each + // time it's iterated on. + .collect::>(); + if is_loader_deprecated { + serialize_parameters_unaligned( + accounts, + instruction_context.get_instruction_data(), + &program_id, + copy_account_data, + ) + } else { + serialize_parameters_aligned( + accounts, + instruction_context.get_instruction_data(), + &program_id, + copy_account_data, + ) + } } } @@ -261,7 +266,9 @@ pub fn deserialize_parameters( .get_owner() == bpf_loader_deprecated::id(); let account_lengths = accounts_metadata.iter().map(|a| a.original_data_len); - if is_loader_deprecated { + if is_abi_v2 { + Ok(()) + } else if is_loader_deprecated { deserialize_parameters_unaligned( transaction_context, instruction_context, @@ -606,6 +613,145 @@ fn deserialize_parameters_aligned>( Ok(()) } +const TRANACTION_HEADER_VM_ADDRESS: u64 = MM_INPUT_START; +const SCRATCHPAD_DATA_VM_ADDRESS: u64 = MM_INPUT_START + MM_REGION_SIZE; +const INSTRUCTION_HEADER_VM_ADDRESS: u64 = MM_INPUT_START + MM_REGION_SIZE * 2; +const INSTRUCTION_DATA_VM_ADDRESS: u64 = MM_INPUT_START + MM_REGION_SIZE * 3; +const ACCOUNT_DATA_VM_ADDRESS: u64 = MM_INPUT_START + MM_REGION_SIZE * 4; + +pub(crate) fn abiv2_serialize_transaction( + transaction_context: &TransactionContext, +) -> (AlignedMemory, MemoryRegion) { + let size = 56 // size_of::<>() + + transaction_context.get_number_of_accounts() as usize * 88; // size_of::<>() + let mut s = Serializer::new(size, TRANACTION_HEADER_VM_ADDRESS, true, false); + // s.write::(0x76494241u32.to_le()); + // s.write::(0x00000002u32.to_le()); + let scratchpad = transaction_context.get_return_data(); + s.write_all(scratchpad.0.as_ref()); + s.write::(SCRATCHPAD_DATA_VM_ADDRESS.to_le()); + s.write::((scratchpad.1.len() as u64).to_le()); + s.write::((transaction_context.get_number_of_accounts() as u64).to_le()); + for index_in_transaction in 0..transaction_context.get_number_of_accounts() { + use solana_account::ReadableAccount; + let transaction_account = transaction_context + .get_account_at_index(index_in_transaction) + .unwrap() + .borrow(); + let _vm_key_addr = s.write_all( + transaction_context + .get_key_of_account_at_index(index_in_transaction) + .unwrap() + .as_ref(), + ); + let _vm_owner_addr = s.write_all(transaction_account.owner().as_ref()); + let _vm_lamports_addr = s.write::(transaction_account.lamports().to_le()); + let vm_data_addr = ACCOUNT_DATA_VM_ADDRESS + MM_REGION_SIZE * index_in_transaction as u64; + s.write::(vm_data_addr.to_le()); + s.write::((transaction_account.data().len() as u64).to_le()); + // s.write::((transaction_account.capacity() as u64).to_le()); + } + let (mem, regions) = s.finish(false); + (mem, regions.into_iter().next().unwrap()) +} + +fn abiv2_serialize_instruction( + transaction_context: &TransactionContext, + instruction_context: &InstructionContext, +) -> Result< + ( + AlignedMemory, + Vec, + Vec, + ), + InstructionError, +> { + /*let number_of_unique_instruction_accounts = accounts.iter().fold(0, |accumulator, account| { + if matches!(account, SerializeAccount::Account(_, _)) { + accumulator + 1 + } else { + accumulator + } + });*/ + + let size = 20 // size_of::<>() + + instruction_context.get_number_of_instruction_accounts() as usize * 4; // size_of::<>(); + let mut s = Serializer::new(size, INSTRUCTION_HEADER_VM_ADDRESS, true, false); + let instruction_data = instruction_context.get_instruction_data(); + s.write::(INSTRUCTION_DATA_VM_ADDRESS.to_le()); + s.write::((instruction_data.len() as u64).to_le()); + let program_account_index_in_transaction = instruction_context + .get_index_of_program_account_in_transaction( + instruction_context + .get_number_of_program_accounts() + .saturating_sub(1), + ) + .unwrap(); + s.write::(program_account_index_in_transaction.to_le()); + s.write::( + instruction_context + .get_number_of_instruction_accounts() + .to_le(), + ); + for index_in_instruction in 0..instruction_context.get_number_of_instruction_accounts() { + let mut borrowed_account = instruction_context + .try_borrow_instruction_account(transaction_context, index_in_instruction) + .unwrap(); + let index_in_transaction = borrowed_account.get_index_in_transaction(); + s.write::(index_in_transaction.to_le()); + let mut flags = 0u16; + if borrowed_account.is_signer() { + flags |= 1 << 0; + } + if borrowed_account.is_writable() { + flags |= 1 << 1; + } + s.write::(flags.to_le()); + if instruction_context + .is_instruction_account_duplicate(index_in_instruction) + .unwrap() + .is_none() + { + let vm_data_addr = + ACCOUNT_DATA_VM_ADDRESS + MM_REGION_SIZE * index_in_transaction as u64; + s.push_account_data_region(vm_data_addr, &mut borrowed_account)?; + } + } + let (mem, mut regions) = s.finish(false); + + /*let mut index: IndexOfAccount = 0; + let mut deduplicated_account_indices = Vec::with_capacity(accounts.len()); + for account in accounts.iter() { + match account { + SerializeAccount::Account(instruction_account_index, _) => { + deduplicated_account_indices.push(instruction_account_index - index); + } + SerializeAccount::Duplicate(position) => { + deduplicated_account_indices.push( + *deduplicated_account_indices + .get(*position as usize) + .unwrap(), + ); + index += 1; + } + } + } + for deduplicated_account_index in deduplicated_account_indices { + s.write::(deduplicated_account_index.to_le()); + }*/ + + let scratchpad = transaction_context.get_return_data(); + regions.push(MemoryRegion::new_readonly( + scratchpad.1, + SCRATCHPAD_DATA_VM_ADDRESS, + )); + regions.push(MemoryRegion::new_readonly( + instruction_data, + INSTRUCTION_DATA_VM_ADDRESS, + )); + Ok((mem, regions, Vec::default())) +} + pub fn account_data_region_memory_state(account: &BorrowedAccount<'_>) -> MemoryState { if account.can_data_be_changed().is_ok() { if account.is_shared() {