From 22def9eedfc268c8d6416135a11385f92e015792 Mon Sep 17 00:00:00 2001 From: Yufeng Zhou Date: Thu, 12 Dec 2024 20:18:24 +0000 Subject: [PATCH 1/2] flamenco: native to bpf migration sets target account rent epoch to 0 --- src/flamenco/runtime/fd_runtime.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index 3766cf7abf..33f34c0cbe 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -4116,6 +4116,9 @@ static int fd_new_target_program_account( fd_exec_slot_ctx_t * slot_ctx, const fd_pubkey_t * target_program_data_address, fd_borrowed_account_t * out_rec ) { + /* https://github.com/anza-xyz/agave/blob/v2.1.0/sdk/account/src/lib.rs#L471 */ + out_rec->meta->info.rent_epoch = 0UL; + /* https://github.com/anza-xyz/agave/blob/v2.1.0/runtime/src/bank/builtins/core_bpf_migration/mod.rs#L86-L88 */ fd_bpf_upgradeable_loader_state_t state = { .discriminant = fd_bpf_upgradeable_loader_state_enum_program, From a46e94f34e5094aae80bd416d2e5b007cada0046 Mon Sep 17 00:00:00 2001 From: Yufeng Zhou Date: Fri, 13 Dec 2024 00:51:48 +0000 Subject: [PATCH 2/2] ledger: compute account hashes on snapshot load by default --- src/app/ledger/main.c | 11 +++++++++-- src/flamenco/runtime/fd_hashes.c | 23 +++++++++++++---------- src/flamenco/runtime/tests/Local.mk | 1 + 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/app/ledger/main.c b/src/app/ledger/main.c index e931cc2530..599c9fd851 100644 --- a/src/app/ledger/main.c +++ b/src/app/ledger/main.c @@ -1340,8 +1340,8 @@ initial_setup( int argc, char ** argv, fd_ledger_args_t * args ) { ulong shred_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--shred-max", NULL, 1UL << 17 ); ulong start_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--start-slot", NULL, 0UL ); ulong end_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--end-slot", NULL, ULONG_MAX ); - uint verify_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--verify-acc-hash", NULL, 0 ); - uint check_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--check-acc-hash", NULL, 0 ); + uint verify_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--verify-acc-hash", NULL, 1 ); + uint check_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--check-acc-hash", NULL, 1 ); char const * restore = fd_env_strip_cmdline_cstr ( &argc, &argv, "--restore", NULL, NULL ); char const * restore_funk = fd_env_strip_cmdline_cstr ( &argc, &argv, "--funk-restore", NULL, NULL ); char const * restore_archive = fd_env_strip_cmdline_cstr ( &argc, &argv, "--restore-archive", NULL, NULL ); @@ -1374,6 +1374,13 @@ initial_setup( int argc, char ** argv, fd_ledger_args_t * args ) { char const * one_off_features = fd_env_strip_cmdline_cstr ( &argc, &argv, "--one-off-features", NULL, NULL ); char const * lthash = fd_env_strip_cmdline_cstr ( &argc, &argv, "--lthash", NULL, "false" ); + if( FD_UNLIKELY( !verify_acc_hash ) ) { + /* We've got full snapshots that contain all 0s for the account + hash in account meta. Running hash verify allows us to + populate the hash in account meta with real values. */ + FD_LOG_ERR(( "verify-acc-hash needs to be 1" )); + } + // TODO: Add argument validation. Make sure that we aren't including any arguments that aren't parsed for char hostname[64]; diff --git a/src/flamenco/runtime/fd_hashes.c b/src/flamenco/runtime/fd_hashes.c index a2bcdaa279..7a6f0e321b 100644 --- a/src/flamenco/runtime/fd_hashes.c +++ b/src/flamenco/runtime/fd_hashes.c @@ -326,12 +326,15 @@ fd_account_hash_task( void *tpool, iterator. Instead, we will store away the record and erase it later where appropriate. */ task_info->should_erase = 1; + /* In the exceedingly unlikely event that the account's old hash is + actually 0, this would cause the account not to be included in + the bank hash. */ if( memcmp( task_info->acc_hash->hash, acc_meta->hash, sizeof(fd_hash_t) ) != 0 ) { task_info->hash_changed = 1; } } else { uchar * acc_data = fd_account_get_data((fd_account_meta_t *) acc_meta); - fd_pubkey_t const * acc_key = fd_type_pun_const( task_info->rec->pair.key[0].uc ); + fd_pubkey_t const * acc_key = fd_funk_key_to_acc( task_info->rec->pair.key ); fd_lthash_value_t new_lthash_value; fd_lthash_zero(&new_lthash_value); fd_hash_account_current( task_info->acc_hash->hash, &new_lthash_value, acc_meta, acc_key->key, acc_data ); @@ -344,7 +347,7 @@ fd_account_hash_task( void *tpool, if( FD_LIKELY(task_info->hash_changed && ((NULL != acc_meta_parent) && (acc_meta_parent->info.lamports != 0) ) ) ) { uchar * acc_data = fd_account_get_data(acc_meta_parent); - fd_pubkey_t const * acc_key = fd_type_pun_const( task_info->rec->pair.key[0].uc ); + fd_pubkey_t const * acc_key = fd_funk_key_to_acc( task_info->rec->pair.key ); fd_lthash_value_t old_lthash_value; fd_lthash_zero(&old_lthash_value); fd_hash_t old_hash; @@ -373,7 +376,7 @@ fd_collect_modified_accounts( fd_exec_slot_ctx_t * slot_ctx, if( !fd_funk_key_is_acc( rec->pair.key ) ) continue; - fd_pubkey_t const * pubkey = fd_type_pun_const( rec->pair.key[0].uc ); + fd_pubkey_t const * pubkey = fd_funk_key_to_acc( rec->pair.key ); if (((pubkey->ul[0] == 0) & (pubkey->ul[1] == 0) & (pubkey->ul[2] == 0) & (pubkey->ul[3] == 0))) FD_LOG_WARNING(( "null pubkey (system program?) showed up as modified" )); @@ -390,7 +393,7 @@ fd_collect_modified_accounts( fd_exec_slot_ctx_t * slot_ctx, NULL != rec; rec = fd_funk_txn_next_rec( funk, rec ) ) { - fd_pubkey_t const * acc_key = fd_type_pun_const( rec->pair.key[0].uc ); + fd_pubkey_t const * acc_key = fd_funk_key_to_acc( rec->pair.key ); if( !fd_funk_key_is_acc( rec->pair.key ) ) continue; @@ -450,7 +453,7 @@ fd_update_hash_bank_tpool( fd_exec_slot_ctx_t * slot_ctx, FD_BORROWED_ACCOUNT_DECL(acc_rec); acc_rec->const_rec = task_info->rec; - fd_pubkey_t const * acc_key = fd_type_pun_const( task_info->rec->pair.key[0].uc ); + fd_pubkey_t const * acc_key = fd_funk_key_to_acc( task_info->rec->pair.key ); int err = fd_acc_mgr_modify( acc_mgr, txn, acc_key, 0, 0UL, acc_rec); if( FD_UNLIKELY( err!=FD_ACC_MGR_SUCCESS ) ) { FD_LOG_ERR(( "failed to modify account during bank hash" )); @@ -757,8 +760,8 @@ fd_accounts_sorted_subrange( fd_exec_slot_ctx_t * slot_ctx, uint range_idx, uint fd_lthash_add( &accum, &new_lthash_value ); fd_hash_t * h = (fd_hash_t *) metadata->hash; - if( (h->ul[0] | h->ul[1] | h->ul[2] | h->ul[3]) != 0 ) { - if( fd_acc_exists( metadata ) && memcmp( metadata->hash, &hash, 32 ) != 0 ) { + if( FD_LIKELY( (h->ul[0] | h->ul[1] | h->ul[2] | h->ul[3]) != 0 ) ) { + if( FD_UNLIKELY( fd_acc_exists( metadata ) && memcmp( metadata->hash, &hash, 32 ) != 0 ) ) { FD_LOG_WARNING(( "snapshot hash (%s) doesn't match calculated hash (%s)", FD_BASE58_ENC_32_ALLOCA( metadata->hash ), FD_BASE58_ENC_32_ALLOCA( &hash ) )); } } else @@ -913,10 +916,10 @@ fd_accounts_hash_inc_only( fd_exec_slot_ctx_t * slot_ctx, fd_hash_t *accounts_ha fd_hash_account_current( (uchar *) metadata->hash, NULL, metadata, rec->pair.key->uc, fd_account_get_data(metadata) ); } else if( do_hash_verify ) { uchar hash[32]; - ulong old_slot = slot_ctx->slot_bank.slot; - slot_ctx->slot_bank.slot = metadata->slot; + // ulong old_slot = slot_ctx->slot_bank.slot; + // slot_ctx->slot_bank.slot = metadata->slot; fd_hash_account_current( (uchar *) &hash, NULL, metadata, rec->pair.key->uc, fd_account_get_data(metadata) ); - slot_ctx->slot_bank.slot = old_slot; + // slot_ctx->slot_bank.slot = old_slot; if ( fd_acc_exists( metadata ) && memcmp( metadata->hash, &hash, 32 ) != 0 ) { FD_LOG_WARNING(( "snapshot hash (%s) doesn't match calculated hash (%s)", FD_BASE58_ENC_32_ALLOCA( metadata->hash ), FD_BASE58_ENC_32_ALLOCA( &hash ) )); } diff --git a/src/flamenco/runtime/tests/Local.mk b/src/flamenco/runtime/tests/Local.mk index 6e0edf0cc9..916c8816b4 100644 --- a/src/flamenco/runtime/tests/Local.mk +++ b/src/flamenco/runtime/tests/Local.mk @@ -30,3 +30,4 @@ run-runtime-test-nightly: $(OBJDIR)/bin/fd_ledger OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l mainnet-257033306 -s snapshot-257033306-EE3WdRoE4J1LTjegJMK3ZzxKZbSMQhLMaTM5Jp4SygMU.tar.zst -p 100 -y 450 -m 500000000 -e 257213306 --zst OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l mainnet-296243940 -s snapshot-296400651-HDt9Gf1YKcruvuBV4q442qV4xjHer4KZ9sZao9XQspZP.tar.zst -p 75 -y 750 -m 700000000 -e 296550651 --zst OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l devnet-340941576 -s snapshot-340924320-8j9h6EKmuZ3G93Y3Pb3FqrNdCDTGE5PKowHMY3xkXG1K.tar.zst -p 75 -y 400 -m 200000000 -e 340941580 --zst + OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l testnet-305516256 -s snapshot-305516254-C4oM7ajmCMo1aDakR8Q8FriSVpXW53jwbb3da37jm7bN.tar.zst -p 75 -y 400 -m 150000000 -e 305516292 --zst