Skip to content

Commit

Permalink
ledger: compute account hashes on snapshot load by default
Browse files Browse the repository at this point in the history
  • Loading branch information
@yufeng-jump @kbhargava-jump
yufeng-jump authored and kbhargava-jump committed Dec 13, 2024
1 parent 94cf194 commit 7152b42
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 12 deletions.
11 changes: 9 additions & 2 deletions src/app/ledger/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1340,8 +1340,8 @@ initial_setup( int argc, char ** argv, fd_ledger_args_t * args ) {
ulong shred_max = fd_env_strip_cmdline_ulong( &argc, &argv, "--shred-max", NULL, 1UL << 17 );
ulong start_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--start-slot", NULL, 0UL );
ulong end_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--end-slot", NULL, ULONG_MAX );
uint verify_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--verify-acc-hash", NULL, 0 );
uint check_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--check-acc-hash", NULL, 0 );
uint verify_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--verify-acc-hash", NULL, 1 );
uint check_acc_hash = fd_env_strip_cmdline_uint ( &argc, &argv, "--check-acc-hash", NULL, 1 );
char const * restore = fd_env_strip_cmdline_cstr ( &argc, &argv, "--restore", NULL, NULL );
char const * restore_funk = fd_env_strip_cmdline_cstr ( &argc, &argv, "--funk-restore", NULL, NULL );
char const * restore_archive = fd_env_strip_cmdline_cstr ( &argc, &argv, "--restore-archive", NULL, NULL );
Expand Down Expand Up @@ -1374,6 +1374,13 @@ initial_setup( int argc, char ** argv, fd_ledger_args_t * args ) {
char const * one_off_features = fd_env_strip_cmdline_cstr ( &argc, &argv, "--one-off-features", NULL, NULL );
char const * lthash = fd_env_strip_cmdline_cstr ( &argc, &argv, "--lthash", NULL, "false" );

if( FD_UNLIKELY( !verify_acc_hash ) ) {
/* We've got full snapshots that contain all 0s for the account
hash in account meta. Running hash verify allows us to
populate the hash in account meta with real values. */
FD_LOG_ERR(( "verify-acc-hash needs to be 1" ));
}

// TODO: Add argument validation. Make sure that we aren't including any arguments that aren't parsed for

char hostname[64];
Expand Down
23 changes: 13 additions & 10 deletions src/flamenco/runtime/fd_hashes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,12 +326,15 @@ fd_account_hash_task( void *tpool,
iterator. Instead, we will store away the record and erase
it later where appropriate. */
task_info->should_erase = 1;
/* In the exceedingly unlikely event that the account's old hash is
actually 0, this would cause the account not to be included in
the bank hash. */
if( memcmp( task_info->acc_hash->hash, acc_meta->hash, sizeof(fd_hash_t) ) != 0 ) {
task_info->hash_changed = 1;
}
} else {
uchar * acc_data = fd_account_get_data((fd_account_meta_t *) acc_meta);
fd_pubkey_t const * acc_key = fd_type_pun_const( task_info->rec->pair.key[0].uc );
fd_pubkey_t const * acc_key = fd_funk_key_to_acc( task_info->rec->pair.key );
fd_lthash_value_t new_lthash_value;
fd_lthash_zero(&new_lthash_value);
fd_hash_account_current( task_info->acc_hash->hash, &new_lthash_value, acc_meta, acc_key->key, acc_data );
Expand All @@ -344,7 +347,7 @@ fd_account_hash_task( void *tpool,

if( FD_LIKELY(task_info->hash_changed && ((NULL != acc_meta_parent) && (acc_meta_parent->info.lamports != 0) ) ) ) {
uchar * acc_data = fd_account_get_data(acc_meta_parent);
fd_pubkey_t const * acc_key = fd_type_pun_const( task_info->rec->pair.key[0].uc );
fd_pubkey_t const * acc_key = fd_funk_key_to_acc( task_info->rec->pair.key );
fd_lthash_value_t old_lthash_value;
fd_lthash_zero(&old_lthash_value);
fd_hash_t old_hash;
Expand Down Expand Up @@ -373,7 +376,7 @@ fd_collect_modified_accounts( fd_exec_slot_ctx_t * slot_ctx,
if( !fd_funk_key_is_acc( rec->pair.key ) )
continue;

fd_pubkey_t const * pubkey = fd_type_pun_const( rec->pair.key[0].uc );
fd_pubkey_t const * pubkey = fd_funk_key_to_acc( rec->pair.key );

if (((pubkey->ul[0] == 0) & (pubkey->ul[1] == 0) & (pubkey->ul[2] == 0) & (pubkey->ul[3] == 0)))
FD_LOG_WARNING(( "null pubkey (system program?) showed up as modified" ));
Expand All @@ -390,7 +393,7 @@ fd_collect_modified_accounts( fd_exec_slot_ctx_t * slot_ctx,
NULL != rec;
rec = fd_funk_txn_next_rec( funk, rec ) ) {

fd_pubkey_t const * acc_key = fd_type_pun_const( rec->pair.key[0].uc );
fd_pubkey_t const * acc_key = fd_funk_key_to_acc( rec->pair.key );

if( !fd_funk_key_is_acc( rec->pair.key ) )
continue;
Expand Down Expand Up @@ -450,7 +453,7 @@ fd_update_hash_bank_tpool( fd_exec_slot_ctx_t * slot_ctx,
FD_BORROWED_ACCOUNT_DECL(acc_rec);
acc_rec->const_rec = task_info->rec;

fd_pubkey_t const * acc_key = fd_type_pun_const( task_info->rec->pair.key[0].uc );
fd_pubkey_t const * acc_key = fd_funk_key_to_acc( task_info->rec->pair.key );
int err = fd_acc_mgr_modify( acc_mgr, txn, acc_key, 0, 0UL, acc_rec);
if( FD_UNLIKELY( err!=FD_ACC_MGR_SUCCESS ) ) {
FD_LOG_ERR(( "failed to modify account during bank hash" ));
Expand Down Expand Up @@ -757,8 +760,8 @@ fd_accounts_sorted_subrange( fd_exec_slot_ctx_t * slot_ctx, uint range_idx, uint
fd_lthash_add( &accum, &new_lthash_value );

fd_hash_t * h = (fd_hash_t *) metadata->hash;
if( (h->ul[0] | h->ul[1] | h->ul[2] | h->ul[3]) != 0 ) {
if( fd_acc_exists( metadata ) && memcmp( metadata->hash, &hash, 32 ) != 0 ) {
if( FD_LIKELY( (h->ul[0] | h->ul[1] | h->ul[2] | h->ul[3]) != 0 ) ) {
if( FD_UNLIKELY( fd_acc_exists( metadata ) && memcmp( metadata->hash, &hash, 32 ) != 0 ) ) {
FD_LOG_WARNING(( "snapshot hash (%s) doesn't match calculated hash (%s)", FD_BASE58_ENC_32_ALLOCA( metadata->hash ), FD_BASE58_ENC_32_ALLOCA( &hash ) ));
}
} else
Expand Down Expand Up @@ -913,10 +916,10 @@ fd_accounts_hash_inc_only( fd_exec_slot_ctx_t * slot_ctx, fd_hash_t *accounts_ha
fd_hash_account_current( (uchar *) metadata->hash, NULL, metadata, rec->pair.key->uc, fd_account_get_data(metadata) );
} else if( do_hash_verify ) {
uchar hash[32];
ulong old_slot = slot_ctx->slot_bank.slot;
slot_ctx->slot_bank.slot = metadata->slot;
// ulong old_slot = slot_ctx->slot_bank.slot;
// slot_ctx->slot_bank.slot = metadata->slot;
fd_hash_account_current( (uchar *) &hash, NULL, metadata, rec->pair.key->uc, fd_account_get_data(metadata) );
slot_ctx->slot_bank.slot = old_slot;
// slot_ctx->slot_bank.slot = old_slot;
if ( fd_acc_exists( metadata ) && memcmp( metadata->hash, &hash, 32 ) != 0 ) {
FD_LOG_WARNING(( "snapshot hash (%s) doesn't match calculated hash (%s)", FD_BASE58_ENC_32_ALLOCA( metadata->hash ), FD_BASE58_ENC_32_ALLOCA( &hash ) ));
}
Expand Down
1 change: 1 addition & 0 deletions src/flamenco/runtime/tests/Local.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,4 @@ run-runtime-test-nightly: $(OBJDIR)/bin/fd_ledger
OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l mainnet-257033306 -s snapshot-257033306-EE3WdRoE4J1LTjegJMK3ZzxKZbSMQhLMaTM5Jp4SygMU.tar.zst -p 100 -y 450 -m 500000000 -e 257213306 --zst
OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l mainnet-296243940 -s snapshot-296400651-HDt9Gf1YKcruvuBV4q442qV4xjHer4KZ9sZao9XQspZP.tar.zst -p 75 -y 750 -m 700000000 -e 296550651 --zst
OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l devnet-340941576 -s snapshot-340924320-8j9h6EKmuZ3G93Y3Pb3FqrNdCDTGE5PKowHMY3xkXG1K.tar.zst -p 75 -y 400 -m 200000000 -e 340941580 --zst
OBJDIR=$(OBJDIR) src/flamenco/runtime/tests/run_nightly_test.sh -l testnet-305516256 -s snapshot-305516254-C4oM7ajmCMo1aDakR8Q8FriSVpXW53jwbb3da37jm7bN.tar.zst -p 75 -y 400 -m 150000000 -e 305516292 --zst

0 comments on commit 7152b42

Please sign in to comment.