From 801cce687d83698c73765748f3f66361625d863a Mon Sep 17 00:00:00 2001 From: ibhatt-jumptrading Date: Wed, 18 Dec 2024 18:53:01 +0000 Subject: [PATCH] funk: adding txn metadata to erased records --- src/app/ledger/main.c | 2 +- src/flamenco/runtime/fd_hashes.c | 2 +- src/flamenco/runtime/fd_runtime.c | 2 +- .../runtime/program/fd_bpf_program_util.c | 4 +-- src/funk/fd_funk_rec.c | 18 ++++++++++++- src/funk/fd_funk_rec.h | 25 +++++++++++++++--- src/funk/fd_funk_txn.c | 20 +++++++------- src/funk/test_funk_common.hpp | 2 +- src/funk/test_funk_part.c | 2 +- src/funk/test_funk_rec.c | 26 +++++++++---------- src/funk/test_funk_val.c | 2 +- 11 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/app/ledger/main.c b/src/app/ledger/main.c index f386bab7df..ce0e5c11a4 100644 --- a/src/app/ledger/main.c +++ b/src/app/ledger/main.c @@ -1217,7 +1217,7 @@ prune( fd_ledger_args_t * args ) { fd_funk_val_sz( original_rec ) ) == 0 )); } else { fd_funk_rec_t * mod_rec = fd_funk_rec_modify( pruned_funk, rec ); - int res = fd_funk_rec_remove( pruned_funk, mod_rec ); + int res = fd_funk_rec_remove( pruned_funk, mod_rec, rec->pair.xid->ul[0] ); FD_TEST(( res == 0 )); } } diff --git a/src/flamenco/runtime/fd_hashes.c b/src/flamenco/runtime/fd_hashes.c index eabd7c4c36..e61f10bf74 100644 --- a/src/flamenco/runtime/fd_hashes.c +++ b/src/flamenco/runtime/fd_hashes.c @@ -525,7 +525,7 @@ fd_update_hash_bank_tpool( fd_exec_slot_ctx_t * slot_ctx, continue; } - fd_funk_rec_remove(funk, fd_funk_rec_modify(funk, task_info->rec)); + fd_funk_rec_remove( funk, fd_funk_rec_modify(funk, task_info->rec), task_info->rec->pair.xid->ul[0] ); } // Sanity-check LT Hash diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index 2dbde7c674..51cd0b8428 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -3694,7 +3694,7 @@ fd_runtime_cleanup_incinerator( fd_exec_slot_ctx_t * slot_ctx ) { fd_funk_t * funk = slot_ctx->acc_mgr->funk; fd_funk_rec_t const * rec = fd_funk_rec_query( funk, slot_ctx->funk_txn, &id ); if( rec ) - fd_funk_rec_remove( funk, fd_funk_rec_modify( funk, rec )); + fd_funk_rec_remove( funk, fd_funk_rec_modify( funk, rec ), rec->pair.xid->ul[0] ); } void diff --git a/src/flamenco/runtime/program/fd_bpf_program_util.c b/src/flamenco/runtime/program/fd_bpf_program_util.c index 0c303ce23e..d7a60a3c77 100644 --- a/src/flamenco/runtime/program/fd_bpf_program_util.c +++ b/src/flamenco/runtime/program/fd_bpf_program_util.c @@ -224,7 +224,7 @@ fd_bpf_create_bpf_program_cache_entry( fd_exec_slot_ctx_t * slot_ctx, if( FD_UNLIKELY( 0!=fd_sbpf_program_load( prog, program_data, program_data_len, syscalls, false ) ) ) { /* Remove pending funk record */ FD_LOG_DEBUG(( "fd_sbpf_program_load() failed: %s", fd_sbpf_strerror() )); - fd_funk_rec_remove( funk, rec ); + fd_funk_rec_remove( funk, rec, funk_txn->xid.ul[0] ); return -1; } @@ -267,7 +267,7 @@ fd_bpf_create_bpf_program_cache_entry( fd_exec_slot_ctx_t * slot_ctx, if( FD_UNLIKELY( res ) ) { /* Remove pending funk record */ FD_LOG_DEBUG(( "fd_vm_validate() failed" )); - fd_funk_rec_remove( funk, rec ); + fd_funk_rec_remove( funk, rec, 0UL ); return -1; } diff --git a/src/funk/fd_funk_rec.c b/src/funk/fd_funk_rec.c index c2c5afdc17..30927a042b 100644 --- a/src/funk/fd_funk_rec.c +++ b/src/funk/fd_funk_rec.c @@ -414,7 +414,8 @@ fd_funk_rec_insert( fd_funk_t * funk, int fd_funk_rec_remove( fd_funk_t * funk, - fd_funk_rec_t * rec ) { + fd_funk_rec_t * rec, + ulong erase_data ) { if( FD_UNLIKELY( !funk ) ) return FD_FUNK_ERR_INVAL; fd_funk_check_write( funk ); @@ -461,9 +462,24 @@ fd_funk_rec_remove( fd_funk_t * funk, fd_funk_part_set_intern( fd_funk_get_partvec( funk, wksp ), rec_map, rec, FD_FUNK_PART_NULL ); rec->flags |= FD_FUNK_REC_FLAG_ERASE; + /* At this point, the 5 most significant bytes should store data about the + transaction that the record was updated in. */ + + fd_funk_rec_set_erase_data( rec, erase_data ); + return FD_FUNK_SUCCESS; } +void +fd_funk_rec_set_erase_data( fd_funk_rec_t * rec, ulong erase_data ) { + rec->flags |= ((erase_data & 0xFFFFFFFFFFUL) << (sizeof(unsigned long) * 8 - 40)); +} + +ulong +fd_funk_rec_get_erase_data( fd_funk_rec_t const * rec ) { + return (rec->flags >> (sizeof(unsigned long) * 8 - 40)) & 0xFFFFFFFFFFUL; +} + fd_funk_rec_t * fd_funk_rec_write_prepare( fd_funk_t * funk, fd_funk_txn_t * txn, diff --git a/src/funk/fd_funk_rec.h b/src/funk/fd_funk_rec.h index 03ee6d776c..a128edb89c 100644 --- a/src/funk/fd_funk_rec.h +++ b/src/funk/fd_funk_rec.h @@ -14,7 +14,8 @@ #define FD_FUNK_REC_ALIGN (32UL) /* FD_FUNK_REC_FLAG_* are flags that can be bit-ored together to specify - how records are to be interpreted. + how records are to be interpreted. The 5 most signifcant bytes of a + rec's flag are reserved to be used in conjunction with the ERASE flag. - ERASE indicates a record in an in-preparation transaction should be erased if and when the in-preparation transaction is published. If @@ -22,7 +23,10 @@ be set on a published record. Will not be set if an in-preparation transaction ancestor has this record with erase set. If set, the first ancestor transaction encountered (going from youngest to - oldest) will not have erased set. */ + oldest) will not have erased set. + + If the ERASE flag is set, then the five most significant bytes of the + flags field for the record will be used to store user-specified data. */ #define FD_FUNK_REC_FLAG_ERASE (1UL<<0) @@ -440,7 +444,22 @@ fd_funk_rec_insert( fd_funk_t * funk, int fd_funk_rec_remove( fd_funk_t * funk, - fd_funk_rec_t * rec ); + fd_funk_rec_t * rec, + ulong erase_data ); + + +/* When a record is erased there is metadata stored in the five most + significant bytes of a record. These are helpers to make setting + and getting these values simple. The caller is responsible for doing + a check on the flag of the record before using the value of the erase + data. The 5 least significant bytes of the erase data parameter will + be used and set into the erase flag. */ + +void +fd_funk_rec_set_erase_data( fd_funk_rec_t * rec, ulong erase_data ); + +ulong +fd_funk_rec_get_erase_data( fd_funk_rec_t const * rec ); /* fd_funk_rec_write_prepare combines several operations into one convenient package. There are 3 basic cases: diff --git a/src/funk/fd_funk_txn.c b/src/funk/fd_funk_txn.c index 62ad5b7d9d..e19780bc12 100644 --- a/src/funk/fd_funk_txn.c +++ b/src/funk/fd_funk_txn.c @@ -552,12 +552,12 @@ fd_funk_txn_update( ulong * _dst_rec_head_idx, /* Pointer to t the record map below. Note that we can't just reuse rec_idx in the update case because that could break map queries. */ - ulong val_sz = (ulong)rec_map[ rec_idx ].val_sz; - ulong val_max = (ulong)rec_map[ rec_idx ].val_max; - ulong val_gaddr = rec_map[ rec_idx ].val_gaddr; - int val_no_free = rec_map[ rec_idx ].val_no_free; - uint part = rec_map[ rec_idx ].part; - int erase = rec_map[ rec_idx ].flags & FD_FUNK_REC_FLAG_ERASE; + ulong val_sz = (ulong)rec_map[ rec_idx ].val_sz; + ulong val_max = (ulong)rec_map[ rec_idx ].val_max; + ulong val_gaddr = rec_map[ rec_idx ].val_gaddr; + int val_no_free = rec_map[ rec_idx ].val_no_free; + uint part = rec_map[ rec_idx ].part; + ulong flags = rec_map[ rec_idx ].flags; fd_funk_part_set_intern( partvec, rec_map, &rec_map[ rec_idx ], FD_FUNK_PART_NULL ); fd_funk_rec_map_remove( rec_map, fd_funk_rec_pair( &rec_map[ rec_idx ] ) ); @@ -590,11 +590,11 @@ fd_funk_txn_update( ulong * _dst_rec_head_idx, /* Pointer to t /* Unstash value metadata from stack temporaries into dst_rec */ - dst_rec->val_sz = (uint)val_sz; - dst_rec->val_max = (uint)val_max; - dst_rec->val_gaddr = val_gaddr; + dst_rec->val_sz = (uint)val_sz; + dst_rec->val_max = (uint)val_max; + dst_rec->val_gaddr = val_gaddr; dst_rec->val_no_free = val_no_free; - dst_rec->flags = ( erase ? FD_FUNK_REC_FLAG_ERASE : 0 ); + dst_rec->flags = flags; /* Use the new partition */ diff --git a/src/funk/test_funk_common.hpp b/src/funk/test_funk_common.hpp index 66c0383cb1..169e7b962c 100644 --- a/src/funk/test_funk_common.hpp +++ b/src/funk/test_funk_common.hpp @@ -170,7 +170,7 @@ struct fake_funk { auto key = rec->real_id(); auto* rec2 = fd_funk_rec_query(_real, txn2, &key); assert(rec2 != NULL); - assert(fd_funk_rec_remove(_real, (fd_funk_rec_t *)rec2) == FD_FUNK_SUCCESS); + assert(fd_funk_rec_remove(_real, (fd_funk_rec_t *)rec2, 0UL) == FD_FUNK_SUCCESS); rec->_erased = true; rec->_data.clear(); diff --git a/src/funk/test_funk_part.c b/src/funk/test_funk_part.c index a5e3aae0db..34159611da 100644 --- a/src/funk/test_funk_part.c +++ b/src/funk/test_funk_part.c @@ -113,7 +113,7 @@ main( int argc, fd_funk_rec_key_t * key = &recs[i]; fd_funk_rec_t * rec = fd_funk_rec_write_prepare(funk, txn, key, 0, 0, NULL, NULL); if (rec && !(rec->flags & FD_FUNK_REC_FLAG_ERASE)) { - int err = fd_funk_rec_remove(funk, rec); + int err = fd_funk_rec_remove(funk, rec, 0); FD_TEST(!err); } break; diff --git a/src/funk/test_funk_rec.c b/src/funk/test_funk_rec.c index 4255567d9e..1ee752a8c2 100644 --- a/src/funk/test_funk_rec.c +++ b/src/funk/test_funk_rec.c @@ -128,15 +128,15 @@ main( int argc, FD_TEST( !fd_funk_rec_modify( tst, rec_map+rec_max ) ); FD_TEST( !fd_funk_rec_modify( tst, (fd_funk_rec_t *)1UL ) ); - FD_TEST( fd_funk_rec_remove( NULL, NULL )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( NULL, (fd_funk_rec_t *)trec )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( tst, NULL )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( tst, rec_map+rec_max )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)1UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( NULL, NULL , 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( NULL, (fd_funk_rec_t *)trec, 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( tst, NULL , 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( tst, rec_map+rec_max , 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)1UL , 0UL )==FD_FUNK_ERR_INVAL ); if( trec ) { if( is_frozen ) { - FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)trec )==FD_FUNK_ERR_FROZEN ); + FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)trec, 0UL )==FD_FUNK_ERR_FROZEN ); } } @@ -248,14 +248,14 @@ main( int argc, FD_TEST( !fd_funk_rec_modify( tst, rec_map+rec_max ) ); FD_TEST( !fd_funk_rec_modify( tst, (fd_funk_rec_t *)1UL ) ); - FD_TEST( fd_funk_rec_remove( NULL, NULL )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( NULL, (fd_funk_rec_t *)trec )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( tst, NULL )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( tst, rec_map+rec_max )==FD_FUNK_ERR_INVAL ); - FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)1UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( NULL, NULL , 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( NULL, (fd_funk_rec_t *)trec, 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( tst, NULL , 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( tst, rec_map+rec_max , 0UL )==FD_FUNK_ERR_INVAL ); + FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)1UL , 0UL )==FD_FUNK_ERR_INVAL ); if( trec && ttxn_is_frozen ) { - FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)trec )==FD_FUNK_ERR_FROZEN ); + FD_TEST( fd_funk_rec_remove( tst, (fd_funk_rec_t *)trec, 0UL )==FD_FUNK_ERR_FROZEN ); } int err; @@ -406,7 +406,7 @@ main( int argc, fd_funk_rec_t * _trec = fd_funk_rec_modify( tst, trec ); FD_TEST( trec==(fd_funk_rec_t const *)_trec ); - FD_TEST( !fd_funk_rec_remove( tst, _trec ) ); + FD_TEST( !fd_funk_rec_remove( tst, _trec, 0UL ) ); } else if( op>=2 ) { /* Prepare 8x as publish and cancel combined */ diff --git a/src/funk/test_funk_val.c b/src/funk/test_funk_val.c index f002056fdf..1a4023aa81 100644 --- a/src/funk/test_funk_val.c +++ b/src/funk/test_funk_val.c @@ -528,7 +528,7 @@ main( int argc, fd_funk_rec_t const * trec = fd_funk_rec_query( tst, ttxn, key_set( tkey, rkey ) ); fd_funk_rec_t * _trec = fd_funk_rec_modify( tst, trec ); - FD_TEST( !fd_funk_rec_remove( tst, _trec ) ); + FD_TEST( !fd_funk_rec_remove( tst, _trec, 0UL ) ); } else if( op>=2 ) { /* Prepare 8x as publish and cancel combined */