diff --git a/.github/workflows/run_integration_tests.sh b/.github/workflows/run_integration_tests.sh index 4f46ff0034..9e18a1f33c 100755 --- a/.github/workflows/run_integration_tests.sh +++ b/.github/workflows/run_integration_tests.sh @@ -19,8 +19,9 @@ rm -rf ./mainnet/results/ # erigon_getBalanceChangesInBlock: new algo using TKV # erigon_getLatestLogs: new algo using TKV # eth_getLogs: new algo using TKV -# ots_getTransactionBySenderAndNonce: new algo using TKV # ots_getContractCreator: new algo using TKV +# ots_getTransactionBySenderAndNonce/test_04.json: erigon3 bug in limit and page_size management in IndexRangeQuery query +# ots_getTransactionBySenderAndNonce/test_07.json: erigon3 bug in limit and page_size management in IndexRangeQuery query # ots_hasCode: new algo using TKV # ots_searchTransactionsAfter: new algo using TKV # ots_searchTransactionsBefore: new algo using TKV @@ -28,7 +29,7 @@ rm -rf ./mainnet/results/ # trace_rawTransaction: different implementation # trace_replayTransaction/trace_replyBlockTransaction: silkworm has different response with erigon3 but could be erigon3 problem (to be analyzed) -python3 ./run_tests.py --continue --blockchain mainnet --jwt "$2" --display-only-fail --json-diff --port 51515 -x \ +python3 ./run_tests.py --continue --blockchain mainnet --jwt "$2" --display-only-fail --json-diff --port 51515 --transport_type http,websocket -x \ debug_accountRange,\ debug_getModifiedAccounts,\ debug_storageRangeAt,\ @@ -50,13 +51,14 @@ erigon_getLatestLogs,\ eth_estimateGas,\ eth_getBlockReceipts/test_07.json,\ eth_getLogs,\ -ots_getTransactionBySenderAndNonce,\ +ots_getTransactionBySenderAndNonce/test_04.json,\ +ots_getTransactionBySenderAndNonce/test_07.json,\ ots_getContractCreator,\ ots_hasCode/test_09.json,\ ots_searchTransactionsAfter,\ ots_searchTransactionsBefore,\ parity_listStorageKeys/test_12.json,\ -trace_rawTransaction --transport_type http,websocket +trace_rawTransaction failed_test=$? diff --git a/silkworm/db/tables.hpp b/silkworm/db/tables.hpp index 9ab4c95dd8..cd7c98ccd3 100644 --- a/silkworm/db/tables.hpp +++ b/silkworm/db/tables.hpp @@ -430,4 +430,10 @@ inline constexpr const char* kStorageDomain{"storage"}; //! \details Domain storing the account code information inline constexpr const char* kCodeDomain{"code"}; +//! \details History storing the account common information +inline constexpr const char* kAccountsHistory{"AccountsHistory"}; + +//! \details Inverted Index storing the account common information +inline constexpr const char* kAccountsHistoryIdx{"AccountsHistoryIdx"}; + } // namespace silkworm::db::table diff --git a/silkworm/rpc/commands/eth_api.cpp b/silkworm/rpc/commands/eth_api.cpp index 264da07e29..7e64efd000 100644 --- a/silkworm/rpc/commands/eth_api.cpp +++ b/silkworm/rpc/commands/eth_api.cpp @@ -2063,7 +2063,7 @@ Task EthereumRpcApi::handle_fee_history(const nlohmann::json& request, nlo Task EthereumRpcApi::handle_base_fee(const nlohmann::json& request, nlohmann::json& reply) { const auto& params = request["params"]; - if (params.size() != 0) { + if (!params.empty()) { const auto error_msg = "invalid eth_baseFee params: " + params.dump(); SILK_ERROR << error_msg; reply = make_json_error(request, 100, error_msg); @@ -2106,7 +2106,7 @@ Task EthereumRpcApi::handle_base_fee(const nlohmann::json& request, nlohma Task EthereumRpcApi::handle_blob_base_fee(const nlohmann::json& request, nlohmann::json& reply) { const auto& params = request["params"]; - if (params.size() != 0) { + if (!params.empty()) { const auto error_msg = "invalid eth_blobBaseFee params: " + params.dump(); SILK_ERROR << error_msg; reply = make_json_error(request, 100, error_msg); diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index 6cf3dfb7a7..2857041d8a 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -22,19 +22,21 @@ #include #include -#include #include #include #include #include +#include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -269,88 +271,121 @@ Task OtsRpcApi::handle_ots_get_transaction_by_sender_and_nonce(const nlohm const auto sender = params[0].get(); const auto nonce = params[1].get(); - SILK_DEBUG << "sender: " << sender << " nonce: " << nonce; + SILK_DEBUG << "sender: " << sender << ", nonce: " << nonce; auto tx = co_await database_->begin(); try { - auto account_history_cursor = co_await tx->cursor(table::kAccountHistoryName); - auto account_change_set_cursor = co_await tx->cursor_dup_sort(table::kAccountChangeSetName); - const ByteView sender_byte_view{sender.bytes}; - auto key_value = co_await account_history_cursor->seek(sender_byte_view); - - std::vector account_block_numbers; - - BlockNum max_block_prev_chunk = 0; - roaring::Roaring64Map bitmap; - - while (true) { - if (key_value.key.empty() || !key_value.key.starts_with(sender_byte_view)) { - auto plain_state_cursor = co_await tx->cursor(table::kPlainStateName); - auto account_payload = co_await plain_state_cursor->seek(sender_byte_view); - auto account = Account::from_encoded_storage(account_payload.value); - - if (account.has_value() && account.value().nonce > nonce) { - break; - } - + auto key = db::code_domain_key(sender); + + db::kv::api::IndexRangeQuery query{ + .table = db::table::kAccountsHistoryIdx, + .key = key, + .from_timestamp = -1, + .to_timestamp = -1, + .ascending_order = true}; + auto paginated_result = co_await tx->index_range(std::move(query)); + auto it = co_await paginated_result.begin(); + + std::vector keys; + std::uint64_t count = 0; + TxnId prev_txn_id = 0; + TxnId next_txn_id = 0; + while (const auto value = co_await it.next()) { + const auto txn_id = static_cast(*value); + if (count++ % 4096 != 0) { + next_txn_id = txn_id; + continue; + } + SILK_DEBUG << "count: " << count << ", txnId: " << txn_id; + db::kv::api::HistoryPointQuery hpq{ + .table = db::table::kAccountsHistory, + .key = key, + .timestamp = *value}; + auto result = co_await tx->history_seek(std::move(hpq)); + if (!result.success) { reply = make_json_content(request, nlohmann::detail::value_t::null); co_await tx->close(); co_return; } + SILK_DEBUG << "history: len:" << result.value.size() << " [" << result.value << "]"; - bitmap = bitmap::parse(key_value.value); - auto const max_block = bitmap.maximum(); - auto block_key{db::block_key(max_block)}; - auto account_payload = co_await account_change_set_cursor->seek_both(block_key, sender_byte_view); - if (account_payload.starts_with(sender_byte_view)) { - account_payload = account_payload.substr(sender_byte_view.length()); - auto account = Account::from_encoded_storage(account_payload); - - if (account.has_value() && account.value().nonce > nonce) { - break; - } + if (result.value.empty()) { + SILK_DEBUG << "history bytes empty"; + prev_txn_id = txn_id; + continue; } - max_block_prev_chunk = max_block; - key_value = co_await account_history_cursor->next(); + const auto account{Account::from_encoded_storage_v3(result.value)}; + SILK_DEBUG << "Account: " << *account; + if (account->nonce > nonce) { + break; + } + prev_txn_id = txn_id; } + SILK_DEBUG << "range -> prev_txn_id: " << prev_txn_id << ", next_txn_id: " << next_txn_id; - uint64_t cardinality = bitmap.cardinality(); - account_block_numbers.reserve(cardinality); - bitmap.toUint64Array(account_block_numbers.data()); - - uint64_t idx = 0; - for (uint64_t i = 0; i < cardinality; ++i) { - auto block_number = account_block_numbers[i]; - auto block_key{db::block_key(block_number)}; - auto account_payload = co_await account_change_set_cursor->seek_both(block_key, sender_byte_view); - if (account_payload.starts_with(sender_byte_view)) { - account_payload = account_payload.substr(sender_byte_view.length()); - auto account = Account::from_encoded_storage(account_payload); + if (next_txn_id == 0) { + next_txn_id = prev_txn_id + 1; + } - if (account.has_value() && account.value().nonce > nonce) { - idx = i; - break; - } + db::txn::TxNum creation_txn_id = 0; + auto index = co_await async_binary_search(static_cast(next_txn_id - prev_txn_id), [&](size_t i) -> Task { + auto txn_id = i + prev_txn_id; + + SILK_DEBUG << "searching for txnId: " << txn_id << ", i: " << i; + db::kv::api::HistoryPointQuery hpq{ + .table = db::table::kAccountsHistory, + .key = key, + .timestamp = static_cast(txn_id)}; + auto result = co_await tx->history_seek(std::move(hpq)); + if (!result.success) { + co_return false; } - } + if (result.value.empty()) { + creation_txn_id = static_cast(txn_id); + co_return false; + } + const auto account{Account::from_encoded_storage_v3(result.value)}; + SILK_DEBUG << "account.nonce: " << account->nonce << ", nonce: " << nonce; + if (account->nonce <= nonce) { + creation_txn_id = std::max(creation_txn_id, static_cast(txn_id)); + co_return false; + } + co_return true; + }); + SILK_DEBUG << "after search -> index: " << index << " creationTxnId: " << creation_txn_id; - auto nonce_block = max_block_prev_chunk; - if (idx > 0) { - nonce_block = account_block_numbers[idx - 1]; + if (creation_txn_id == 0) { + SILK_DEBUG << "binary search in [" << prev_txn_id << ", " << next_txn_id << "] found nothing"; + reply = make_json_content(request, nlohmann::detail::value_t::null); } - const auto chain_storage{tx->create_storage()}; - auto block_with_hash = co_await core::read_block_by_number(*block_cache_, *chain_storage, nonce_block); - if (block_with_hash) { - for (const auto& transaction : block_with_hash->block.transactions) { - if (transaction.sender() == sender && transaction.nonce == nonce) { - reply = make_json_content(request, transaction.hash()); - co_await tx->close(); - co_return; - } + auto provider = ethdb::kv::canonical_body_for_storage_provider(backend_); + const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, creation_txn_id, provider); + if (block_number_opt) { + const auto block_number = block_number_opt.value(); + const auto min_txn_id = co_await db::txn::min_tx_num(*tx, block_number, provider); + const auto first_txn_id = co_await tx->first_txn_num_in_block(block_number); + SILK_DEBUG << "block_number: " << block_number << ", min_txn_id: " << min_txn_id << ", first_txn_id: " << first_txn_id; + + TxnId tx_index{0}; + if (creation_txn_id == min_txn_id) { + tx_index = index + prev_txn_id - min_txn_id - 1; + } else { + tx_index = creation_txn_id - min_txn_id - 1; + } + SILK_DEBUG << "block_number: " << block_number << ", tx_index: " << tx_index; + + const auto chain_storage{tx->create_storage()}; + + const auto transaction = co_await chain_storage->read_transaction_by_idx_in_block(block_number, tx_index); + if (!transaction) { + SILK_INFO << "No transaction found in block " << block_number << " for index " << tx_index; + reply = make_json_content(request, nlohmann::detail::value_t::null); + } else { + reply = make_json_content(request, transaction.value().hash()); } - reply = make_json_content(request, nlohmann::detail::value_t::null); } else { + SILK_INFO << "No block found for txn_id " << creation_txn_id; reply = make_json_content(request, nlohmann::detail::value_t::null); } } catch (const std::invalid_argument& iv) {