From 59e531a7c2b6c90de172ced6cf7abe9a6fd60e5d Mon Sep 17 00:00:00 2001 From: Julian Meyer Date: Wed, 10 Jan 2018 15:38:13 -0800 Subject: [PATCH] Address Index (#16) * Add address index Fixed bugs with address index * Patch addrindex for Zerocoin --- src/init.cpp | 8 ++- src/main.cpp | 106 +++++++++++++++++++++++++++++++++++--- src/main.h | 47 +++++++++++++++++ src/masternodeconfig.cpp | 2 +- src/rpcclient.cpp | 4 ++ src/rpcrawtransaction.cpp | 60 +++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 54 +++++++++++++++++++ src/txdb.h | 3 ++ 10 files changed, 278 insertions(+), 8 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index a7104c6..790401f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -341,6 +341,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), 0)); + strUsage += HelpMessageOpt("-addrindex", strprintf(_("Maintain a full address index, used by the searchrawtransactions rpc call (default: %u)"), 0)); strUsage += HelpMessageOpt("-forcestart", _("Attempt to force blockchain corruption recovery") + " " + _("on startup")); strUsage += HelpMessageGroup(_("Connection options:")); @@ -1298,7 +1299,7 @@ bool AppInit2(boost::thread_group& threadGroup) else if (nTotalCache > (nMaxDbCache << 20)) nTotalCache = (nMaxDbCache << 20); // total cache cannot be greater than nMaxDbCache size_t nBlockTreeDBCache = nTotalCache / 8; - if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", true)) + if (nBlockTreeDBCache > (1 << 21) && !GetBoolArg("-txindex", true) && !GetBoolArg("-addrindex", true)) nBlockTreeDBCache = (1 << 21); // block tree db cache shouldn't be larger than 2 MiB nTotalCache -= nBlockTreeDBCache; size_t nCoinDBCache = nTotalCache / 2; // use half of the remaining cache for coindb cache @@ -1360,6 +1361,11 @@ bool AppInit2(boost::thread_group& threadGroup) break; } + if (fAddrIndex != GetBoolArg("-addrindex", true)) { + strLoadError = _("You need to rebuild the database using -reindex to change -addrindex"); + break; + } + // Populate list of invalid/fraudulent outpoints that are banned from the chain PopulateInvalidOutPointMap(); diff --git a/src/main.cpp b/src/main.cpp index 04b7248..5b0db0c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -11,6 +11,7 @@ #include "accumulators.h" #include "addrman.h" #include "alert.h" +#include "base58.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" @@ -74,6 +75,7 @@ int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; bool fTxIndex = true; +bool fAddrIndex = true; bool fIsBareMultisigStd = true; bool fCheckBlockIndex = false; bool fVerifyingBlocks = false; @@ -1826,6 +1828,44 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState& state, const CTransa return true; } +bool ReadTransaction(CTransaction& tx, const CDiskTxPos &pos, uint256 &hashBlock) { + CAutoFile file(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), pos.nTxOffset, SEEK_CUR); + file >> tx; + } catch (std::exception &e) { + return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__); + } + hashBlock = header.GetHash(); + return true; +} + +bool FindTransactionsByDestination(const CTxDestination &dest, std::set &setpos) { + uint160 addrid; + const CKeyID *pkeyid = boost::get(&dest); + if (pkeyid) { + addrid = static_cast(*pkeyid); + } else { + const CScriptID *pscriptid = boost::get(&dest); + if (pscriptid) { + addrid = static_cast(*pscriptid); + } else { + return false; + } + } + + LOCK(cs_main); + if (!fAddrIndex) + return false; + std::vector vPos; + if (!pblocktree->ReadAddrIndex(addrid, vPos)) + return false; + setpos.insert(vPos.begin(), vPos.end()); + return true; +} + bool AcceptableInputs(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, bool* pfMissingInputs, bool fRejectInsaneFee, bool isDSTX) { AssertLockHeld(cs_main); @@ -2932,6 +2972,33 @@ static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; +// Index either: a) every data push >=8 bytes, b) if no such pushes, the entire script +void static BuildAddrIndex(const CScript &script, const CExtDiskTxPos &pos, std::vector > &out) +{ + CScript::const_iterator pc = script.begin(); + CScript::const_iterator pend = script.end(); + std::vector data; + opcodetype opcode; + bool fHaveData = false; + while (pc < pend) { + script.GetOp(pc, opcode, data); + if (0 <= opcode && opcode <= OP_PUSHDATA4 && data.size() >= 8) { // data element + uint160 addrid; + if (data.size() <= 20) { + memcpy(&addrid, &data[0], data.size()); + } else { + addrid = Hash160(data); + } + out.push_back(std::make_pair(addrid, pos)); + fHaveData = true; + } + } + if (!fHaveData) { + uint160 addrid = Hash160(script); + out.push_back(std::make_pair(addrid, pos)); + } +} + bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck, bool fAlreadyChecked) { AssertLockHeld(cs_main); @@ -3005,9 +3072,13 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin CAmount nFees = 0; int nInputs = 0; unsigned int nSigOps = 0; - CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); - std::vector > vPos; - vPos.reserve(block.vtx.size()); + CExtDiskTxPos pos(CDiskTxPos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())), pindex->nHeight); + std::vector > vPosTxid; + std::vector > vPosAddrid; + if (fTxIndex) + vPosTxid.reserve(block.vtx.size()); + if (fAddrIndex) + vPosAddrid.reserve(block.vtx.size() * 4); blockundo.vtxundo.reserve(block.vtx.size() - 1); CAmount nValueOut = 0; CAmount nValueIn = 0; @@ -3104,9 +3175,23 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); } - UpdateCoins(tx, state, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); + if (fTxIndex) + vPosTxid.push_back(std::make_pair(tx.GetHash(), pos)); + if (fAddrIndex) { + if (!tx.IsCoinBase()) { + BOOST_FOREACH(const CTxIn &txin, tx.vin) { + CCoins coins; + view.GetCoins(txin.prevout.hash, coins); + if (coins.IsAvailable(txin.prevout.n)) { + BuildAddrIndex(coins.vout[txin.prevout.n].scriptPubKey, pos, vPosAddrid); + } + } + } + BOOST_FOREACH(const CTxOut &txout, tx.vout) + BuildAddrIndex(txout.scriptPubKey, pos, vPosAddrid); + } - vPos.push_back(std::make_pair(tx.GetHash(), pos)); + UpdateCoins(tx, state, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } @@ -3224,9 +3309,13 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } if (fTxIndex) - if (!pblocktree->WriteTxIndex(vPos)) + if (!pblocktree->WriteTxIndex(vPosTxid)) return state.Abort("Failed to write transaction index"); + if (fAddrIndex) + if (!pblocktree->AddAddrIndex(vPosAddrid)) + return state.Abort("Failed to write address index"); + // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -4868,6 +4957,9 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("txindex", fTxIndex); LogPrintf("LoadBlockIndexDB(): transaction index %s\n", fTxIndex ? "enabled" : "disabled"); + pblocktree->ReadFlag("addrindex", fAddrIndex); + LogPrintf("LoadBlockIndexDB(): address index %s\n", fAddrIndex ? "enabled" : "disabled"); + // If this is written true before the next client init, then we know the shutdown process failed pblocktree->WriteFlag("shutdown", false); @@ -5001,6 +5093,8 @@ bool InitBlockIndex() // Use the provided setting for -txindex in the new database fTxIndex = GetBoolArg("-txindex", true); pblocktree->WriteFlag("txindex", fTxIndex); + fAddrIndex = GetBoolArg("-addrindex", true); + pblocktree->WriteFlag("addrindex", fAddrIndex); LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 7e5ce51..afa6dc1 100644 --- a/src/main.h +++ b/src/main.h @@ -140,6 +140,7 @@ extern bool fImporting; extern bool fReindex; extern int nScriptCheckThreads; extern bool fTxIndex; +extern bool fAddrIndex; extern bool fIsBareMultisigStd; extern bool fCheckBlockIndex; extern unsigned int nCoinCacheSize; @@ -300,8 +301,52 @@ struct CDiskTxPos : public CDiskBlockPos { CDiskBlockPos::SetNull(); nTxOffset = 0; } + + friend bool operator<(const CDiskTxPos &a, const CDiskTxPos &b) { + return (a.nFile < b.nFile || ( + (a.nFile == b.nFile) && (a.nPos < b.nPos || ( + (a.nPos == b.nPos) && (a.nTxOffset < b.nTxOffset))))); + } }; +struct CExtDiskTxPos : public CDiskTxPos +{ + unsigned int nHeight; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(*(CDiskTxPos*)this); + READWRITE(VARINT(nHeight)); + } + + CExtDiskTxPos(const CDiskTxPos &pos, int nHeightIn) : CDiskTxPos(pos), nHeight(nHeightIn) { + } + + CExtDiskTxPos() { + SetNull(); + } + + void SetNull() { + CDiskTxPos::SetNull(); + nHeight = 0; + } + + friend bool operator==(const CExtDiskTxPos &a, const CExtDiskTxPos &b) { + return (a.nHeight == b.nHeight && a.nFile == b.nFile && a.nPos == b.nPos && a.nTxOffset == b.nTxOffset); + } + + friend bool operator!=(const CExtDiskTxPos &a, const CExtDiskTxPos &b) { + return !(a == b); + } + + friend bool operator<(const CExtDiskTxPos &a, const CExtDiskTxPos &b) { + if (a.nHeight < b.nHeight) return true; + if (a.nHeight > b.nHeight) return false; + return ((const CDiskTxPos)a < (const CDiskTxPos)b); + } +}; CAmount GetMinRelayFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree); bool MoneyRange(CAmount nValueOut); @@ -451,6 +496,8 @@ class CScriptCheck bool WriteBlockToDisk(CBlock& block, CDiskBlockPos& pos); bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex); +bool ReadTransaction(CTransaction& tx, const CDiskTxPos &pos, uint256 &hashBlock); +bool FindTransactionsByDestination(const CTxDestination &dest, std::set &setpos); /** Functions for validating blocks and updating the block tree */ diff --git a/src/masternodeconfig.cpp b/src/masternodeconfig.cpp index a096c2c..f4a5435 100644 --- a/src/masternodeconfig.cpp +++ b/src/masternodeconfig.cpp @@ -8,7 +8,7 @@ #include "masternodeconfig.h" #include "util.h" #include "ui_interface.h" -#include +#include "base58.h" // clang-format on CMasternodeConfig masternodeConfig; diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index f50b878..dbe229f 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -79,6 +79,10 @@ static const CRPCConvertParam vRPCConvertParams[] = {"signrawtransaction", 1}, {"signrawtransaction", 2}, {"sendrawtransaction", 1}, + { "searchrawtransactions", 1 }, + { "searchrawtransactions", 2 }, + { "searchrawtransactions", 3 }, + { "searchrawtransactions", 4 }, {"gettxout", 1}, {"gettxout", 2}, {"lockunspent", 0}, diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 06faecb..91cd931 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -107,6 +107,66 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) } } +Value searchrawtransactions(const Array& params, bool fHelp) +{ + if (fHelp || params.size() < 1 || params.size() > 4) + throw runtime_error("searchrawtransactions
[verbose=1] [skip=0] [count=100]\n"); + + if (!fAddrIndex) + throw JSONRPCError(RPC_MISC_ERROR, "Address index not enabled"); + + CBitcoinAddress address(params[0].get_str()); + if (!address.IsValid()) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + + CTxDestination dest = address.Get(); + + std::set setpos; + if (!FindTransactionsByDestination(dest, setpos)) + throw JSONRPCError(RPC_DATABASE_ERROR, "Cannot search for address"); + + int nSkip = 0; + int nCount = 100; + bool fVerbose = true; + if (params.size() > 1) + fVerbose = (params[1].get_int() != 0); + if (params.size() > 2) + nSkip = params[2].get_int(); + if (params.size() > 3) + nCount = params[3].get_int(); + + if (nSkip < 0) + nSkip += setpos.size(); + if (nSkip < 0) + nSkip = 0; + if (nCount < 0) + nCount = 0; + + std::set::const_iterator it = setpos.begin(); + while (it != setpos.end() && nSkip--) it++; + + Array result; + while (it != setpos.end() && nCount--) { + CTransaction tx; + uint256 hashBlock; + if (!ReadTransaction(tx, *it, hashBlock)) + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "Cannot read transaction from disk"); + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << tx; + string strHex = HexStr(ssTx.begin(), ssTx.end()); + if (fVerbose) { + Object object; + TxToJSON(tx, hashBlock, object); + object.push_back(Pair("hex", strHex)); + result.push_back(object); + } else { + result.push_back(strHex); + } + it++; + } + return result; +} + Value getrawtransaction(const Array& params, bool fHelp) { if (fHelp || params.size() < 1 || params.size() > 2) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 97576e2..34e38c1 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -304,6 +304,7 @@ static const CRPCCommand vRPCCommands[] = {"rawtransactions", "decoderawtransaction", &decoderawtransaction, true, false, false}, {"rawtransactions", "decodescript", &decodescript, true, false, false}, {"rawtransactions", "getrawtransaction", &getrawtransaction, true, false, false}, + {"rawtransactions", "searchrawtransactions", &searchrawtransactions, true, false, false}, {"rawtransactions", "sendrawtransaction", &sendrawtransaction, false, false, false}, {"rawtransactions", "signrawtransaction", &signrawtransaction, false, false, false}, /* uses wallet if enabled */ diff --git a/src/rpcserver.h b/src/rpcserver.h index 8ff25a7..e115467 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -236,6 +236,7 @@ extern json_spirit::Value decoderawtransaction(const json_spirit::Array& params, extern json_spirit::Value decodescript(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value signrawtransaction(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value sendrawtransaction(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value searchrawtransactions(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getblockcount(const json_spirit::Array& params, bool fHelp); // in rpcblockchain.cpp extern json_spirit::Value getbestblockhash(const json_spirit::Array& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index a975d7b..38b4e4e 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -8,6 +8,7 @@ #include "main.h" #include "pow.h" +#include "random.h" #include "uint256.h" #include "accumulators.h" @@ -76,6 +77,10 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap& mapCoins, const uint256& hashBlock) CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CLevelDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) { + if (!Read('S', salt)) { + salt = GetRandHash(); + Write('S', salt); + } } bool CBlockTreeDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) @@ -184,6 +189,55 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::ReadAddrIndex(uint160 addrid, std::vector &list) { + boost::scoped_ptr pcursor(NewIterator()); + + uint64_t lookupid; + { + CHashWriter ss(SER_GETHASH, 0); + ss << salt; + ss << addrid; + lookupid = (ss.GetHash()).GetLow64(); + } + + CDataStream firstKey(SER_DISK, CLIENT_VERSION); + firstKey << make_pair('a', lookupid); + pcursor->Seek(firstKey.str()); + + for(pcursor->Seek(firstKey.str()); pcursor->Valid(); pcursor->Next()) { + leveldb::Slice key = pcursor->key(); + CDataStream ssKey(key.data(), key.data() + key.size(), SER_DISK, CLIENT_VERSION); + char id; + ssKey >> id; + if(id == 'a') { + uint64_t lid; + ssKey >> lid; + if(lid == lookupid) { + CExtDiskTxPos position; + ssKey >> position; + list.push_back(position); + } else { + break; + } + } else { + break; + } + } + return true; +} + +bool CBlockTreeDB::AddAddrIndex(const std::vector > &list) { + unsigned char foo[0]; + CLevelDBBatch batch; + for (std::vector >::const_iterator it=list.begin(); it!=list.end(); it++) { + CHashWriter ss(SER_GETHASH, 0); + ss << salt; + ss << it->first; + batch.Write(make_pair(make_pair('a', (ss.GetHash()).GetLow64()), it->second), FLATDATA(foo)); + } + return WriteBatch(batch, true); +} + bool CBlockTreeDB::WriteFlag(const std::string& name, bool fValue) { return Write(std::make_pair('F', name), fValue ? '1' : '0'); diff --git a/src/txdb.h b/src/txdb.h index d4c29db..e169dea 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -50,6 +50,7 @@ class CBlockTreeDB : public CLevelDBWrapper CBlockTreeDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); private: + uint256 salt; CBlockTreeDB(const CBlockTreeDB&); void operator=(const CBlockTreeDB&); @@ -63,6 +64,8 @@ class CBlockTreeDB : public CLevelDBWrapper bool ReadReindexing(bool& fReindex); bool ReadTxIndex(const uint256& txid, CDiskTxPos& pos); bool WriteTxIndex(const std::vector >& list); + bool ReadAddrIndex(uint160 addrid, std::vector &list); + bool AddAddrIndex(const std::vector > &list); bool WriteFlag(const std::string& name, bool fValue); bool ReadFlag(const std::string& name, bool& fValue); bool WriteInt(const std::string& name, int nValue);