From 281be67729c15ad0c3f2f5dac13ed1cc4ad70007 Mon Sep 17 00:00:00 2001 From: Mariano Sorgente Date: Thu, 9 Jan 2020 09:07:52 +0900 Subject: [PATCH] Alpha 1.2 (#64) Remove database access from blockchain, and handle headers instead of blocks Avoid processing blocks and unfinished blocks that we have already seen. Also adds test for load. Plotting improvements --- README.md | 12 +- lib/chiapos/src/calculate_bucket.hpp | 87 ++++----- lib/chiapos/src/verifier.hpp | 6 +- src/blockchain.py | 269 +++++++++++---------------- src/database.py | 24 ++- src/full_node.py | 58 ++++-- src/server/start_full_node.py | 35 +++- src/ui/prompt_ui.py | 16 +- tests/pytest.ini | 3 - tests/test_blockchain.py | 81 ++++---- tests/test_database.py | 13 ++ tests/test_node_load.py | 61 +++++- 12 files changed, 355 insertions(+), 310 deletions(-) delete mode 100644 tests/pytest.ini diff --git a/README.md b/README.md index 163316040689..87135e61a1e0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # chia-blockchain -Python 3.7 is used for this project. Make sure your default python version is >=3.7 by typing python3. +Python 3.7 is used for this project. Make sure your default python version is >=3.7 by typing python3. You will need to enable [UPnP](https://www.homenethowto.com/ports-and-nat/upnp-automatic-port-forward/) on your router or add a NAT (for IPv4 but not IPv6) and firewall rule to allow TCP port 8444 access to your peer. These methods tend to be router make/model specific. @@ -82,12 +82,12 @@ mongod --fork --dbpath ./db/ --logpath mongod.log ``` ### Windows (WSL + Ubuntu) -#### Install WSL + Ubuntu 18.04 LTS, upgrade to Ubuntu 19.x +#### Install WSL + Ubuntu 18.04 LTS, upgrade to Ubuntu 19.x This will require multiple reboots. From an Administrator PowerShell -`Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux` -and then -`Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform`. +`Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux` +and then +`Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform`. Once that is complete, install Ubuntu 18.04 LTS from the Windows Store. ```bash # Upgrade to 19.x @@ -260,7 +260,7 @@ Due to the nature of proof of space lookups by the harvester you should limit th on a physical drive to 50 or less. This limit should significantly increase before beta. You can also run the simulation, which runs all servers and multiple full nodes, locally, at once. -If you want to run the simulation, change the introducer ip in ./config/config.yaml so that the +If you want to run the simulation, change the introducer ip in ./config/config.yaml so that the full node points to the local introducer (127.0.0.1:8445). Note the the simulation is local only and requires installation of timelords and VDFs. diff --git a/lib/chiapos/src/calculate_bucket.hpp b/lib/chiapos/src/calculate_bucket.hpp index 9c22c5df8e96..ec00ea190a89 100644 --- a/lib/chiapos/src/calculate_bucket.hpp +++ b/lib/chiapos/src/calculate_bucket.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include "util.hpp" #include "bits.hpp" @@ -60,20 +61,22 @@ std::map kVectorLens = { {8, 0} }; -// Precomputed shifts that specify which entries match with which other entries -// in adjacent buckets. -uint16_t matching_shifts_c[2][kC]; - -// Performs the precomputation of shifts. -void precompute_shifts() { +uint32_t L_targets[2][kBC][kExtraBitsPow]; +bool initialized = false; +void load_tables() +{ for (uint8_t parity = 0; parity < 2; parity++) { - for (uint16_t r = 0; r < kExtraBitsPow; r++) { - uint16_t v = (uint16_t)pow((2 * r + parity), 2) % kC; - matching_shifts_c[parity][r] = v; + for (uint16_t i = 0; i < kBC; i++) { + uint16_t indJ = i / kC; + for (uint16_t m = 0; m < kExtraBitsPow; m++) { + uint16_t yr = ((indJ + m) % kB) * kC + (((2*m + parity) * (2*m + parity) + i) % kC); + L_targets[parity][i][m] = yr; + } } } } + // Class to evaluate F1 class F1Calculator { public: @@ -87,9 +90,6 @@ class F1Calculator { // Loads the key into the global AES context aes_load_key(this->aes_key_, 32); - - // Precomputes the shifts, this is only done once - precompute_shifts(); } inline ~F1Calculator() { @@ -243,15 +243,13 @@ class FxCalculator { // for these f functions (as opposed to f1, which uses a 32 byte key). Note that, however, // block sizes are still 128 bits (32 bytes). aes_load_key(this->aes_key_, 16); - - // One time precomputation of the shifts - precompute_shifts(); - - // Preallocates vector to be used for matching - for (uint16_t i = 0; i < kC; i++) { + for (uint16_t i = 0; i < kBC; i++) { std::vector new_vec; - this->R_positions.push_back(new_vec); - this->R_bids.push_back(new_vec); + this->rmap.push_back(new_vec); + } + if(!initialized) { + initialized = true; + load_tables(); } } @@ -352,43 +350,32 @@ class FxCalculator { inline std::vector> FindMatches(const std::vector& bucket_L, const std::vector& bucket_R) { std::vector> matches; - for (uint16_t i = 0; i < kC; i++) { - this->R_bids[i].clear(); - this->R_positions[i].clear(); - } uint16_t parity = (bucket_L[0].y / kBC) % 2; + for (uint16_t i = 0; i < rmap_clean.size(); i++) { + uint16_t yl = rmap_clean[i]; + this->rmap[yl].clear(); + } + rmap_clean.clear(); + + uint64_t remove = (bucket_R[0].y / kBC) * kBC; for (uint16_t pos_R = 0; pos_R < bucket_R.size(); pos_R++) { - R_bids[bucket_R[pos_R].y % kC].push_back((bucket_R[pos_R].y % kBC) / kC); - R_positions[bucket_R[pos_R].y % kC].push_back(pos_R); + uint64_t r_y = bucket_R[pos_R].y - remove; + rmap[r_y].push_back(pos_R); + rmap_clean.push_back(r_y); } + uint64_t remove_y = remove - kBC; for (uint16_t pos_L = 0; pos_L < bucket_L.size(); pos_L++) { - uint16_t yl_bid = (bucket_L[pos_L].y % kBC) / kC; - uint16_t yl_cid = bucket_L[pos_L].y % kC; - for (uint8_t m = 0; m < kExtraBitsPow; m++) { - uint16_t target_bid = (yl_bid + m); - uint16_t target_cid = yl_cid + matching_shifts_c[parity][m]; - - // This is faster than % - if (target_bid >= kB) { - target_bid -= kB; - } - if (target_cid >= kC) { - target_cid -= kC; - } - - for (uint32_t i = 0; i < R_bids[target_cid].size(); i++) { - uint16_t R_bid = R_bids[target_cid][i]; - if (target_bid == R_bid) { - uint64_t yl_bucket = bucket_L[pos_L].y / kBC; - if (yl_bucket + 1 == bucket_R[R_positions[target_cid][i]].y / kBC) { - matches.push_back(std::make_pair(pos_L, R_positions[target_cid][i])); - } - } + uint64_t r = bucket_L[pos_L].y - remove_y; + for (uint8_t i = 0; i < kExtraBitsPow; i++) { + uint16_t r_target = L_targets[parity][r][i]; + for (uint8_t j = 0; j < rmap[r_target].size(); j++) { + matches.push_back(std::make_pair(pos_L, rmap[r_target][j])); } } } + return matches; } @@ -402,8 +389,8 @@ class FxCalculator { uint8_t block_3[kBlockSizeBits/8]; uint8_t block_4[kBlockSizeBits/8]; uint8_t ciphertext[kBlockSizeBits/8]; - std::vector > R_positions; - std::vector > R_bids; + std::vector> rmap; + std::vector rmap_clean; }; #endif // SRC_CPP_CALCULATE_BUCKET_HPP_ diff --git a/lib/chiapos/src/verifier.hpp b/lib/chiapos/src/verifier.hpp index 350bbdd6a584..b809affe5183 100644 --- a/lib/chiapos/src/verifier.hpp +++ b/lib/chiapos/src/verifier.hpp @@ -79,9 +79,13 @@ class Verifier { vector bucket_R = {r_plot_entry}; // If there is no match, fails. - if (f.FindMatches(bucket_L, bucket_R).size() != 1) { + uint64_t cdiff = r_plot_entry.y / kBC - l_plot_entry.y / kBC; + if (cdiff != 1) { + return LargeBits(); + } else if (f.FindMatches(bucket_L, bucket_R).size() != 1) { return LargeBits(); } + std::pair results = f.CalculateBucket(ys[i], ys[i+1], metadata[i], metadata[i+1]); new_ys.push_back(std::get<0>(results)); diff --git a/src/blockchain.py b/src/blockchain.py index 5525c5849818..e076e926fd9e 100644 --- a/src/blockchain.py +++ b/src/blockchain.py @@ -11,7 +11,6 @@ calculate_ips_from_iterations, calculate_iterations_quality, ) -from src.database import FullNodeStore from src.types.full_block import FullBlock from src.types.header_block import HeaderBlock from src.types.sized_bytes import bytes32 @@ -38,54 +37,48 @@ class ReceiveBlockResult(Enum): class Blockchain: - def __init__(self, store: FullNodeStore, override_constants: Dict = {}): + def __init__(self, override_constants: Dict = {}): # Allow passing in custom overrides for any consesus parameters self.constants: Dict = consensus_constants for key, value in override_constants.items(): self.constants[key] = value - self.store = store - self.tips: List[FullBlock] = [] - self.lca_block: FullBlock + self.tips: List[HeaderBlock] = [] + self.lca_block: HeaderBlock # Defines the path from genesis to the tip self.height_to_hash: Dict[uint32, bytes32] = {} # All headers (but not orphans) from genesis to the tip are guaranteed to be in header_blocks self.header_blocks: Dict[bytes32, HeaderBlock] = {} - async def initialize(self): - seen_blocks = {} - async for block in self.store.get_blocks(): - if not self.tips or block.weight > self.tips[0].weight: - self.tips = [block] - seen_blocks[block.header_hash] = block + async def initialize(self, header_blocks: Dict[str, HeaderBlock]): + self.genesis = FullBlock.from_bytes(self.constants["GENESIS_BLOCK"]) - if len(self.tips) > 0: - curr = self.tips[0] - reverse_blocks = [curr] - while curr.height > 0: - curr = seen_blocks[curr.prev_header_hash] - reverse_blocks.append(curr) + result = await self.receive_block(self.genesis) + if result != ReceiveBlockResult.ADDED_TO_HEAD: + raise InvalidGenesisBlock() - for block in reversed(reverse_blocks): - self.height_to_hash[block.height] = block.header_hash - self.header_blocks[block.header_hash] = block.header_block - - self.lca_block = self.tips[0] - - else: - self.genesis = FullBlock.from_bytes(self.constants["GENESIS_BLOCK"]) - - result = await self.receive_block(self.genesis) - if result != ReceiveBlockResult.ADDED_TO_HEAD: - raise InvalidGenesisBlock() - assert self.lca_block + assert self.lca_block is not None + if len(header_blocks) > 0: + self.header_blocks = header_blocks + for _, header_block in self.header_blocks.items(): + self.height_to_hash[header_block.height] = header_block.header_hash + await self._reconsider_heads(header_block, False) + assert ( + self.header_blocks[self.height_to_hash[uint32(0)]] + == self.genesis.header_block + ) + if len(header_blocks) > 1: + assert ( + self.header_blocks[self.height_to_hash[uint32(1)]].prev_header_hash + == self.genesis.header_hash + ) def get_current_tips(self) -> List[HeaderBlock]: """ Return the heads. """ - return [b.header_block for b in self.tips] + return self.tips[:] def is_child_of_head(self, block: FullBlock): """ @@ -96,16 +89,6 @@ def is_child_of_head(self, block: FullBlock): return True return False - async def get_block(self, header_hash: bytes32) -> Optional[FullBlock]: - return await self.store.get_block(header_hash) - - async def get_header_block(self, header_hash: bytes32) -> Optional[HeaderBlock]: - bl = await self.store.get_block(header_hash) - if bl: - return bl.header_block - else: - return None - def get_header_hashes(self, tip_header_hash: bytes32) -> List[bytes32]: if tip_header_hash not in self.header_blocks: raise ValueError("Invalid tip requested") @@ -152,7 +135,7 @@ def find_fork_point(self, alternate_chain: List[bytes32]) -> uint32: Takes in an alternate blockchain (headers), and compares it to self. Returns the last header where both blockchains are equal. """ - lca: HeaderBlock = self.lca_block.header_block + lca: HeaderBlock = self.lca_block if lca.height >= len(alternate_chain) - 1: raise ValueError("Alternate chain is shorter") @@ -179,14 +162,12 @@ def find_fork_point(self, alternate_chain: List[bytes32]) -> uint32: else: raise ValueError("Invalid genesis block") - async def get_next_difficulty(self, header_hash: bytes32) -> uint64: + def get_next_difficulty(self, header_hash: bytes32) -> uint64: """ Returns the difficulty of the next block that extends onto header_hash. Used to calculate the number of iterations. """ - block = await self.store.get_block(header_hash) - if block is None: - raise Exception("Given header_hash must reference block already added") + block: HeaderBlock = self.header_blocks[header_hash] next_height: uint32 = uint32(block.height + 1) if next_height < self.constants["DIFFICULTY_EPOCH"]: @@ -201,16 +182,13 @@ async def get_next_difficulty(self, header_hash: bytes32) -> uint64: != self.constants["DIFFICULTY_DELAY"] ): # Not at a point where difficulty would change - prev_block = await self.store.get_block(block.prev_header_hash) - assert block.header_block.challenge is not None - assert ( - prev_block is not None and prev_block.header_block.challenge is not None - ) + prev_block: HeaderBlock = self.header_blocks[block.prev_header_hash] + assert block.challenge is not None + assert prev_block is not None and prev_block.challenge is not None if prev_block is None: raise Exception("Previous block is invalid.") return uint64( - block.header_block.challenge.total_weight - - prev_block.header_block.challenge.total_weight + block.challenge.total_weight - prev_block.challenge.total_weight ) # old diff curr diff new diff @@ -232,13 +210,10 @@ async def get_next_difficulty(self, header_hash: bytes32) -> uint64: # current difficulty block1, block2, block3 = None, None, None - if ( - block.header_block not in self.get_current_tips() - or height3 not in self.height_to_hash - ): + if block not in self.get_current_tips() or height3 not in self.height_to_hash: # This means we are either on a fork, or on one of the chains, but after the LCA, # so we manually backtrack. - curr: Optional[FullBlock] = block + curr: Optional[HeaderBlock] = block assert curr is not None while ( curr.height not in self.height_to_hash @@ -250,36 +225,34 @@ async def get_next_difficulty(self, header_hash: bytes32) -> uint64: block2 = curr elif curr.height == height3: block3 = curr - curr = await self.store.get_block(curr.prev_header_hash) + curr = self.header_blocks.get(curr.prev_header_hash, None) assert curr is not None # Once we are before the fork point (and before the LCA), we can use the height_to_hash map if not block1 and height1 >= 0: # height1 could be -1, for the first difficulty calculation - block1 = await self.store.get_block(self.height_to_hash[height1]) + block1 = self.header_blocks[self.height_to_hash[height1]] if not block2: - block2 = await self.store.get_block(self.height_to_hash[height2]) + block2 = self.header_blocks[self.height_to_hash[height2]] if not block3: - block3 = await self.store.get_block(self.height_to_hash[height3]) + block3 = self.header_blocks[self.height_to_hash[height3]] assert block2 is not None and block3 is not None # Current difficulty parameter (diff of block h = i - 1) - Tc = await self.get_next_difficulty(block.prev_header_hash) + Tc = self.get_next_difficulty(block.prev_header_hash) # Previous difficulty parameter (diff of block h = i - 2048 - 1) - Tp = await self.get_next_difficulty(block2.prev_header_hash) + Tp = self.get_next_difficulty(block2.prev_header_hash) if block1: - timestamp1 = block1.header_block.header.data.timestamp # i - 512 - 1 + timestamp1 = block1.header.data.timestamp # i - 512 - 1 else: # In the case of height == -1, there is no timestamp here, so assume the genesis block # took constants["BLOCK_TIME_TARGET"] seconds to mine. - genesis = await self.store.get_block(self.height_to_hash[uint32(0)]) - assert genesis is not None + genesis = self.header_blocks[self.height_to_hash[uint32(0)]] timestamp1 = ( - genesis.header_block.header.data.timestamp - - self.constants["BLOCK_TIME_TARGET"] + genesis.header.data.timestamp - self.constants["BLOCK_TIME_TARGET"] ) - timestamp2 = block2.header_block.header.data.timestamp # i - 2048 + 512 - 1 - timestamp3 = block3.header_block.header.data.timestamp # i - 512 - 1 + timestamp2 = block2.header.data.timestamp # i - 2048 + 512 - 1 + timestamp3 = block3.header.data.timestamp # i - 512 - 1 # Numerator fits in 128 bits, so big int is not necessary # We multiply by the denominators here, so we only have one fraction in the end (avoiding floating point) @@ -319,28 +292,26 @@ async def get_next_difficulty(self, header_hash: bytes32) -> uint64: ] ) - async def get_next_ips(self, header_hash) -> uint64: + def get_next_ips(self, header_hash) -> uint64: """ Returns the VDF speed in iterations per seconds, to be used for the next block. This depends on the number of iterations of the last epoch, and changes at the same block as the difficulty. """ - block = await self.store.get_block(header_hash) - if block is None or block.header_block.challenge is None: - raise Exception("Given header_hash must reference block already added") + block: HeaderBlock = self.header_blocks[header_hash] + assert block.challenge is not None next_height: uint32 = uint32(block.height + 1) if next_height < self.constants["DIFFICULTY_EPOCH"]: # First epoch has a hardcoded vdf speed return self.constants["VDF_IPS_STARTING"] - prev_block = await self.store.get_block(block.prev_header_hash) - if prev_block is None or prev_block.header_block.challenge is None: - raise Exception("Previous block is invalid.") - proof_of_space = block.header_block.proof_of_space - difficulty = await self.get_next_difficulty(prev_block.header_hash) + prev_block: HeaderBlock = self.header_blocks[block.prev_header_hash] + assert prev_block.challenge is not None + + proof_of_space = block.proof_of_space + difficulty = self.get_next_difficulty(prev_block.header_hash) iterations = uint64( - block.header_block.challenge.total_iters - - prev_block.header_block.challenge.total_iters + block.challenge.total_iters - prev_block.challenge.total_iters ) prev_ips = calculate_ips_from_iterations( proof_of_space, difficulty, iterations, self.constants["MIN_BLOCK_TIME"] @@ -369,15 +340,12 @@ async def get_next_ips(self, header_hash) -> uint64: # Height2 is the last block in the previous epoch height2 = uint32(next_height - self.constants["DIFFICULTY_DELAY"] - 1) - block1: Optional[FullBlock] = None - block2: Optional[FullBlock] = None - if ( - block.header_block not in self.get_current_tips() - or height2 not in self.height_to_hash - ): + block1: Optional[HeaderBlock] = None + block2: Optional[HeaderBlock] = None + if block not in self.get_current_tips() or height2 not in self.height_to_hash: # This means we are either on a fork, or on one of the chains, but after the LCA, # so we manually backtrack. - curr: Optional[FullBlock] = block + curr: Optional[HeaderBlock] = block assert curr is not None while ( curr.height not in self.height_to_hash @@ -387,35 +355,33 @@ async def get_next_ips(self, header_hash) -> uint64: block1 = curr elif curr.height == height2: block2 = curr - curr = await self.store.get_block(curr.prev_header_hash) + curr = self.header_blocks.get(curr.prev_header_hash, None) assert curr is not None # Once we are before the fork point (and before the LCA), we can use the height_to_hash map if block1 is None and height1 >= 0: # height1 could be -1, for the first difficulty calculation - block1 = await self.store.get_block(self.height_to_hash[height1]) + block1 = self.header_blocks.get(self.height_to_hash[height1], None) if block2 is None: - block2 = await self.store.get_block(self.height_to_hash[height2]) + block2 = self.header_blocks.get(self.height_to_hash[height2], None) assert block2 is not None - assert block2.header_block.challenge is not None + assert block2.challenge is not None if block1 is not None: - assert block1.header_block.challenge is not None - timestamp1 = block1.header_block.header.data.timestamp - iters1 = block1.header_block.challenge.total_iters + assert block1.challenge is not None + timestamp1 = block1.header.data.timestamp + iters1 = block1.challenge.total_iters else: # In the case of height == -1, there is no timestamp here, so assume the genesis block # took constants["BLOCK_TIME_TARGET"] seconds to mine. - genesis = await self.store.get_block(self.height_to_hash[uint32(0)]) - assert genesis is not None + genesis: HeaderBlock = self.header_blocks[self.height_to_hash[uint32(0)]] timestamp1 = ( - genesis.header_block.header.data.timestamp - - self.constants["BLOCK_TIME_TARGET"] + genesis.header.data.timestamp - self.constants["BLOCK_TIME_TARGET"] ) - assert genesis.header_block.challenge is not None - iters1 = genesis.header_block.challenge.total_iters + assert genesis.challenge is not None + iters1 = genesis.challenge.total_iters - timestamp2 = block2.header_block.header.data.timestamp - iters2 = block2.header_block.challenge.total_iters + timestamp2 = block2.header.data.timestamp + iters2 = block2.challenge.total_iters new_ips = uint64((iters2 - iters1) // (timestamp2 - timestamp1)) @@ -434,21 +400,19 @@ async def receive_block(self, block: FullBlock) -> ReceiveBlockResult: """ genesis: bool = block.height == 0 and not self.tips - if await self.store.get_block(block.header_hash) is not None: + if block.header_hash in self.header_blocks: return ReceiveBlockResult.ALREADY_HAVE_BLOCK - if await self.store.get_block(block.prev_header_hash) is None and not genesis: + if block.prev_header_hash not in self.header_blocks and not genesis: return ReceiveBlockResult.DISCONNECTED_BLOCK if not await self.validate_block(block, genesis): return ReceiveBlockResult.INVALID_BLOCK - # Block is valid and connected, so it can be added to the blockchain. - await self.store.add_block(block) # Cache header in memory self.header_blocks[block.header_hash] = block.header_block - if await self._reconsider_heads(block, genesis): + if await self._reconsider_heads(block.header_block, genesis): return ReceiveBlockResult.ADDED_TO_HEAD else: return ReceiveBlockResult.ADDED_AS_ORPHAN @@ -462,27 +426,27 @@ async def validate_unfinished_block( and challenge validation. """ # 1. Check previous pointer(s) / flyclient - if not genesis and await self.store.get_block(block.prev_header_hash) is None: + if not genesis and block.prev_header_hash not in self.header_blocks: return False # 2. Check Now+2hrs > timestamp > avg timestamp of last 11 blocks - prev_block: Optional[FullBlock] = None + prev_block: Optional[HeaderBlock] = None if not genesis: # TODO: do something about first 11 blocks last_timestamps: List[uint64] = [] - prev_block = await self.store.get_block(block.prev_header_hash) - if not prev_block or not prev_block.header_block: + prev_block = self.header_blocks.get(block.prev_header_hash, None) + if not prev_block: return False curr = prev_block while len(last_timestamps) < self.constants["NUMBER_OF_TIMESTAMPS"]: - last_timestamps.append(curr.header_block.header.data.timestamp) - fetched = await self.store.get_block(curr.prev_header_hash) + last_timestamps.append(curr.header.data.timestamp) + fetched = self.header_blocks.get(curr.prev_header_hash, None) if not fetched: break curr = fetched if ( len(last_timestamps) != self.constants["NUMBER_OF_TIMESTAMPS"] - and curr.body.coinbase.height != 0 + and curr.height != 0 ): return False prev_time: uint64 = uint64(int(sum(last_timestamps) / len(last_timestamps))) @@ -513,8 +477,8 @@ async def validate_unfinished_block( challenge_hash: bytes32 if not genesis: assert prev_block - assert prev_block.header_block.challenge - challenge_hash = prev_block.header_block.challenge.get_hash() + assert prev_block.challenge + challenge_hash = prev_block.challenge.get_hash() # 8. Check challenge hash of prev is the same as in pos if challenge_hash != block.header_block.proof_of_space.challenge_hash: @@ -538,10 +502,10 @@ async def validate_unfinished_block( if not pos_quality: return False - # 11. Check coinbase height = parent coinbase height + 1 + # 11. Check coinbase height = prev height + 1 if not genesis: assert prev_block - if block.body.coinbase.height != prev_block.body.coinbase.height + 1: + if block.body.coinbase.height != prev_block.height + 1: return False else: if block.body.coinbase.height != 0: @@ -582,8 +546,8 @@ async def validate_block(self, block: FullBlock, genesis: bool = False) -> bool: difficulty: uint64 ips: uint64 if not genesis: - difficulty = await self.get_next_difficulty(block.prev_header_hash) - ips = await self.get_next_ips(block.prev_header_hash) + difficulty = self.get_next_difficulty(block.prev_header_hash) + ips = self.get_next_ips(block.prev_header_hash) else: difficulty = uint64(self.constants["DIFFICULTY_STARTING"]) ips = uint64(self.constants["VDF_IPS_STARTING"]) @@ -624,37 +588,34 @@ async def validate_block(self, block: FullBlock, genesis: bool = False) -> bool: return False if not genesis: - prev_block: Optional[FullBlock] = await self.store.get_block( - block.prev_header_hash + prev_block: Optional[HeaderBlock] = self.header_blocks.get( + block.prev_header_hash, None ) - if not prev_block or not prev_block.header_block.challenge: + if not prev_block or not prev_block.challenge: return False # 5. and check if PoT.challenge_hash matches if ( block.header_block.proof_of_time.challenge_hash - != prev_block.header_block.challenge.get_hash() + != prev_block.challenge.get_hash() ): return False # 6a. Check challenge height = parent height + 1 - if ( - block.header_block.challenge.height - != prev_block.header_block.challenge.height + 1 - ): + if block.header_block.challenge.height != prev_block.challenge.height + 1: return False # 7a. Check challenge total_weight = parent total_weight + difficulty if ( block.header_block.challenge.total_weight - != prev_block.header_block.challenge.total_weight + difficulty + != prev_block.challenge.total_weight + difficulty ): return False # 8a. Check challenge total_iters = parent total_iters + number_iters if ( block.header_block.challenge.total_iters - != prev_block.header_block.challenge.total_iters + number_of_iters + != prev_block.challenge.total_iters + number_of_iters ): return False else: @@ -672,65 +633,47 @@ async def validate_block(self, block: FullBlock, genesis: bool = False) -> bool: return True - async def _reconsider_heights( - self, old_lca: Optional[FullBlock], new_lca: FullBlock - ): + def _reconsider_heights(self, old_lca: Optional[HeaderBlock], new_lca: HeaderBlock): """ Update the mapping from height to block hash, when the lca changes. """ - curr_old: Optional[HeaderBlock] = old_lca.header_block if old_lca else None - curr_new: HeaderBlock = new_lca.header_block + curr_old: Optional[HeaderBlock] = old_lca if old_lca else None + curr_new: HeaderBlock = new_lca while True: - fetched: Optional[FullBlock] + fetched: Optional[HeaderBlock] if not curr_old or curr_old.height < curr_new.height: self.height_to_hash[uint32(curr_new.height)] = curr_new.header_hash self.header_blocks[curr_new.header_hash] = curr_new if curr_new.height == 0: return - fetched = await self.store.get_block(curr_new.prev_header_hash) - assert fetched is not None - curr_new = fetched.header_block + curr_new = self.header_blocks[curr_new.prev_header_hash] elif curr_old.height > curr_new.height: del self.height_to_hash[uint32(curr_old.height)] - fetched = await self.store.get_block(curr_old.prev_header_hash) - assert fetched is not None - curr_old = fetched.header_block + curr_old = self.header_blocks[curr_old.prev_header_hash] else: if curr_new.header_hash == curr_old.header_hash: return self.height_to_hash[uint32(curr_new.height)] = curr_new.header_hash - self.header_blocks[curr_new.header_hash] = curr_new - fetched_new: Optional[FullBlock] = await self.store.get_block( - curr_new.prev_header_hash - ) - fetched_old: Optional[FullBlock] = await self.store.get_block( - curr_old.prev_header_hash - ) - assert fetched_new is not None and fetched_old is not None - curr_new = fetched_new.header_block - curr_old = fetched_old.header_block + curr_new = self.header_blocks[curr_new.prev_header_hash] + curr_old = self.header_blocks[curr_old.prev_header_hash] async def _reconsider_lca(self, genesis: bool): """ Update the least common ancestor of the heads. This is useful, since we can just assume there is one block per height before the LCA (and use the height_to_hash dict). """ - cur: List[FullBlock] = self.tips[:] + cur: List[HeaderBlock] = self.tips[:] while any(b.header_hash != cur[0].header_hash for b in cur): heights = [b.height for b in cur] i = heights.index(max(heights)) - fetched: Optional[FullBlock] = await self.store.get_block( - cur[i].prev_header_hash - ) - assert fetched is not None - cur[i] = fetched + cur[i] = self.header_blocks[cur[i].prev_header_hash] if genesis: - await self._reconsider_heights(None, cur[0]) + self._reconsider_heights(None, cur[0]) else: - await self._reconsider_heights(self.lca_block, cur[0]) + self._reconsider_heights(self.lca_block, cur[0]) self.lca_block = cur[0] - async def _reconsider_heads(self, block: FullBlock, genesis: bool) -> bool: + async def _reconsider_heads(self, block: HeaderBlock, genesis: bool) -> bool: """ When a new block is added, this is called, to check if the new block is heavier than one of the heads. diff --git a/src/database.py b/src/database.py index 24b87d7836d6..19ede6c8486e 100644 --- a/src/database.py +++ b/src/database.py @@ -79,7 +79,11 @@ def __init__(self, db_name): ] = {} # Blocks which are not finalized yet (no proof of time), old ones are cleared self.unfinished_blocks: Dict[Tuple[bytes32, uint64], FullBlock] = {} - # Blocks which we have received but our blockchain dose not reach, old ones are cleared + # Header hashes of unfinished blocks that we have seen recently + self.seen_unfinished_blocks: set = set() + # Header hashes of blocks that we have seen recently + self.seen_blocks: set = set() + # Blocks which we have received but our blockchain does not reach, old ones are cleared self.disconnected_blocks: Dict[bytes32, FullBlock] = {} # Lock @@ -223,6 +227,24 @@ async def get_unfinished_block( ) -> Optional[FullBlock]: return self.unfinished_blocks.get(key, None) + def seen_unfinished_block(self, header_hash: bytes32) -> bool: + if header_hash in self.seen_unfinished_blocks: + return True + self.seen_unfinished_blocks.add(header_hash) + return False + + def clear_seen_unfinished_blocks(self) -> None: + self.seen_unfinished_blocks.clear() + + def seen_block(self, header_hash: bytes32) -> bool: + if header_hash in self.seen_blocks: + return True + self.seen_blocks.add(header_hash) + return False + + def clear_seen_blocks(self) -> None: + self.seen_blocks.clear() + async def get_unfinished_blocks(self) -> Dict[Tuple[bytes32, uint64], FullBlock]: return self.unfinished_blocks.copy() diff --git a/src/full_node.py b/src/full_node.py index 514db650139b..71eff8f98800 100644 --- a/src/full_node.py +++ b/src/full_node.py @@ -76,7 +76,7 @@ async def _send_tips_to_farmers( height = tip.challenge.height quality = tip.proof_of_space.verify_and_get_quality() if tip.height > 0: - difficulty: uint64 = await self.blockchain.get_next_difficulty( + difficulty: uint64 = self.blockchain.get_next_difficulty( tip.prev_header_hash ) else: @@ -86,7 +86,7 @@ async def _send_tips_to_farmers( challenge_hash, height, tip.weight, quality, difficulty ) ) - proof_of_time_rate: uint64 = await self.blockchain.get_next_ips( + proof_of_time_rate: uint64 = self.blockchain.get_next_ips( tips[0].header_hash ) rate_update = farmer_protocol.ProofOfTimeRate(proof_of_time_rate) @@ -148,7 +148,7 @@ async def _on_connect(self) -> OutboundMessageGenerator: async with self.store.lock: heads: List[HeaderBlock] = self.blockchain.get_current_tips() for h in heads: - block = await self.blockchain.get_block(h.header.get_hash()) + block = await self.store.get_block(h.header.get_hash()) assert block blocks.append(block) for block in blocks: @@ -498,12 +498,14 @@ async def _sync(self) -> OutboundMessageGenerator: log.info( f"Took {time.time() - start} seconds to validate and add block {block.height}." ) + # Always immediately add the block to the database, after updating blockchain state + await self.store.add_block(block) assert ( max([h.height for h in self.blockchain.get_current_tips()]) >= height ) await self.store.set_proof_of_time_estimate_ips( - await self.blockchain.get_next_ips(block.header_hash) + self.blockchain.get_next_ips(block.header_hash) ) assert max([h.height for h in self.blockchain.get_current_tips()]) == tip_height log.info(f"Finished sync up to height {tip_height}") @@ -612,7 +614,7 @@ async def request_sync_blocks( Responsd to a peers request for syncing blocks. """ blocks: List[FullBlock] = [] - tip_block: Optional[FullBlock] = await self.blockchain.get_block( + tip_block: Optional[FullBlock] = await self.store.get_block( request.tip_header_hash ) if tip_block is not None: @@ -629,9 +631,7 @@ async def request_sync_blocks( request.heights, request.tip_header_hash ) for header_block in header_blocks: - fetched = await self.blockchain.get_block( - header_block.header.get_hash() - ) + fetched = await self.store.get_block(header_block.header.get_hash()) assert fetched blocks.append(fetched) except KeyError: @@ -812,13 +812,14 @@ async def proof_of_time_finished( f"Received a proof of time that we cannot use to complete a block {dict_key}" ) return - prev_block: Optional[HeaderBlock] = await self.blockchain.get_header_block( + prev_full_block = await self.store.get_block( unfinished_block_obj.prev_header_hash ) - difficulty: uint64 = await self.blockchain.get_next_difficulty( + assert prev_full_block + prev_block: HeaderBlock = prev_full_block.header_block + difficulty: uint64 = self.blockchain.get_next_difficulty( unfinished_block_obj.prev_header_hash ) - assert prev_block assert prev_block.challenge challenge: Challenge = Challenge( @@ -894,23 +895,30 @@ async def unfinished_block( We can validate it and if it's a good block, propagate it to other peers and timelords. """ + # Adds the unfinished block to seen, and check if it's seen before + if self.store.seen_unfinished_block(unfinished_block.block.header_hash): + return + if not self.blockchain.is_child_of_head(unfinished_block.block): return if not await self.blockchain.validate_unfinished_block(unfinished_block.block): raise InvalidUnfinishedBlock() - prev_block: Optional[HeaderBlock] = await self.blockchain.get_header_block( + prev_full_block: Optional[FullBlock] = await self.store.get_block( unfinished_block.block.prev_header_hash ) - assert prev_block + assert prev_full_block + + prev_block: HeaderBlock = prev_full_block.header_block + assert prev_block.challenge challenge_hash: bytes32 = prev_block.challenge.get_hash() - difficulty: uint64 = await self.blockchain.get_next_difficulty( + difficulty: uint64 = self.blockchain.get_next_difficulty( unfinished_block.block.header_block.prev_header_hash ) - vdf_ips: uint64 = await self.blockchain.get_next_ips( + vdf_ips: uint64 = self.blockchain.get_next_ips( unfinished_block.block.header_block.prev_header_hash ) @@ -990,6 +998,10 @@ async def block(self, block: peer_protocol.Block) -> OutboundMessageGenerator: """ header_hash = block.block.header_block.header.get_hash() + # Adds the block to seen, and check if it's seen before + if self.store.seen_block(header_hash): + return + async with self.store.lock: if await self.store.get_sync_mode(): # Add the block to our potential tips list @@ -998,6 +1010,13 @@ async def block(self, block: peer_protocol.Block) -> OutboundMessageGenerator: # Tries to add the block to the blockchain added: ReceiveBlockResult = await self.blockchain.receive_block(block.block) + + # Always immediately add the block to the database, after updating blockchain state + if ( + added == ReceiveBlockResult.ADDED_AS_ORPHAN + or added == ReceiveBlockResult.ADDED_TO_HEAD + ): + await self.store.add_block(block.block) if added == ReceiveBlockResult.ALREADY_HAVE_BLOCK: return elif added == ReceiveBlockResult.INVALID_BLOCK: @@ -1058,12 +1077,10 @@ async def block(self, block: peer_protocol.Block) -> OutboundMessageGenerator: f"Updated heads, new heights: {[b.height for b in self.blockchain.get_current_tips()]}" ) - difficulty = await self.blockchain.get_next_difficulty( + difficulty = self.blockchain.get_next_difficulty( block.block.prev_header_hash ) - next_vdf_ips = await self.blockchain.get_next_ips( - block.block.header_hash - ) + next_vdf_ips = self.blockchain.get_next_ips(block.block.header_hash) log.info(f"Difficulty {difficulty} IPS {next_vdf_ips}") if next_vdf_ips != await self.store.get_proof_of_time_estimate_ips(): await self.store.set_proof_of_time_estimate_ips(next_vdf_ips) @@ -1076,6 +1093,8 @@ async def block(self, block: peer_protocol.Block) -> OutboundMessageGenerator: Message("proof_of_time_rate", rate_update), Delivery.BROADCAST, ) + self.store.clear_seen_unfinished_blocks() + self.store.clear_seen_blocks() assert block.block.header_block.proof_of_time assert block.block.header_block.challenge @@ -1126,6 +1145,7 @@ async def block(self, block: peer_protocol.Block) -> OutboundMessageGenerator: assert False # Recursively process the next block if we have it + # This code path is reached if added == ADDED_AS_ORPHAN or ADDED_TO_HEAD async with self.store.lock: next_block: Optional[ FullBlock diff --git a/src/server/start_full_node.py b/src/server/start_full_node.py index 281df78f33a9..532d2eec6cf2 100644 --- a/src/server/start_full_node.py +++ b/src/server/start_full_node.py @@ -4,14 +4,18 @@ import sys import miniupnpc import uvloop +from typing import Dict, List from src.blockchain import Blockchain from src.database import FullNodeStore from src.full_node import FullNode from src.server.outbound_message import NodeType from src.server.server import ChiaServer +from src.consensus.constants import constants from src.types.peer_info import PeerInfo +from src.types.header_block import HeaderBlock from src.util.network import parse_host_port +from src.types.full_block import FullBlock logging.basicConfig( format="FullNode %(name)-23s: %(levelname)-8s %(asctime)s.%(msecs)03d %(message)s", @@ -23,6 +27,29 @@ server_closed = False +async def load_header_blocks_from_store( + store: FullNodeStore, +) -> Dict[str, HeaderBlock]: + seen_blocks: Dict[str, HeaderBlock] = {} + tips: List[HeaderBlock] = [] + async for full_block in store.get_blocks(): + if not tips or full_block.weight > tips[0].weight: + tips = [full_block.header_block] + seen_blocks[full_block.header_hash] = full_block.header_block + + header_blocks = {} + if len(tips) > 0: + curr: HeaderBlock = tips[0] + reverse_blocks: List[HeaderBlock] = [curr] + while curr.height > 0: + curr = seen_blocks[curr.prev_header_hash] + reverse_blocks.append(curr) + + for block in reversed(reverse_blocks): + header_blocks[block.header_hash] = block + return header_blocks + + async def main(): # Create the store (DB) and full node instance db_id = 0 @@ -30,9 +57,13 @@ async def main(): db_id = int(sys.argv[sys.argv.index("-id") + 1]) store = FullNodeStore(f"fndb_{db_id}") - blockchain = Blockchain(store) + genesis: FullBlock = FullBlock.from_bytes(constants["GENESIS_BLOCK"]) + await store.add_block(genesis) + log.info("Initializing blockchain from disk") - await blockchain.initialize() + header_blocks: Dict[str, HeaderBlock] = await load_header_blocks_from_store(store) + blockchain = Blockchain() + await blockchain.initialize(header_blocks) full_node = FullNode(store, blockchain) # Starts the full node server (which full nodes can connect to) diff --git a/src/ui/prompt_ui.py b/src/ui/prompt_ui.py index 4b88db731f6d..c5ce9bb00451 100644 --- a/src/ui/prompt_ui.py +++ b/src/ui/prompt_ui.py @@ -353,18 +353,14 @@ def inner(): self.syncing.text = "Not syncing" heads: List[HeaderBlock] = self.blockchain.get_current_tips() - lca_block: FullBlock = self.blockchain.lca_block + lca_block: HeaderBlock = self.blockchain.lca_block if lca_block.height > 0: - difficulty = await self.blockchain.get_next_difficulty( - lca_block.prev_header_hash - ) - ips = await self.blockchain.get_next_ips(lca_block.prev_header_hash) + difficulty = self.blockchain.get_next_difficulty(lca_block.prev_header_hash) + ips = self.blockchain.get_next_ips(lca_block.prev_header_hash) else: - difficulty = await self.blockchain.get_next_difficulty( - lca_block.header_hash - ) - ips = await self.blockchain.get_next_ips(lca_block.header_hash) - total_iters = lca_block.header_block.challenge.total_iters + difficulty = self.blockchain.get_next_difficulty(lca_block.header_hash) + ips = self.blockchain.get_next_ips(lca_block.header_hash) + total_iters = lca_block.challenge.total_iters new_block_labels = [] for i, b in enumerate(self.latest_blocks): diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index 06b6d52d01cc..000000000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -log_cli=true -log_level=NOTSET \ No newline at end of file diff --git a/tests/test_blockchain.py b/tests/test_blockchain.py index b48b15c2cf22..77099d2b6578 100644 --- a/tests/test_blockchain.py +++ b/tests/test_blockchain.py @@ -7,7 +7,6 @@ from src.blockchain import Blockchain, ReceiveBlockResult from src.consensus.constants import constants -from src.database import FullNodeStore from src.types.body import Body from src.types.coinbase import CoinbaseInfo from src.types.full_block import FullBlock @@ -43,10 +42,8 @@ def event_loop(): class TestGenesisBlock: @pytest.mark.asyncio async def test_basic_blockchain(self): - store = FullNodeStore("fndb_test") - await store._clear_database() - bc1: Blockchain = Blockchain(store) - await bc1.initialize() + bc1: Blockchain = Blockchain() + await bc1.initialize({}) assert len(bc1.get_current_tips()) == 1 genesis_block = bc1.get_current_tips()[0] assert genesis_block.height == 0 @@ -55,9 +52,9 @@ async def test_basic_blockchain(self): bc1.get_header_blocks_by_height([uint32(0)], genesis_block.header_hash) )[0] == genesis_block assert ( - await bc1.get_next_difficulty(genesis_block.header_hash) + bc1.get_next_difficulty(genesis_block.header_hash) ) == genesis_block.challenge.total_weight - assert await bc1.get_next_ips(genesis_block.header_hash) > 0 + assert bc1.get_next_ips(genesis_block.header_hash) > 0 class TestBlockValidation: @@ -66,11 +63,9 @@ async def initial_blockchain(self): """ Provides a list of 10 valid blocks, as well as a blockchain with 9 blocks added to it. """ - store = FullNodeStore("fndb_test") - await store._clear_database() blocks = bt.get_consecutive_blocks(test_constants, 10, [], 10) - b: Blockchain = Blockchain(store, test_constants) - await b.initialize() + b: Blockchain = Blockchain(test_constants) + await b.initialize({}) for i in range(1, 9): assert ( await b.receive_block(blocks[i]) @@ -246,37 +241,33 @@ async def test_difficulty_change(self): # Make it 5x faster than target time blocks = bt.get_consecutive_blocks(test_constants, num_blocks, [], 2) - store = FullNodeStore("fndb_test") - await store._clear_database() - b: Blockchain = Blockchain(store, test_constants) - await b.initialize() + b: Blockchain = Blockchain(test_constants) + await b.initialize({}) for i in range(1, num_blocks): assert ( await b.receive_block(blocks[i]) ) == ReceiveBlockResult.ADDED_TO_HEAD - diff_25 = await b.get_next_difficulty(blocks[24].header_hash) - diff_26 = await b.get_next_difficulty(blocks[25].header_hash) - diff_27 = await b.get_next_difficulty(blocks[26].header_hash) + diff_25 = b.get_next_difficulty(blocks[24].header_hash) + diff_26 = b.get_next_difficulty(blocks[25].header_hash) + diff_27 = b.get_next_difficulty(blocks[26].header_hash) assert diff_26 == diff_25 assert diff_27 > diff_26 assert (diff_27 / diff_26) <= test_constants["DIFFICULTY_FACTOR"] - assert (await b.get_next_ips(blocks[1].header_hash)) == constants[ - "VDF_IPS_STARTING" - ] - assert (await b.get_next_ips(blocks[24].header_hash)) == ( - await b.get_next_ips(blocks[23].header_hash) + assert (b.get_next_ips(blocks[1].header_hash)) == constants["VDF_IPS_STARTING"] + assert (b.get_next_ips(blocks[24].header_hash)) == ( + b.get_next_ips(blocks[23].header_hash) ) - assert (await b.get_next_ips(blocks[25].header_hash)) == ( - await b.get_next_ips(blocks[24].header_hash) + assert (b.get_next_ips(blocks[25].header_hash)) == ( + b.get_next_ips(blocks[24].header_hash) ) - assert (await b.get_next_ips(blocks[26].header_hash)) > ( - await b.get_next_ips(blocks[25].header_hash) + assert (b.get_next_ips(blocks[26].header_hash)) > ( + b.get_next_ips(blocks[25].header_hash) ) - assert (await b.get_next_ips(blocks[27].header_hash)) == ( - await b.get_next_ips(blocks[26].header_hash) + assert (b.get_next_ips(blocks[27].header_hash)) == ( + b.get_next_ips(blocks[26].header_hash) ) @@ -284,10 +275,8 @@ class TestReorgs: @pytest.mark.asyncio async def test_basic_reorg(self): blocks = bt.get_consecutive_blocks(test_constants, 100, [], 9) - store = FullNodeStore("fndb_test") - await store._clear_database() - b: Blockchain = Blockchain(store, test_constants) - await b.initialize() + b: Blockchain = Blockchain(test_constants) + await b.initialize({}) for block in blocks: await b.receive_block(block) @@ -309,10 +298,8 @@ async def test_basic_reorg(self): @pytest.mark.asyncio async def test_reorg_from_genesis(self): blocks = bt.get_consecutive_blocks(test_constants, 20, [], 9, b"0") - store = FullNodeStore("fndb_test") - await store._clear_database() - b: Blockchain = Blockchain(store, test_constants) - await b.initialize() + b: Blockchain = Blockchain(test_constants) + await b.initialize({}) for block in blocks: await b.receive_block(block) assert b.get_current_tips()[0].height == 20 @@ -348,34 +335,30 @@ async def test_reorg_from_genesis(self): @pytest.mark.asyncio async def test_lca(self): blocks = bt.get_consecutive_blocks(test_constants, 5, [], 9, b"0") - store = FullNodeStore("fndb_test") - await store._clear_database() - b: Blockchain = Blockchain(store, test_constants) - await b.initialize() + b: Blockchain = Blockchain(test_constants) + await b.initialize({}) for block in blocks: await b.receive_block(block) - assert b.lca_block == blocks[3] + assert b.lca_block == blocks[3].header_block block_5_2 = bt.get_consecutive_blocks(test_constants, 1, blocks[:5], 9, b"1")[5] block_5_3 = bt.get_consecutive_blocks(test_constants, 1, blocks[:5], 9, b"2")[5] await b.receive_block(block_5_2) - assert b.lca_block == blocks[4] + assert b.lca_block == blocks[4].header_block await b.receive_block(block_5_3) - assert b.lca_block == blocks[4] + assert b.lca_block == blocks[4].header_block reorg = bt.get_consecutive_blocks(test_constants, 6, [], 9, b"3") for block in reorg: await b.receive_block(block) - assert b.lca_block == blocks[0] + assert b.lca_block == blocks[0].header_block @pytest.mark.asyncio async def test_get_header_hashes(self): blocks = bt.get_consecutive_blocks(test_constants, 5, [], 9, b"0") - store = FullNodeStore("fndb_test") - await store._clear_database() - b: Blockchain = Blockchain(store, test_constants) - await b.initialize() + b: Blockchain = Blockchain(test_constants) + await b.initialize({}) for block in blocks: await b.receive_block(block) header_hashes = b.get_header_hashes(blocks[-1].header_hash) diff --git a/tests/test_database.py b/tests/test_database.py index 9504af61089c..a19b8f2bf5fd 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -1,12 +1,14 @@ import asyncio import pytest +from secrets import token_bytes from typing import Any, Dict from tests.block_tools import BlockTools from src.util.ints import uint32, uint64 from src.consensus.constants import constants from src.database import FullNodeStore from src.types.full_block import FullBlock +from src.types.sized_bytes import bytes32 bt = BlockTools() @@ -103,3 +105,14 @@ async def test_basic_database(self): await db.clear_disconnected_blocks_below(uint32(5)) assert await db.get_disconnected_block(blocks[4].prev_header_hash) is None + + h_hash_1 = bytes32(token_bytes(32)) + assert not db.seen_unfinished_block(h_hash_1) + assert db.seen_unfinished_block(h_hash_1) + db.clear_seen_unfinished_blocks() + assert not db.seen_unfinished_block(h_hash_1) + + assert not db.seen_block(h_hash_1) + assert db.seen_block(h_hash_1) + db.clear_seen_blocks() + assert not db.seen_block(h_hash_1) diff --git a/tests/test_node_load.py b/tests/test_node_load.py index 6d4a150c5986..0d458ecfd102 100644 --- a/tests/test_node_load.py +++ b/tests/test_node_load.py @@ -13,11 +13,12 @@ from src.server.outbound_message import OutboundMessage, Message, Delivery from src.util.ints import uint16 + bt = BlockTools() test_constants: Dict[str, Any] = { "DIFFICULTY_STARTING": 5, - "DISCRIMINANT_SIZE_BITS": 16, + "DISCRIMINANT_SIZE_BITS": 32, "BLOCK_TIME_TARGET": 10, "MIN_BLOCK_TIME": 2, "DIFFICULTY_FACTOR": 3, @@ -42,12 +43,14 @@ async def test1(self): store = FullNodeStore("fndb_test") await store._clear_database() blocks = bt.get_consecutive_blocks(test_constants, 10, [], 10) - b: Blockchain = Blockchain(store, test_constants) - await b.initialize() + b: Blockchain = Blockchain(test_constants) + await store.add_block(blocks[0]) + await b.initialize({}) for i in range(1, 9): assert ( await b.receive_block(blocks[i]) ) == ReceiveBlockResult.ADDED_TO_HEAD + await store.add_block(blocks[i]) full_node_1 = FullNode(store, b) server_1 = ChiaServer(21234, full_node_1, NodeType.FULL_NODE) @@ -60,13 +63,11 @@ async def test1(self): await server_2.start_client(PeerInfo("127.0.0.1", uint16(21234)), None) - print("Starting sleep") await asyncio.sleep(2) # Allow connections to get made num_unfinished_blocks = 1000 start_unf = time.time() for i in range(num_unfinished_blocks): - print("Pushing") msg = Message("unfinished_block", peer_protocol.UnfinishedBlock(blocks[9])) server_1.push_message( OutboundMessage(NodeType.FULL_NODE, msg, Delivery.BROADCAST) @@ -78,7 +79,7 @@ async def test1(self): OutboundMessage(NodeType.FULL_NODE, block_msg, Delivery.BROADCAST) ) - while time.time() - start_unf < 100: + while time.time() - start_unf < 300: if max([h.height for h in b.get_current_tips()]) == 9: print( f"Time taken to process {num_unfinished_blocks} is {time.time() - start_unf}" @@ -95,3 +96,51 @@ async def test1(self): await server_1.await_closed() await server_2.await_closed() raise Exception("Took too long to process blocks") + + @pytest.mark.asyncio + async def test2(self): + num_blocks = 100 + store = FullNodeStore("fndb_test") + await store._clear_database() + blocks = bt.get_consecutive_blocks(test_constants, num_blocks, [], 10) + b: Blockchain = Blockchain(test_constants) + await store.add_block(blocks[0]) + await b.initialize({}) + + full_node_1 = FullNode(store, b) + server_1 = ChiaServer(21236, full_node_1, NodeType.FULL_NODE) + _ = await server_1.start_server("127.0.0.1", None) + full_node_1._set_server(server_1) + + full_node_2 = FullNode(store, b) + server_2 = ChiaServer(21237, full_node_2, NodeType.FULL_NODE) + full_node_2._set_server(server_2) + + await server_2.start_client(PeerInfo("127.0.0.1", uint16(21236)), None) + + await asyncio.sleep(2) # Allow connections to get made + + start_unf = time.time() + for i in range(1, num_blocks): + msg = Message("block", peer_protocol.Block(blocks[i])) + server_1.push_message( + OutboundMessage(NodeType.FULL_NODE, msg, Delivery.BROADCAST) + ) + + while time.time() - start_unf < 300: + if max([h.height for h in b.get_current_tips()]) == num_blocks - 1: + print( + f"Time taken to process {num_blocks} is {time.time() - start_unf}" + ) + server_1.close_all() + server_2.close_all() + await server_1.await_closed() + await server_2.await_closed() + return + await asyncio.sleep(0.1) + + server_1.close_all() + server_2.close_all() + await server_1.await_closed() + await server_2.await_closed() + raise Exception("Took too long to process blocks")