From 0e7a1aee0ad17f086f7c9a55b68ba0f08ad18c80 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 28 Jul 2022 22:03:38 -0300 Subject: [PATCH] dht_crawler: clean in memory set for expired peers --- scripts/dht_crawler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/dht_crawler.py b/scripts/dht_crawler.py index 3766b3e995..d7d2d19790 100644 --- a/scripts/dht_crawler.py +++ b/scripts/dht_crawler.py @@ -71,7 +71,10 @@ async def open(self): self.db.writer_connection.row_factory = dict_row_factory async def all_peers(self): - return [DHTPeer(**peer) for peer in await self.db.execute_fetchall("select * from peer")] + return [ + DHTPeer(**peer) for peer in await self.db.execute_fetchall( + "select * from peer where latency > 0 or last_seen < datetime('now', '-1h')") + ] async def save_peers(self, *peers): log.info("Saving graph nodes (peers) to DB") @@ -263,6 +266,11 @@ def get_peers_needing_check(self): to_check = [peer for peer in self.all_peers if peer.last_check is None or peer.last_check < self.refresh_limit] return to_check + def remove_expired_peers(self): + for key, peer in list(self._memory_peers.items()): + if (peer.latency or 0) < 1 and peer.last_seen < self.refresh_limit: + del self._memory_peers[key] + def add_peers(self, *peers): for peer in peers: db_peer = self.get_from_peer(peer) @@ -278,6 +286,7 @@ async def flush_to_db(self): connections_to_save = self._connections self._connections = {} # await self.db.save_connections(connections_to_save) heavy call + self.remove_expired_peers() def get_from_peer(self, peer): return self._memory_peers.get((peer.address, peer.udp_port), None)