Skip to content

Commit

Permalink
Merge branch 'expls' into 'master'
Browse files Browse the repository at this point in the history 
NetDB: Add aggressive LS expire

See merge request i2p-hackers/i2p.i2p!221
  • Loading branch information
zzz committed Dec 18, 2024
2 parents 6ffdb70 + 492056e commit 372dbce
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 28 deletions.
4 changes: 2 additions & 2 deletions core/java/src/net/i2p/data/LeaseSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,13 +185,13 @@ public void setSigningKey(SigningPublicKey key) {
}

/**
* Also sets receivedAsReply to true
* As of 0.9.65, no longer sets receivedAsReply to true
* @param localClient may be null
* @since 0.9.47
*/
public void setReceivedBy(Hash localClient) {
super.setReceivedBy(localClient);
super.setReceivedAsReply();
//setReceivedAsReply();
}

/**
Expand Down
27 changes: 17 additions & 10 deletions router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,13 @@
import net.i2p.router.JobImpl;
import net.i2p.router.LeaseSetKeys;
import net.i2p.router.MessageSelector;
import net.i2p.router.NetworkDatabaseFacade;
import net.i2p.router.ReplyJob;
import net.i2p.router.Router;
import net.i2p.router.RouterContext;
import net.i2p.router.TunnelInfo;
import net.i2p.router.crypto.ratchet.ReplyCallback;
import net.i2p.router.networkdb.kademlia.KademliaNetworkDatabaseFacade;
import net.i2p.util.Log;

/**
Expand Down Expand Up @@ -295,9 +297,10 @@ public void runJob() {
}

SendJob success = new SendJob(getContext());
KademliaNetworkDatabaseFacade kndf = (KademliaNetworkDatabaseFacade) getContext().clientNetDb(_from.calculateHash());
// set in constructor
if (_leaseSet != null) {
if (!_leaseSet.getReceivedAsReply()) {
if (!kndf.isClientDb() && !_leaseSet.getReceivedAsReply()) {
boolean shouldFetch = true;
if (_leaseSet.getType() != DatabaseEntry.KEY_TYPE_LEASESET) {
LeaseSet2 ls2 = (LeaseSet2) _leaseSet;
Expand All @@ -307,7 +310,7 @@ public void runJob() {
if (_log.shouldInfo())
_log.info(getJobId() + ": RAP LS, firing search: " + _leaseSet.getHash().toBase32());
LookupLeaseSetFailedJob failed = new LookupLeaseSetFailedJob(getContext());
getContext().clientNetDb(_from.calculateHash()).lookupLeaseSetRemotely(_leaseSet.getHash(), success, failed,
kndf.lookupLeaseSetRemotely(_leaseSet.getHash(), success, failed,
LS_LOOKUP_TIMEOUT, _from.calculateHash());
} else {
dieFatal(MessageStatusMessage.STATUS_SEND_FAILURE_NO_LEASESET);
Expand All @@ -330,7 +333,7 @@ public void runJob() {
long exp = now - _leaseSet.getLatestLeaseDate();
_log.info(getJobId() + ": leaseSet expired " + DataHelper.formatDuration(exp) + " ago, firing search: " + _leaseSet.getHash().toBase32());
}
getContext().clientNetDb(_from.calculateHash()).lookupLeaseSetRemotely(_leaseSet.getHash(), _from.calculateHash());
kndf.lookupLeaseSetRemotely(_leaseSet.getHash(), _from.calculateHash());
}
}
success.runJob();
Expand All @@ -340,7 +343,7 @@ public void runJob() {
_log.debug(getJobId() + ": Send outbound client message - sending off leaseSet lookup job for " + _toString + " from client " + _from.calculateHash().toBase32());
LookupLeaseSetFailedJob failed = new LookupLeaseSetFailedJob(getContext());
Hash key = _to.calculateHash();
getContext().clientNetDb(_from.calculateHash()).lookupLeaseSet(key, success, failed, LS_LOOKUP_TIMEOUT, _from.calculateHash());
kndf.lookupLeaseSet(key, success, failed, LS_LOOKUP_TIMEOUT, _from.calculateHash());
}
}

Expand Down Expand Up @@ -422,14 +425,18 @@ public void runJob() {
*/
private int getNextLease() {
// set in runJob if found locally
if (_leaseSet == null || !_leaseSet.getReceivedAsReply()) {
_leaseSet = getContext().clientNetDb(_from.calculateHash()).lookupLeaseSetLocally(_to.calculateHash());
KademliaNetworkDatabaseFacade kndf = (KademliaNetworkDatabaseFacade) getContext().clientNetDb(_from.calculateHash());
if (_leaseSet == null || (!kndf.isClientDb() && _leaseSet.getReceivedAsPublished())) {
if (_leaseSet == null) {
// shouldn't happen
if (_log.shouldLog(Log.WARN))
_log.warn(getJobId() + ": Lookup locally didn't find the leaseSet for " + _toString);
return MessageStatusMessage.STATUS_SEND_FAILURE_NO_LEASESET;
} else if (_leaseSet.getReceivedAsPublished()) {
_leaseSet = kndf.lookupLeaseSetLocally(_to.calculateHash());
if (_leaseSet == null) {
if (_log.shouldLog(Log.WARN))
_log.warn(getJobId() + ": Lookup locally didn't find the leaseSet for " + _toString);
return MessageStatusMessage.STATUS_SEND_FAILURE_NO_LEASESET;
}
}
if (!kndf.isClientDb() && _leaseSet.getReceivedAsPublished()) {
if (_log.shouldLog(Log.WARN))
_log.warn(getJobId() + ": Only have RAP LS for " + _toString);
return MessageStatusMessage.STATUS_SEND_FAILURE_NO_LEASESET;
Expand Down
91 changes: 78 additions & 13 deletions router/java/src/net/i2p/router/networkdb/kademlia/ExpireLeasesJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,21 @@
*/

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import net.i2p.data.DatabaseEntry;
import net.i2p.data.Hash;
import net.i2p.data.LeaseSet;
import net.i2p.data.router.RouterKeyGenerator;
import net.i2p.router.JobImpl;
import net.i2p.router.Router;
import net.i2p.router.RouterContext;
import net.i2p.util.Log;
import net.i2p.util.SystemVersion;

/**
* Periodically search through all leases to find expired ones, failing those
Expand All @@ -31,6 +36,8 @@ class ExpireLeasesJob extends JobImpl {
private final KademliaNetworkDatabaseFacade _facade;

private final static long RERUN_DELAY_MS = 1*60*1000;
private static final int LIMIT_LEASES_FF = 1250;
private static final int LIMIT_LEASES_CLIENT = SystemVersion.isSlow() ? 750 : 300;

public ExpireLeasesJob(RouterContext ctx, KademliaNetworkDatabaseFacade facade) {
super(ctx);
Expand All @@ -42,16 +49,13 @@ public ExpireLeasesJob(RouterContext ctx, KademliaNetworkDatabaseFacade facade)

public void runJob() {
List<Hash> toExpire = selectKeysToExpire();
for (Hash key : toExpire) {
_facade.fail(key);
//_log.info("Lease " + key + " is expiring, so lets look for it again", new Exception("Expire and search"));
//_facade.lookupLeaseSet(key, null, null, RERUN_DELAY_MS);
if (!toExpire.isEmpty()) {
for (Hash key : toExpire) {
_facade.fail(key);
}
if (_log.shouldInfo())
_log.info(_facade + " Leases expired: " + toExpire.size());
}
if (_log.shouldInfo())
_log.info("(dbid: " + _facade
+ "; db size: " + _facade.getKnownLeaseSets()
+ ") Leases expired: " + toExpire);
//_facade.queueForExploration(toExpire); // don't do explicit searches, just explore passively
requeue(RERUN_DELAY_MS);
}

Expand All @@ -62,19 +66,80 @@ public void runJob() {
*/
private List<Hash> selectKeysToExpire() {
RouterContext ctx = getContext();
List<Hash> toExpire = new ArrayList<Hash>(128);
for (Map.Entry<Hash, DatabaseEntry> entry : _facade.getDataStore().getMapEntries()) {
boolean isClient = _facade.isClientDb();
boolean isFFDB = _facade.floodfillEnabled() && !isClient;
Set<Map.Entry<Hash, DatabaseEntry>> entries = _facade.getDataStore().getMapEntries();
// clientdb only has leasesets
List<LeaseSet> current = new ArrayList<LeaseSet>(isFFDB ? 512 : (isClient ? entries.size() : 128));
List<Hash> toExpire = new ArrayList<Hash>(Math.min(entries.size(), 128));
int sz = 0;
for (Map.Entry<Hash, DatabaseEntry> entry : entries) {
DatabaseEntry obj = entry.getValue();
if (obj.isLeaseSet()) {
LeaseSet ls = (LeaseSet)obj;
Hash h = entry.getKey();
boolean isLocal = ctx.clientManager().isLocal(h);
if (!ls.isCurrent(Router.CLOCK_FUDGE_FACTOR)) {
Hash h = entry.getKey();
toExpire.add(h);
if (ctx.clientManager().isLocal(h))
if (isLocal)
_log.logAlways(Log.WARN, "Expired local leaseset " + h.toBase32());
} else if (!isLocal) {
// do not aggressive expire RAR LS but still count them
sz++;
if (!ls.getReceivedAsReply())
current.add(ls);
}
}
}
int origsz = sz;
int limit = isFFDB ? LIMIT_LEASES_FF : LIMIT_LEASES_CLIENT;
if (sz > limit) {
// aggressive drop strategy
if (isFFDB) {
RouterKeyGenerator gen = ctx.routerKeyGenerator();
byte[] ourRKey = ctx.routerHash().getData();
for (LeaseSet ls : current) {
Hash h = ls.getHash();
// don't drop very close to us
byte[] rkey = gen.getRoutingKey(h).getData();
int distance = (((rkey[0] ^ ourRKey[0]) & 0xff) << 8) |
((rkey[1] ^ ourRKey[1]) & 0xff);
// they have to be within 1/256 of the keyspace
if (distance >= 256) {
toExpire.add(h);
if (--sz <= limit)
break;
}
}
} else {
Collections.sort(current, new LeaseSetComparator());
for (LeaseSet ls : current) {
toExpire.add(ls.getHash());
//if (_log.shouldInfo())
// _log.info("Aggressive LS expire for " + _facade + '\n' + ls);
if (--sz <= limit)
break;
}
}
int exp = origsz - sz;
if (exp > 0 && _log.shouldWarn())
_log.warn("Aggressive LS expire for " + _facade + " removed " + exp +
" leasesets, limit " + limit + ", size now " + sz);
}
return toExpire;
}

/**
* Oldest first
* @since 0.9.65
*/
private static class LeaseSetComparator implements Comparator<LeaseSet> {
public int compare(LeaseSet l, LeaseSet r) {
long dl = l.getLatestLeaseDate();
long dr = r.getLatestLeaseDate();
if (dl < dr) return -1;
if (dl > dr) return 1;
return 0;
}
}
}
7 changes: 4 additions & 3 deletions ...er/java/src/net/i2p/router/networkdb/kademlia/HandleFloodfillDatabaseStoreMessageJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,9 +122,10 @@ public void runJob() {
// for it. This flag must NOT get set on entries that we
// receive in response to our own lookups.
// See ../HDLMJ for more info
if (!ls.getReceivedAsReply())
ls.setReceivedAsPublished();
if (_facade.isClientDb()) {
if (!_facade.isClientDb()) {
if (!ls.getReceivedAsReply())
ls.setReceivedAsPublished();
} else {
// This is where we deal with what happens if a client subDB tries to store
// a leaseSet which it is the owner/publisher of.
// Look up a ls hash in the netDbSegmentor, and compare it to the _facade that we have.
Expand Down

0 comments on commit 372dbce

Please sign in to comment.