Merge branch 'i2p.i2p.2.4.0-client-netdbs-on-client-runner' into 'mas…

…ter'

Router: move segmented netDbs into CCRs.

Closes #404, #423, #420, and #439

See merge request i2p-hackers/i2p.i2p!120

apps/routerconsole/java/src/net/i2p/router/web/helpers/ConfigKeyringHandler.java

@@ -90,11 +90,11 @@ protected void processForm() {
                         return;
                     }
                     // from BlindCache
-                    List<String> clientBase32s = _context.netDbSegmentor().lookupClientBySigningPublicKey(spk);
+                    List<Hash> clientBase32s = _context.netDbSegmentor().lookupClientBySigningPublicKey(spk);
                     // TODO: This updates all of the blind data for all clients, turning the blind cache into a shared context for the owner of an encrypted leaseSet.
                     // This is probably not ideal, with some social-engineering a service operator who owns an encrypted destination could associate 2 tunnels.
                     // How realistic is it? Maybe not very, but I don't like it. Still, this is better than nothing.
-                    for (String clientBase32 : clientBase32s) {
+                    for (Hash clientBase32 : clientBase32s) {
                         BlindData bdold = _context.clientNetDb(clientBase32).getBlindData(spk);
                         if (bdold != null && d == null)
                             d = bdold.getDestination();

apps/routerconsole/java/src/net/i2p/router/web/helpers/NetDbHelper.java

@@ -6,11 +6,14 @@
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 
 import net.i2p.crypto.EncType;
 import net.i2p.crypto.SigType;
 import net.i2p.data.DataHelper;
+import net.i2p.data.Hash;
+import net.i2p.data.router.RouterInfo;
 import net.i2p.util.SystemVersion;
 import net.i2p.router.sybil.Analysis;
 import net.i2p.router.web.FormHandler;
@@ -327,7 +330,7 @@ public String getFloodfillNetDbSummary() {
         return getNetDbSummary(null, false);
     }
 
-    public String getNetDbSummary(String client, boolean clientOnly) {
+    public String getNetDbSummary(Hash client, boolean clientOnly) {
         NetDbRenderer renderer = new NetDbRenderer(_context);
         try {
             if (client == null && !clientOnly)
@@ -359,7 +362,7 @@ else if ((_mode == 13 || _mode == 16) && !_postOK)
             } else if (_full == 6) {
                 renderer.renderStatusHTML(_out, _limit, _page, _full, null, true);
             } else if (_clientOnly && client == null) {
-                for (String _client : _context.netDbSegmentor().getClients()) {
+                for (Hash _client : _context.clientManager().getPrimaryHashes()) {
                     renderer.renderLeaseSetHTML(_out, _debug, _client, clientOnly);
                 }
             } else {
@@ -373,7 +376,7 @@ else if ((_mode == 13 || _mode == 16) && !_postOK)
         return "";
     }
 
-    public String getClientNetDbSummary(String client) {
+    public String getClientNetDbSummary(Hash client) {
         return getNetDbSummary(client, true);
     }
 
@@ -428,6 +431,10 @@ private void renderNavBar() throws IOException {
                 continue;   // can't nav to lookup
             if (i > 2 && i != tab && !isAdvanced())
                 continue;
+            if (i == 10 || i == 11) {
+                if (_context.netDbSegmentor().getRoutersKnownToClients().size() == 0)
+                    continue;
+            }
             if (i == tab) {
                 // we are there
                 if (span)

apps/routerconsole/java/src/net/i2p/router/web/helpers/NetDbRenderer.java

@@ -127,15 +127,19 @@ public void renderRouterInfoHTML(Writer out, int pageSize, int page,
                                      String country, String family, String caps,
                                      String ip, String sybil, int port, int highPort, SigType type, EncType etype,
                                      String mtu, String ipv6, String ssucaps,
-                                     String tr, int cost, int icount, String client, boolean allClients) throws IOException {
+                                     String tr, int cost, int icount, Hash client, boolean allClients) throws IOException {
         StringBuilder buf = new StringBuilder(4*1024);
         List<Hash> sybils = sybil != null ? new ArrayList<Hash>(128) : null;
         FloodfillNetworkDatabaseFacade netdb = _context.netDb();
         if (allClients) {
             netdb = _context.netDb();
         }else{
-            if (client != null)
+            if (client != null) {
+                Log _log = _context.logManager().getLog(NetDbRenderer.class);
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("client subdb for: " + client);
                 netdb = _context.clientNetDb(client);
+            }
             else
                 netdb = _context.netDb();
         }
@@ -608,7 +612,7 @@ private static boolean hasCap(RouterInfo ri, String caps) {
      *  @param debug @since 0.7.14 sort by distance from us, display
      *               median distance, and other stuff, useful when floodfill
      */
-    public void renderLeaseSetHTML(Writer out, boolean debug, String client, boolean clientsOnly) throws IOException {
+    public void renderLeaseSetHTML(Writer out, boolean debug, Hash client, boolean clientsOnly) throws IOException {
         StringBuilder buf = new StringBuilder(4*1024);
         if (debug)
             buf.append("<p id=\"debugmode\">Debug mode - Sorted by hash distance, closest first</p>\n");
@@ -619,8 +623,12 @@ public void renderLeaseSetHTML(Writer out, boolean debug, String client, boolean
         if (clientsOnly){
             netdb = _context.netDb();
         }else{
-            if (client != null)
+            if (client != null) {
+                Log _log = _context.logManager().getLog(NetDbRenderer.class);
+                if (_log.shouldLog(Log.DEBUG))
+                    _log.debug("client subdb for: " + client);
                 netdb = _context.clientNetDb(client);
+            }
             else
                 netdb = _context.netDb();
         }
@@ -635,8 +643,9 @@ public void renderLeaseSetHTML(Writer out, boolean debug, String client, boolean
         }
         if (clientsOnly)
             leases.addAll(_context.netDbSegmentor().getLeasesKnownToClients());
-        else
+        else{
             leases.addAll(netdb.getLeases());
+        }
         int medianCount = 0;
         int rapCount = 0;
         BigInteger median = null;
@@ -951,7 +960,7 @@ private void renderLeaseSet(StringBuilder buf, LeaseSet ls, boolean debug, long
      *  @param mode 0: charts only; 1: full routerinfos; 2: abbreviated routerinfos
      *         mode 3: Same as 0 but sort countries by count
      */
-    public void renderStatusHTML(Writer out, int pageSize, int page, int mode, String client, boolean clientsOnly) throws IOException {
+    public void renderStatusHTML(Writer out, int pageSize, int page, int mode, Hash client, boolean clientsOnly) throws IOException {
         if (!_context.netDb().isInitialized()) {
             out.write("<div id=\"notinitialized\">");
             out.write(_t("Not initialized"));

apps/routerconsole/java/src/net/i2p/router/web/helpers/SummaryHelper.java

@@ -632,7 +632,7 @@ public String getDestinations() {
                 else
                     buf.append(DataHelper.escapeHTML(ServletUtil.truncate(name, 29))).append("…");
                 buf.append("</a></b></td>\n");
-                LeaseSet ls = _context.netDbSegmentor().lookupLeaseSetHashIsClient(h);
+                LeaseSet ls = _context.clientNetDb(client.calculateHash()).lookupLeaseSetLocally(h);
                 if (ls != null && _context.tunnelManager().getOutboundClientTunnelCount(h) > 0) {
                     if (!ls.isCurrent(0)) {
                         // yellow light

router/java/src/net/i2p/router/ClientManagerFacade.java

@@ -20,6 +20,7 @@
 import net.i2p.data.LeaseSet;
 import net.i2p.data.i2cp.MessageId;
 import net.i2p.data.i2cp.SessionConfig;
+import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseFacade;
 
 /**
  * Manage all interactions with clients 
@@ -121,4 +122,30 @@ public void registerMetaDest(Destination dest) throws I2PSessionException {}
      *  @since 0.9.41
      */
     public void unregisterMetaDest(Destination dest) {}
+
+    /**
+     * get the FloodfillNetworkDatabaseFacade associated with a particular client destination.
+     * This is inside the runner, so it won't be there if the runner isn't ready.
+     * 
+     * @param destHash destination hash associated with the client who's subDb we're looking for
+     * @return non-null FloodfillNetworkDatabaseFacade
+     * @since 0.9.60
+     */
+    public abstract FloodfillNetworkDatabaseFacade getClientFloodfillNetworkDatabaseFacade(Hash destHash);
+
+    /**
+     * get all of the FloodfillNetworkDatabaseFacades for all of the clients.
+     * 
+     * @return non-null set of FloodfillNetworkDatabaseFacades
+     * @since 0.9.60
+     */
+    public abstract Set<FloodfillNetworkDatabaseFacade> getClientFloodfillNetworkDatabaseFacades();
+
+    /**
+     * get a set of all primary hashes
+     * 
+     * @return non-null set of Hashes
+     * @since 0.9.60
+     */
+    public abstract Set<Hash> getPrimaryHashes();
 }

router/java/src/net/i2p/router/RouterContext.java

@@ -375,7 +375,6 @@ public Hash routerHash() {
     public SegmentedNetworkDatabaseFacade netDbSegmentor() { return _netDb; }
     public FloodfillNetworkDatabaseFacade netDb() { return _netDb.mainNetDB(); }
     public FloodfillNetworkDatabaseFacade multihomeNetDb() { return _netDb.multiHomeNetDB(); }
-    public FloodfillNetworkDatabaseFacade clientNetDb(String id) { return _netDb.clientNetDB(id); }
     public FloodfillNetworkDatabaseFacade clientNetDb(Hash id) { return _netDb.clientNetDB(id); }
     /**
      * The actual driver of the router, where all jobs are enqueued and processed.

router/java/src/net/i2p/router/RouterVersion.java

@@ -20,7 +20,7 @@ public class RouterVersion {
     public final static String VERSION = CoreVersion.VERSION;
     /** for example: "beta", "alpha", "rc" */
     public final static String STATUS = "";
-    public final static long BUILD = 4;
+    public final static long BUILD = 5;
     /** for example "-test" */
     public final static String EXTRA = "";
     public final static String FULL_VERSION = VERSION + "-" + STATUS + BUILD + EXTRA;

router/java/src/net/i2p/router/client/ClientConnectionRunner.java

@@ -49,6 +49,8 @@
 import net.i2p.router.RouterContext;
 import net.i2p.router.crypto.TransientSessionKeyManager;
 import net.i2p.router.crypto.ratchet.RatchetSKM;
+import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseFacade;
+import net.i2p.router.networkdb.kademlia.FloodfillNetworkDatabaseSegmentor;
 import net.i2p.router.crypto.ratchet.MuxedSKM;
 import net.i2p.util.ConcurrentHashSet;
 import net.i2p.util.I2PThread;
@@ -90,6 +92,8 @@ class ClientConnectionRunner {
     protected I2CPMessageReader _reader;
     /** Used for all sessions, which must all have the same crypto keys */
     private SessionKeyManager _sessionKeyManager;
+    /** Used for leaseSets sent to and recieved from this client */
+    private FloodfillNetworkDatabaseFacade _floodfillNetworkDatabaseFacade;
     /** 
      * This contains the last 10 MessageIds that have had their (non-ack) status 
      * delivered to the client (so that we can be sure only to update when necessary)
@@ -156,6 +160,8 @@ public ClientConnectionRunner(RouterContext context, ClientManager manager, Sock
         _alreadyProcessed = new ArrayList<MessageId>();
         _acceptedPending = new ConcurrentHashSet<MessageId>();
         _messageId = new AtomicInteger(_context.random().nextInt());
+        // Set up the per-destination FloodfillNetworkDatabaseFacade to prevent clients from being able to
+        // update leaseSet entries in the floodfill netDb
     }
 
     private static final AtomicInteger __id = new AtomicInteger();
@@ -207,23 +213,25 @@ public synchronized void stopRunning() {
         _acceptedPending.clear();
         if (_sessionKeyManager != null)
             _sessionKeyManager.shutdown();
+        if (_floodfillNetworkDatabaseFacade != null)
+            if (_floodfillNetworkDatabaseFacade.isClientDb())
+                _floodfillNetworkDatabaseFacade.shutdown();
         if (_encryptedLSHash != null)
             _manager.unregisterEncryptedDestination(this, _encryptedLSHash);
         _manager.unregisterConnection(this);
         // netdb may be null in unit tests
-        Hash dbid = getDestHash();
-        if (_context.netDbSegmentor() != null) {
+        if (_context.netDb() != null) {
             // Note that if the client sent us a destroy message,
             // removeSession() was called just before this, and
             // _sessions will be empty.
             for (SessionParams sp : _sessions.values()) {
                 LeaseSet ls = sp.currentLeaseSet;
-                if (ls != null)
-                    _context.clientNetDb(dbid).unpublish(ls);
+                if (ls != null && getFloodfillNetworkDatabaseFacade() != null)
+                    getFloodfillNetworkDatabaseFacade().unpublish(ls);
                 // unpublish encrypted LS also
                 ls = sp.currentEncryptedLeaseSet;
-                if (ls != null)
-                    _context.clientNetDb(dbid).unpublish(ls);
+                if (ls != null && getFloodfillNetworkDatabaseFacade() != null)
+                    getFloodfillNetworkDatabaseFacade().unpublish(ls);
                 if (!sp.isPrimary)
                     _context.tunnelManager().removeAlias(sp.dest);
             }
@@ -459,12 +467,12 @@ void removeSession(SessionId id) {
                 // Tell client manger
                 _manager.unregisterSession(id, sp.dest);
                 LeaseSet ls = sp.currentLeaseSet;
-                if (ls != null)
-                    _context.clientNetDb(dbid).unpublish(ls);
+                if (ls != null && getFloodfillNetworkDatabaseFacade() != null)
+                    getFloodfillNetworkDatabaseFacade().unpublish(ls);
                 // unpublish encrypted LS also
                 ls = sp.currentEncryptedLeaseSet;
-                if (ls != null)
-                    _context.clientNetDb(dbid).unpublish(ls);
+                if (ls != null && getFloodfillNetworkDatabaseFacade() != null)
+                    getFloodfillNetworkDatabaseFacade().unpublish(ls);
                 isPrimary = sp.isPrimary;
                 if (isPrimary)
                     _context.tunnelManager().removeTunnels(sp.dest);
@@ -484,12 +492,12 @@ void removeSession(SessionId id) {
                     _log.info("Destroying remaining client subsession " + sp.sessionId);
                 _manager.unregisterSession(sp.sessionId, sp.dest);
                 LeaseSet ls = sp.currentLeaseSet;
-                if (ls != null)
-                    _context.clientNetDb(dbid).unpublish(ls);
+                if (ls != null && getFloodfillNetworkDatabaseFacade() != null)
+                    getFloodfillNetworkDatabaseFacade().unpublish(ls);
                 // unpublish encrypted LS also
                 ls = sp.currentEncryptedLeaseSet;
-                if (ls != null)
-                    _context.clientNetDb(dbid).unpublish(ls);
+                if (ls != null && getFloodfillNetworkDatabaseFacade() != null)
+                    getFloodfillNetworkDatabaseFacade().unpublish(ls);
                 _context.tunnelManager().removeAlias(sp.dest);
                 synchronized(this) {
                     if (sp.rerequestTimer != null)
@@ -564,6 +572,18 @@ void removePayload(MessageId id) {
     public int sessionEstablished(SessionConfig config) {
         Destination dest = config.getDestination();
         Hash destHash = dest.calculateHash();
+        if (destHash != null){
+            if (_log.shouldLog(Log.DEBUG)) {
+                _log.debug("Initializing subDb for client" + destHash);
+            }
+            _floodfillNetworkDatabaseFacade = new FloodfillNetworkDatabaseFacade(_context, destHash);
+            _floodfillNetworkDatabaseFacade.startup();
+        } else {
+            if (_log.shouldLog(Log.ERROR)) {
+                _log.error("Initializing subDb for unknown client" + dest, new Exception());
+            }
+            _floodfillNetworkDatabaseFacade = null;
+        }
         if (_log.shouldLog(Log.DEBUG))
             _log.debug("SessionEstablished called for destination " + destHash);
         if (_sessions.size() > MAX_SESSIONS)
@@ -590,7 +610,6 @@ public int sessionEstablished(SessionConfig config) {
             _dontSendMSM = "none".equals(opts.getProperty(I2PClient.PROP_RELIABILITY, "").toLowerCase(Locale.US));
             _dontSendMSMOnReceive = Boolean.parseBoolean(opts.getProperty(I2PClient.PROP_FAST_RECEIVE));
         }
-
         // Set up the
         // per-destination session key manager to prevent rather easy correlation
         // based on the specified encryption types in the config
@@ -1150,6 +1169,33 @@ private boolean alreadyAccepted(MessageId id) {
      */
     private final static long REQUEUE_DELAY = 500;
     private static final int MAX_REQUEUE = 60;  // 30 sec.
+
+    /**
+     * Get the FloodfillNetworkDatabaseFacade for this runner. This is the client
+     * netDb if the router is configured to use subDbs, or the main netDb if the
+     * router is configured to use a monolithic netDb.
+     * 
+     * If neither a client netDb or the main netDb is available, it will return null.
+     * This should be impossible.
+     * If you get the  `getFloodfillNetworkDatabaseFacade is null for runner` warning,
+     * the main netDb will be returned instead. If the main netDb is null, then null
+     * will be returned.
+     * 
+     * @return _floodfillNetworkDatabaseFacade
+     * @since 0.9.60
+     */
+    public FloodfillNetworkDatabaseFacade getFloodfillNetworkDatabaseFacade() {
+        if (!_context.netDbSegmentor().useSubDbs())
+            return _context.netDb();
+        if (_log.shouldLog(Log.DEBUG))
+            _log.debug("getFloodfillNetworkDatabaseFacade is getting the subDb for dbid: " + this.getDestHash());
+        if (_floodfillNetworkDatabaseFacade == null) {
+            if (_log.shouldLog(Log.ERROR))
+                _log.error("getFloodfillNetworkDatabaseFacade is null for runner", new Exception());
+            return _context.netDb();
+        }
+        return this._floodfillNetworkDatabaseFacade;
+    }
 
     private class MessageDeliveryStatusUpdate extends JobImpl {
         private final SessionId _sessId;