Merge tag 'v14.6.1' into neo-v0.x.x

v14.6.1

Features
--------

* New helper to dump and load a cached channel to/from disk

`dhtproto.client.helper.ChannelSerializer`

The new helper class is designed for use by applications that require a
complete
copy of some DHT channel before they are able to start operating. In the
case of
a DHT outage (either partial or complete), getting a fresh copy of a
channel is
not possible, but many apps can function perfectly well with a copy of
the
channel that was previously saved to disk.

The helper has methods to dump and load from/to associative arrays and
`Map`s
(see `ocean.util.container.map.Map`), or from/to any arbitrary container
that
supports iterating over and inserting <`hash_t`, `Contiguous!(S)`>
records.

Usage examples: see documented unittests in the module.

* Simple methods to connect to a node or cluster

`dhtproto.client.mixins.NeoSupport`

The blocking API now has three additional methods named `connect`. These
add
either a single node (specified either by address & port or purely by
port) or
a cluster of nodes (specified by a config file) to the registry, and
then blocks
the current `Task` until the hash range of all nodes has been fetched.

These new methods are intended to simplify use of the DHT client in test
code
and scripts.

* New, ultra-minimal client ctor for use in scripts/tests

`dhtproto.client.DhtClient`

The existing constructors require an epoll instance and various
user-specified
configuration settings. The newly added constructor allows a DHT client
to be
instantiated with no configuration whatsoever:

```
auto dht = new DhtClient; // uses Task-scheduler's epoll instance
```

This greatly reduces the amount of boilerplate required to use a DHT
client in
a script or test.

Build.mak

@@ -1,6 +1,5 @@
 export ASSERT_ON_STOMPING_PREVENTION=1
 
-override LDFLAGS += -llzo2 -lebtree -lrt -lgcrypt -lgpg-error -lglib-2.0 -lpcre
 override DFLAGS  += -w
 
 ifeq ($(DVER),1)
@@ -15,6 +14,9 @@ $B/fakedht: override LDFLAGS += -llzo2 -lebtree -lrt -lpcre
 
 all += $B/fakedht
 
+$O/%unittests: override LDFLAGS += -llzo2 -lebtree -lrt -lpcre
+
+$O/test-fakedht: override LDFLAGS += -llzo2 -lebtree  -lrt -lpcre
 $O/test-fakedht: $B/fakedht
 
 $B/dhtapp: $C/src/dummydhtapp/main.d

README.rst

@@ -14,27 +14,6 @@ Table (DHT), including:
   The test is run, in this repo, on a fake node, but it can be reused in other
   repos to test real node implementations. (``src.dhttest``)
 
-A Tale of Two Protocols
------------------------
-
-The code in this repo is currently in transition. There exist two parallel
-client/server architectures in the repo, a new architecture (dubbed "neo") --
-based on the core code in the ``swarm/neo`` package -- and a legacy architecture
--- based on the core code located in the other packages of ``swarm``. The neo
-protocol is being introduced in stages, progressively adding features to the
-core client and server code over a series of alpha releases in a separate branch
-(named ``neo``).
-
-Note that the DHT client and node defined in this repo support *both* neo and
-legacy features.
-
-When the alpha releases are considered stable, the ``neo`` branch will be merged
-into the main release branch (currently ``v13.x.x``).
-
-When sufficient neo features have been implemented and the legacy protocol is no
-longer in active use, the legacy protocol will be deprecated and eventually
-removed.
-
 Dependencies
 ============
 

beaver.Dockerfile

@@ -1 +1 @@
-FROM sociomantictsunami/dlang:v3
+FROM sociomantictsunami/dlang:v2

docker/build

@@ -2,6 +2,7 @@
 set -xe
 
 # Install dependencies
+apt-get update
 apt-get install -y \
     liblzo2-dev \
     libebtree6-dev \

integrationtest/channelserializer/main.d

@@ -0,0 +1,98 @@
+/*******************************************************************************
+
+    Test channel serializer.
+
+    Copyright:
+        Copyright (c) 2019 dunnhumby Germany GmbH. All rights reserved.
+
+    License:
+        Boost Software License Version 1.0. See LICENSE.txt for details.
+
+*******************************************************************************/
+
+module integrationtest.channelserializer.main;
+
+import dhtproto.client.helper.ChannelSerializer;
+
+import ocean.transition;
+import ocean.core.Test;
+import ocean.core.DeepCompare;
+import ocean.util.serialize.contiguous.Contiguous;
+import ocean.util.serialize.contiguous.Deserializer;
+import ocean.util.serialize.contiguous.Serializer;
+import ocean.util.test.DirectorySandbox;
+import ocean.util.container.map.HashMap;
+import Path = ocean.io.Path;
+
+version ( UnitTest ) { }
+else
+int main ( istring[] args )
+{
+    struct S
+    {
+        int i;
+        bool b;
+        cstring str;
+    }
+
+    auto test_dir = DirectorySandbox.create(["ChannelSerializer"]);
+    scope (exit) test_dir.remove();
+
+    auto s1_init = S(23, true, "hello");
+    auto s2_init = S(42, false, "bonjour");
+    Contiguous!(S) s1, s2;
+    void[] ser_buf;
+    Deserializer.deserialize(Serializer.serialize(s1_init, ser_buf), s1);
+    Deserializer.deserialize(Serializer.serialize(s2_init, ser_buf), s2);
+
+    // AA dump / load test
+    {
+        auto ser = new ChannelSerializer!(S)("test_channel");
+        Contiguous!(S)[hash_t] out_aa;
+        out_aa[0] = s1;
+        out_aa[1] = s2;
+        ser.dump(out_aa, true);
+        test(Path.exists("test_channel"));
+
+        Contiguous!(S)[hash_t] in_aa;
+        ser.load(in_aa);
+        test!("==")(in_aa.length, 2);
+        test(deepEquals(*(in_aa[0].ptr), *(s1.ptr)));
+        test(deepEquals(*(in_aa[1].ptr), *(s2.ptr)));
+        test!("!=")(in_aa[0].ptr.str.ptr, s1.ptr.str.ptr);
+        test!("!=")(in_aa[1].ptr.str.ptr, s2.ptr.str.ptr);
+    }
+
+    // Map dump / load test
+    {
+        auto ser = new ChannelSerializer!(S)("test_channel");
+        auto map = new HashMap!(Contiguous!(S))(10);
+        *map.put(0) = s1;
+        *map.put(1) = s2;
+        ser.dump(map, true);
+        test(Path.exists("test_channel"));
+
+        map.clear();
+        ser.load(map);
+        test!("==")(map.length, 2);
+        test(deepEquals(*(map[0].ptr), *(s1.ptr)));
+        test(deepEquals(*(map[1].ptr), *(s2.ptr)));
+        test!("!=")(map[0].ptr.str.ptr, s1.ptr.str.ptr);
+        test!("!=")(map[1].ptr.str.ptr, s2.ptr.str.ptr);
+    }
+
+    // Broken dump test
+    {
+        auto ser = new ChannelSerializer!(S)("failed_channel");
+
+        void dumpRecord ( void delegate ( hash_t, ref Contiguous!(S) ) record_dg )
+        {
+            throw new Exception("File write error");
+        }
+
+        testThrown!()(ser.dump(&dumpRecord, true));
+        test(!Path.exists("failed_channel"));
+    }
+
+    return 0;
+}

relnotes/channelcache.feature.md

@@ -0,0 +1,16 @@
+### New helper to dump and load a cached channel to/from disk
+
+`dhtproto.client.helper.ChannelSerializer`
+
+The new helper class is designed for use by applications that require a complete
+copy of some DHT channel before they are able to start operating. In the case of
+a DHT outage (either partial or complete), getting a fresh copy of a channel is
+not possible, but many apps can function perfectly well with a copy of the
+channel that was previously saved to disk.
+
+The helper has methods to dump and load from/to associative arrays and `Map`s
+(see `ocean.util.container.map.Map`), or from/to any arbitrary container that
+supports iterating over and inserting <`hash_t`, `Contiguous!(S)`> records.
+
+Usage examples: see documented unittests in the module.
+