diff --git a/Build.mak b/Build.mak index ca6e8bc8..b9d3c5ab 100644 --- a/Build.mak +++ b/Build.mak @@ -1,6 +1,5 @@ export ASSERT_ON_STOMPING_PREVENTION=1 -override LDFLAGS += -llzo2 -lebtree -lrt -lgcrypt -lgpg-error -lglib-2.0 -lpcre override DFLAGS += -w ifeq ($(DVER),1) @@ -15,6 +14,9 @@ $B/fakedht: override LDFLAGS += -llzo2 -lebtree -lrt -lpcre all += $B/fakedht +$O/%unittests: override LDFLAGS += -llzo2 -lebtree -lrt -lpcre + +$O/test-fakedht: override LDFLAGS += -llzo2 -lebtree -lrt -lpcre $O/test-fakedht: $B/fakedht $B/dhtapp: $C/src/dummydhtapp/main.d diff --git a/README.rst b/README.rst index 29ca3612..b374881b 100644 --- a/README.rst +++ b/README.rst @@ -14,27 +14,6 @@ Table (DHT), including: The test is run, in this repo, on a fake node, but it can be reused in other repos to test real node implementations. (``src.dhttest``) -A Tale of Two Protocols ------------------------ - -The code in this repo is currently in transition. There exist two parallel -client/server architectures in the repo, a new architecture (dubbed "neo") -- -based on the core code in the ``swarm/neo`` package -- and a legacy architecture --- based on the core code located in the other packages of ``swarm``. The neo -protocol is being introduced in stages, progressively adding features to the -core client and server code over a series of alpha releases in a separate branch -(named ``neo``). - -Note that the DHT client and node defined in this repo support *both* neo and -legacy features. - -When the alpha releases are considered stable, the ``neo`` branch will be merged -into the main release branch (currently ``v13.x.x``). - -When sufficient neo features have been implemented and the legacy protocol is no -longer in active use, the legacy protocol will be deprecated and eventually -removed. - Dependencies ============ diff --git a/beaver.Dockerfile b/beaver.Dockerfile index 140287bf..0ddf43b7 100644 --- a/beaver.Dockerfile +++ b/beaver.Dockerfile @@ -1 +1 @@ -FROM sociomantictsunami/dlang:v3 +FROM sociomantictsunami/dlang:v2 diff --git a/docker/build b/docker/build index 18c68998..a4e4251a 100755 --- a/docker/build +++ b/docker/build @@ -2,6 +2,7 @@ set -xe # Install dependencies +apt-get update apt-get install -y \ liblzo2-dev \ libebtree6-dev \ diff --git a/relnotes/connect.feature.md b/relnotes/connect.feature.md deleted file mode 100644 index b4a0704a..00000000 --- a/relnotes/connect.feature.md +++ /dev/null @@ -1,12 +0,0 @@ -### Simple methods to connect to a node or cluster - -`dhtproto.client.mixins.NeoSupport` - -The blocking API now has three additional methods named `connect`. These add -either a single node (specified either by address & port or purely by port) or -a cluster of nodes (specified by a config file) to the registry, and then blocks -the current `Task` until the hash range of all nodes has been fetched. - -These new methods are intended to simplify use of the DHT client in test code -and scripts. - diff --git a/relnotes/ctor.feature.md b/relnotes/ctor.feature.md deleted file mode 100644 index 27ce01ad..00000000 --- a/relnotes/ctor.feature.md +++ /dev/null @@ -1,15 +0,0 @@ -### New, ultra-minimal client ctor for use in scripts/tests - -`dhtproto.client.DhtClient` - -The existing constructors require an epoll instance and various user-specified -configuration settings. The newly added constructor allows a DHT client to be -instantiated with no configuration whatsoever: - -``` -auto dht = new DhtClient; // uses Task-scheduler's epoll instance -``` - -This greatly reduces the amount of boilerplate required to use a DHT client in -a script or test. - diff --git a/relnotes/neo.feature.md b/relnotes/neo.feature.md deleted file mode 100644 index a8597841..00000000 --- a/relnotes/neo.feature.md +++ /dev/null @@ -1,8 +0,0 @@ -### Neo protocol is now mainstream - -`dhtproto.client.DhtClient` - -The DHT client now has the facility to use the neo protocol. For full -documentation on the DHT neo protocol and requests, see -`dhtproto.client.README.md`. - diff --git a/src/dhtproto/client/DhtClient.d b/src/dhtproto/client/DhtClient.d index 1bbab263..9693aece 100644 --- a/src/dhtproto/client/DhtClient.d +++ b/src/dhtproto/client/DhtClient.d @@ -201,9 +201,7 @@ public class ExtensibleDhtClient ( Plugins ... ) : DhtClient /*************************************************************************** - Constructor with support for only the legacy protocol. Automatically - calls addNodes() with the node definition file specified in the Config - instance. + Constructor Params: epoll = EpollSelectDispatcher instance to use @@ -225,9 +223,7 @@ public class ExtensibleDhtClient ( Plugins ... ) : DhtClient /*************************************************************************** - Constructor with support for only the legacy protocol. This constructor - that accepts all arguments manually (i.e. not read from a config file) - is mostly of use in tests. + Constructor Params: epoll = EpollSelectDispatcher instance to use @@ -247,80 +243,6 @@ public class ExtensibleDhtClient ( Plugins ... ) : DhtClient super(epoll, conn_limit, queue_size, fiber_stack_size); } - - - /*************************************************************************** - - Constructor with support for the neo and legacy protocols. Automatically - calls addNodes() with the node definition files specified in the legacy - and neo Config instances. - - Params: - epoll = EpollSelectDispatcher instance to use - plugin_instances = instances of Plugins - config = swarm.client.model.IClient.Config instance. (The Config - class is designed to be read from an application's config.ini - file via ocean.util.config.ConfigFiller.) - neo_config = swarm.neo.client.mixins.ClientCore.Config instance. - (The Config class is designed to be read from an application's - config.ini file via ocean.util.config.ConfigFiller.) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established). Of type: - void delegate ( Neo.DhtConnNotification ) - fiber_stack_size = size (in bytes) of stack of individual connection - fibers - - ***************************************************************************/ - - public this ( EpollSelectDispatcher epoll, Plugins plugin_instances, - IClient.Config config, Neo.Config neo_config, - Neo.DhtConnectionNotifier conn_notifier, - size_t fiber_stack_size = IClient.default_fiber_stack_size ) - { - this.setPlugins(plugin_instances); - - super(epoll, config, neo_config, conn_notifier, fiber_stack_size); - } - - - /*************************************************************************** - - Constructor with support for the neo and legacy protocols. This - constructor that accepts all arguments manually (i.e. not read from - config files) is mostly of use in tests. - - Params: - epoll = EpollSelectDispatcher instance to use - plugin_instances = instances of Plugins - auth_name = client name for authorisation - auth_key = client key (password) for authorisation. This should be a - cryptographic random number which only the client and the - nodes know. See `README_client_neo.rst` for suggestions. The key - must be of the length defined in - swarm.neo.authentication.HmacDef (128 bytes) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established). Of type: - void delegate ( Neo.DhtConnNotification ) - conn_limit = maximum number of connections to each DHT node - queue_size = maximum size of the per-node request queue - fiber_stack_size = size of connection fibers' stack (in bytes) - - ***************************************************************************/ - - public this ( EpollSelectDispatcher epoll, Plugins plugin_instances, - cstring auth_name, ubyte[] auth_key, - Neo.DhtConnectionNotifier conn_notifier, - size_t conn_limit = IClient.Config.default_connection_limit, - size_t queue_size = IClient.Config.default_queue_size, - size_t fiber_stack_size = IClient.default_fiber_stack_size ) - { - this.setPlugins(plugin_instances); - - super(epoll, auth_name, auth_key, conn_notifier, - conn_limit, queue_size, fiber_stack_size); - } } @@ -350,9 +272,9 @@ public class SchedulingDhtClient : ExtensibleDhtClient!(RequestScheduler) /*************************************************************************** - Constructor with support for only the legacy protocol. Automatically - calls addNodes() with the node definition file specified in the Config - instance. + Constructor + + Adds the nodes in the file specified in the config to the node registry Params: epoll = EpollSelectorDispatcher instance to use @@ -371,9 +293,7 @@ public class SchedulingDhtClient : ExtensibleDhtClient!(RequestScheduler) /*************************************************************************** - Constructor with support for only the legacy protocol. This constructor - that accepts all arguments manually (i.e. not read from a config file) - is mostly of use in tests. + Constructor Params: epoll = EpollSelectorDispatcher instance to use @@ -394,79 +314,6 @@ public class SchedulingDhtClient : ExtensibleDhtClient!(RequestScheduler) super(epoll, new RequestScheduler(epoll, max_events), conn_limit, queue_size, fiber_stack_size); } - - /*************************************************************************** - - Constructor with support for the neo and legacy protocols. Automatically - calls addNodes() with the node definition files specified in the legacy - and neo Config instances. - - Params: - epoll = EpollSelectDispatcher instance to use - config = SchedulingDhtClient.Config instance. (The Config class is - designed to be read from an application's config.ini file via - ocean.util.config.ConfigFiller.) - neo_config = swarm.neo.client.mixins.ClientCore.Config instance. - (The Config class is designed to be read from an application's - config.ini file via ocean.util.config.ConfigFiller.) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established). Of type: - void delegate ( Neo.DhtConnNotification ) - fiber_stack_size = size (in bytes) of stack of individual connection - fibers - - ***************************************************************************/ - - public this ( EpollSelectDispatcher epoll, - SchedulingDhtClient.Config config, - Neo.Config neo_config, Neo.DhtConnectionNotifier conn_notifier, - size_t fiber_stack_size = IClient.default_fiber_stack_size ) - { - super(epoll, new RequestScheduler(epoll, config.scheduler_limit), - config, neo_config, conn_notifier, fiber_stack_size); - } - - - /*************************************************************************** - - Constructor with support for the neo and legacy protocols. This - constructor that accepts all arguments manually (i.e. not read from - config files) is mostly of use in tests. - - Params: - epoll = EpollSelectDispatcher instance to use - auth_name = client name for authorisation - auth_key = client key (password) for authorisation. This should be a - cryptographic random number which only the client and the - nodes know. See `README_client_neo.rst` for suggestions. The key - must be of the length defined in - swarm.neo.authentication.HmacDef (128 bytes) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established). Of type: - void delegate ( Neo.DhtConnNotification ) - conn_limit = maximum number of connections to each DHT node - queue_size = maximum size of the per-node request queue - fiber_stack_size = size of connection fibers' stack (in bytes) - max_events = limit on the number of events which can be managed - by the scheduler at one time. (0 = no limit) - - ***************************************************************************/ - - public this ( EpollSelectDispatcher epoll, - cstring auth_name, ubyte[] auth_key, - Neo.DhtConnectionNotifier conn_notifier, - size_t conn_limit = IClient.Config.default_connection_limit, - size_t queue_size = IClient.Config.default_queue_size, - size_t fiber_stack_size = IClient.default_fiber_stack_size, - uint max_events = 0 ) - { - super(epoll, new RequestScheduler(epoll, max_events), - auth_name, auth_key, conn_notifier, - conn_limit, - queue_size, fiber_stack_size); - } } @@ -478,10 +325,6 @@ public class SchedulingDhtClient : ExtensibleDhtClient!(RequestScheduler) public class DhtClient : IClient { - import ocean.task.Scheduler; - import ocean.util.log.Logger; - import swarm.neo.client.requests.NotificationFormatter; - /*************************************************************************** Local alias definitions @@ -715,88 +558,14 @@ public class DhtClient : IClient /*************************************************************************** - Neo protocol support. - - ***************************************************************************/ - - import dhtproto.client.mixins.NeoSupport; - - mixin NeoSupport!(); - - - /*************************************************************************** - - Constructor with support for the neo and legacy protocols, using various - default settings for ease of construction. - - This ultra-minimal constructor is for convenience of scripts and tests. - It uses the following default settings: - * Using the task scheduler's epoll instance. (If the scheduler is not - initialised prior to calling this ctor, it will be initialised with - default settings.) - * For authentication, the name "test" and the init key are used. - * Unless a connection notifier is provided, a default notifier -- which - logs all events -- is used (see this.defaultConnNotifier). - - Params: - conn_notifier = delegate which is called when a connection-related - event occurs on a neo connection. Of type: - void delegate ( Neo.DhtConnNotification ) - - ***************************************************************************/ - - public this ( Neo.DhtConnectionNotifier conn_notifier = null ) - { - if ( !isSchedulerUsed() ) - initScheduler(Scheduler.Configuration.init); - - auto auth_name = "test"; - ubyte[] auth_key = Hmac.Key.init.content; - if ( conn_notifier is null ) - conn_notifier = &this.defaultConnNotifier; - this(theScheduler.epoll, auth_name, auth_key, conn_notifier); - } - - - /*************************************************************************** - - Neo connection notifier used if a user-defined connection notifier is - not passed to the ctor. - - ***************************************************************************/ - - private void defaultConnNotifier ( Neo.DhtConnNotification info ) - { - auto log = Log.lookup("DHTClient"); - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case connected: - case hash_range_queried: - log.trace(this.msg_buf); - break; - - case connection_error: - log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - - /*************************************************************************** - - Constructor with support for only the legacy protocol. Automatically - calls addNodes() with the node definition file specified in the Config - instance. + Constructor -- automatically calls addNodes() with the node definition + file specified in the Config instance. Params: epoll = EpollSelectorDispatcher instance to use config = Config instance (see swarm.client.model.IClient. The Config class is designed to be read from an application's - config.ini file via ocean.util.config.ConfigFiller) + config.ini file via ocean.util.config.ClassFiller) fiber_stack_size = size (in bytes) of stack of individual connection fibers @@ -816,9 +585,7 @@ public class DhtClient : IClient /*************************************************************************** - Constructor with support for only the legacy protocol. This constructor - that accepts all arguments manually (i.e. not read from a config file) - is mostly of use in tests. + Constructor Params: epoll = EpollSelectorDispatcher instance to use @@ -850,80 +617,6 @@ public class DhtClient : IClient } - /*************************************************************************** - - Constructor with support for the neo and legacy protocols. Automatically - calls addNodes() with the node definition files specified in the legacy - and neo Config instances. - - Params: - epoll = EpollSelectDispatcher instance to use - config = swarm.client.model.IClient.Config instance. (The Config - class is designed to be read from an application's config.ini - file via ocean.util.config.ConfigFiller.) - neo_config = swarm.neo.client.mixins.ClientCore.Config instance. - (The Config class is designed to be read from an application's - config.ini file via ocean.util.config.ConfigFiller.) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established). Of type: - void delegate ( Neo.DhtConnNotification ) - fiber_stack_size = size (in bytes) of stack of individual connection - fibers - - ***************************************************************************/ - - public this ( EpollSelectDispatcher epoll, IClient.Config config, - Neo.Config neo_config, Neo.DhtConnectionNotifier conn_notifier, - size_t fiber_stack_size = IClient.default_fiber_stack_size ) - { - with ( config ) - { - this(epoll, connection_limit(), queue_size(), fiber_stack_size); - - this.addNodes(nodes_file); - } - - this.neoInit(neo_config, conn_notifier); - } - - - /*************************************************************************** - - Constructor with support for the neo and legacy protocols. This - constructor that accepts all arguments manually (i.e. not read from - config files) is mostly of use in tests. - - Params: - epoll = EpollSelectDispatcher instance to use - auth_name = client name for authorisation - auth_key = client key (password) for authorisation. This should be a - cryptographic random number which only the client and the - nodes know. See `README_client_neo.rst` for suggestions. The key - must be of the length defined in - swarm.neo.authentication.HmacDef (128 bytes) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established). Of type: - void delegate ( Neo.DhtConnNotification ) - conn_limit = maximum number of connections to each DHT node - queue_size = maximum size of the per-node request queue - fiber_stack_size = size of connection fibers' stack (in bytes) - - ***************************************************************************/ - - public this ( EpollSelectDispatcher epoll, cstring auth_name, ubyte[] auth_key, - Neo.DhtConnectionNotifier conn_notifier, - size_t conn_limit = IClient.Config.default_connection_limit, - size_t queue_size = IClient.Config.default_queue_size, - size_t fiber_stack_size = IClient.default_fiber_stack_size ) - { - this(epoll, conn_limit, queue_size, fiber_stack_size); - - this.neoInit(auth_name, auth_key, conn_notifier); - } - - /*************************************************************************** Constructs the client's dht node registry. Derived classes may override diff --git a/src/dhtproto/client/NotifierTypes.d b/src/dhtproto/client/NotifierTypes.d deleted file mode 100644 index c8d4d526..00000000 --- a/src/dhtproto/client/NotifierTypes.d +++ /dev/null @@ -1,173 +0,0 @@ -/******************************************************************************* - - DHT-specific types passed to client request notifier delegates. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.NotifierTypes; - -import ocean.transition; -import Formatter = ocean.text.convert.Formatter; - -/******************************************************************************* - - A chunk of untyped data along with a key. - -*******************************************************************************/ - -public struct RequestRecordInfo -{ - import swarm.neo.protocol.Message : RequestId; - import swarm.neo.client.mixins.DeserializeMethod; - - /// ID of the request for which the notification is occurring. - RequestId request_id; - - /// Record key. - hash_t key; - - /// Record value. - Const!(void)[] value; - - /// Template method to deserialize `value` as a given struct. - mixin DeserializeMethod!(value); - - /*************************************************************************** - - Formats a description of the notification to the provided sink delegate. - - Params: - sink = delegate to feed formatted strings to - - ***************************************************************************/ - - public void toString ( void delegate ( cstring chunk ) sink ) - { - Formatter.sformat( - sink, - "Request #{} provided the record 0x{:x16}:{}", - this.request_id, this.key, this.value); - } -} - -/******************************************************************************* - - A chunk of untyped data along with a pointer to a buffer to receive a - modified version. - -*******************************************************************************/ - -public struct RequestDataUpdateInfo -{ - import swarm.neo.protocol.Message : RequestId; - import swarm.neo.client.mixins.DeserializeMethod; - - /// ID of the request for which the notification is occurring. - RequestId request_id; - - /// Record value. - Const!(void)[] value; - - /// Buffer to receive updated value. - void[]* updated_value; - - /// Template method to deserialize `value` as a given struct. - mixin DeserializeMethod!(value); - - /// Template method to serialize a given struct into `updated_value`. - mixin SerializeMethod!(updated_value); - - /*************************************************************************** - - Formats a description of the notification to the provided sink delegate. - - Params: - sink = delegate to feed formatted strings to - - ***************************************************************************/ - - public void toString ( void delegate ( cstring chunk ) sink ) - { - Formatter.sformat( - sink, - "Request #{} provided the record {} to be updated", - this.request_id, this.value); - } -} - -/******************************************************************************* - - A record key. - -*******************************************************************************/ - -public struct RequestKeyInfo -{ - import swarm.neo.protocol.Message : RequestId; - - /// ID of the request for which the notification is occurring. - RequestId request_id; - - /// Record key. - hash_t key; - - /*************************************************************************** - - Formats a description of the notification to the provided sink delegate. - - Params: - sink = delegate to feed formatted strings to - - ***************************************************************************/ - - public void toString ( void delegate ( cstring chunk ) sink ) - { - Formatter.sformat( - sink, - "Request #{} provided the key 0x{:x16}", - this.request_id, this.key); - } -} - -/******************************************************************************* - - Mixin for method to serialize a record value. - - Params: - dst = pointer to the buffer to be serialized into - - TODO: deprecated, replace with implementation in swarm - -*******************************************************************************/ - -template SerializeMethod ( alias dst ) -{ - import ocean.util.serialize.contiguous.Contiguous; - import ocean.util.serialize.Version; - import ocean.util.serialize.contiguous.Serializer; - import ocean.util.serialize.contiguous.MultiVersionDecorator; - - /*************************************************************************** - - Serializes `src` into `dst`, using a version decorater, if required. - - Params: - T = type of struct to serialize - src = instance to serialize - - ***************************************************************************/ - - public void serialize ( T ) ( T src ) - { - static if ( Version.Info!(T).exists ) - VersionDecorator.store!(T)(src, *dst); - else - Serializer.serialize(src, *dst); - } -} diff --git a/src/dhtproto/client/README.md b/src/dhtproto/client/README.md deleted file mode 100644 index dc90deb0..00000000 --- a/src/dhtproto/client/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# DHT Client Overview - -This package contains the client to connect to a Distributed Hash Table -(DHT) database. The client is built on top of the framework provided by -[swarm](https://github.com/sociomantic-tsunami/swarm/). Swarm (and thus the DHT -client) currently supports two protocols, known as the "legacy protocol" and the -"neo procotol". Detailed documentation about the workings of swarm-based clients -and the two protocols can be found -[here (legacy protocol)](https://github.com/sociomantic-tsunami/swarm/blob/v5.x.x/src/swarm/README_client.rst) -and [here (neo protocol)](https://github.com/sociomantic-tsunami/swarm/blob/v5.x.x/src/swarm/README_client_neo.rst). - -The legacy protocol is to be phased out, so the remainder of this README focuses -solely on the neo protocol. - -## Usage examples - -Detailed usage examples for all requests (and other features of the client) are -provided in [this module](UsageExamples.d). - -## Requests - -### Request API Modules - -Each request has an API module in `src.dhtproto.client.request`, containing: - -* A description of what the request does and how it works. -* The definition of the notifier delegate type for the request. (A notifier - must be provided by the user, when assigning a request, and is called whenever - anything of interest related to the request happens.) -* The smart union of notifications that is passed to the notifier. The active - member of the union indicates the type of the notification and may carry - additional information (e.g. the address/port of a node, an exception, etc). -* The ``Args`` struct which is passed to the notifier delegate, along with the - notification. This contains a copy of all arguments which were specified by - the user to start the request. - -The request API modules provide a single, centralised point of documentation and -definitions pertaining to each request. - -### Available Requests - -The DHT supports the following requests (links to the API modules): - -* [`Put`](request/Put.d): - puts a record key and value to a channel. -* [`Get`](request/Get.d): - gets a record, specified by its key, from a channel. -* [`Update`](request/Update.d): - gets a record, specified by its key, from a channel, and replaces it with an - updated value, specified by the user. -* [`Exists`](request/Exists.d): - checks whether a record, specified by its key, exists in a channel. -* [`Remove`](request/Remove.d): - removes a record, specified by its key, from a channel. -* [`GetAll`](request/GetAll.d): - gets the keys and values of all records in a channel. -* [`GetChannels`](request/GetChannels.d): - gets the names of all channels. -* [`Mirror`](request/Mirror.d): - receive a stream of updates (additions, changes, deletions) to records in a - channel, including a periodic "refresh" of all records in the channel. -* [`RemoveChannel`](request/RemoveChannel.d): - removes a channel from the DHT, discarding all records in it. - -### Assigning Requests - -The methods to assign requests are in the `DhtClient` class and defined in -[this module](mixins/NeoSupport.d). Note that there are two ways to assign some -requests: - -1. Via the `DhtClient.neo` object. This assigns a request in the normal - manner. -2. Via the `DhtClient.blocking` object. This assigns a request in a `Task`- - blocking manner -- the current task will be suspended until the assigned - request is finished. - diff --git a/src/dhtproto/client/UsageExamples.d b/src/dhtproto/client/UsageExamples.d deleted file mode 100644 index a546cc98..00000000 --- a/src/dhtproto/client/UsageExamples.d +++ /dev/null @@ -1,1252 +0,0 @@ -/******************************************************************************* - - DHT client usage examples. - - Note that the examples are only for the neo requests. - - High-level module overview: - 1. The standard DHT client in dhtproto.client.DhtClient can communicate - with the legacy *and* the neo protocols. - 2. The DHT-specific parts of the neo client (e.g. request methods) are - in dhtproto.client.mixins.NeoSupport. - 3. The generic parts of the neo client (e.g. addNodes()) are in - swarm.neo.client.mixins.ClientCore. - 4. Each request has an API module that defines its public API (the - notifier, the arguments, etc). These live in dhtproto.client.request. - 5. The structs returned by the notifiers are in - swarm.neo.client.NotifierTypes and dhtproto.client.NotifierTypes. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.UsageExamples; - -version ( UnitTest ) -{ - import ocean.transition; - import ocean.core.SmartUnion; - import ocean.util.app.DaemonApp; - import dhtproto.client.DhtClient; - import swarm.neo.client.requests.NotificationFormatter; - - // DaemonApp class showing typical neo DHT client initialisation. The class - // has a single abstract method -- example() -- which is implemented by - // various usage examples in this module, each demonstrating a different - // client feature. - abstract class ExampleApp : DaemonApp - { - import ocean.task.Scheduler; - import ocean.task.Task; - import ocean.util.log.Logger; - import ConfigFiller = ocean.util.config.ConfigFiller; - import ocean.text.convert.Hex : hexToBin; - - // DHT client. (See dhtproto.client.DhtClient.) - private DhtClient dht; - - // Buffer used for message formatting in notifiers. - private mstring msg_buf; - - // Logger used for logging notifications. - protected Logger log; - - // Legacy and neo config instances to be read from the config file. - private DhtClient.Config config; - private DhtClient.Neo.Config neo_config; - - // Constructor. Initialises the scheduler. - public this ( ) - { - super("example", "DHT client neo usage example", VersionInfo.init); - - // Set up the logger for this example. - this.log = Log.lookup("example"); - - // Initialise the global scheduler. - SchedulerConfiguration scheduler_config; - initScheduler(scheduler_config); - } - - // Reads the required config from the config file. - override public void processConfig ( IApplication app, - ConfigParser config_parser ) - { - ConfigFiller.fill("DHT", this.config, config_parser); - ConfigFiller.fill("DHT_Neo", this.neo_config, config_parser); - } - - // Application run method. Initialises the DHT client and starts the - // main application task. - override protected int run ( Arguments args, ConfigParser config ) - { - // Create a DHT client instance, passing the filled config instances - // and the neo connection notifier. The node definitiion files - // specified in the config instances are automatically read and the - // defined nodes added to the client's registry. Note that the neo - // protocol does not require an explicit handshake; it happens - // automatically in the background. - this.dht = new DhtClient(theScheduler.epoll, this.config, - this.neo_config, &this.connNotifier); - - // Schedule the application's main task and start the event loop. - theScheduler.schedule(new AppTask); - theScheduler.eventLoop(); - return 0; - } - - // Application main task. - private class AppTask : Task - { - // Task entry point. Waits for the DHT client to connect then runs - // the example. - protected override void run ( ) - { - this.connect(); - this.outer.example(); - } - - // Waits for the DHT client to connect and query the hash ranges of - // all registered nodes. - private void connect ( ) - { - // Add some nodes. (See swarm.neo.client.mixins.ClientCore.) - // Note: make sure you have a .nodes file which specifies the - // neo ports of the nodes! - this.outer.dht.neo.addNodes("dht.nodes"); - - // Suspend the task until the hash range of all registered nodes - // has been queried. (See dhtproto.client.mixins.NeoSupport.) - this.outer.dht.blocking.waitAllHashRangesKnown(); - } - } - - // Abstract method containing the logic for each example. - abstract protected void example ( ); - - // Notifier which is called when a connection establishment attempt - // succeeds or fails and when the hash-range which a connected node is - // responsible for has been queried. (Also called after re-connection - // attempts are made.) - // (See dhtproto.client.mixins.NeoSupport for the definition of the - // notification union.) - private void connNotifier ( DhtClient.Neo.DhtConnNotification info ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case connected: - this.log.trace(this.msg_buf); - break; - - case hash_range_queried: - case connection_error: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/******************************************************************************* - - Dummy struct to enable ddoc rendering of usage examples. - -*******************************************************************************/ - -struct UsageExamples -{ -} - -/// Example of using the ultra-minimal constructor -unittest -{ - void initDht ( ) - { - // Uses the following default settings: - // * The task scheduler's epoll instance. - // * If the task scheduler has not been initialised, initialises it with - // its default configuration. - // * For authentication, the name "test" and the init key are used. - // * A default connection notifier -- which logs all events. - auto dht = new DhtClient; - } -} - -/// Example of neo Put request usage -unittest -{ - class PutExample : ExampleApp - { - override protected void example ( ) - { - // Assign a neo Put request. Note that the channel and value - // are copied inside the client -- the user does not need to - // maintain them after calling this method. - this.dht.neo.put("channel", 0x1234567812345678, - "value_to_put", &this.putNotifier); - } - - // Notifier which is called when something of interest happens to - // the Put request. See dhtproto.client.request.Put for - // details of the parameters of the notifier. - private void putNotifier ( DhtClient.Neo.Put.Notification info, - Const!(DhtClient.Neo.Put.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case success: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case value_too_big: - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of Task-blocking neo Put request usage with a notifier -unittest -{ - class PutExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Put request. Note that the channel and - // value are copied inside the client -- the user does not need to - // maintain them after calling this method. - this.dht.blocking.put("channel", 0x1234567812345678, - "value_to_put", &this.putNotifier); - } - - // Notifier which is called when something of interest happens to - // the Put request. See dhtproto.client.request.Put for - // details of the parameters of the notifier. - private void putNotifier ( DhtClient.Neo.Put.Notification info, - Const!(DhtClient.Neo.Put.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case success: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case value_too_big: - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of simple Task-blocking neo Put request usage without a notifier -unittest -{ - class PutExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Put request and return a result struct - // indicating success/failure. Notes: - // 1. In a real application, you probably want more information than - // just success/failure and should use the task-blocking method - // with a notifier (see example above). - // 2. The channel and value are copied inside the client -- the user - // does not need to maintain them after calling this method. - auto result = this.dht.blocking.put("channel", 0x1234567812345678, - "value_to_put"); - if ( result.succeeded ) - this.log.trace("Put succeeded"); - else - this.log.error("Put failed"); - } - } -} - -/// Example of neo Get request usage -unittest -{ - class GetExample : ExampleApp - { - override protected void example ( ) - { - // Assign a neo Get request. Note that the channel is copied inside - // the client -- the user does not need to maintain it after calling - // this method. - this.dht.neo.get("channel", 0x1234567812345678, &this.getNotifier); - } - - // Notifier which is called when something of interest happens to - // the Get request. See dhtproto.client.request.Get for - // details of the parameters of the notifier. - private void getNotifier ( DhtClient.Neo.Get.Notification info, - Const!(DhtClient.Neo.Get.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case received: - auto received_record = info.received.value; - goto case; - case no_record: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - case timed_out: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of neo Get request usage, including record deserialization -unittest -{ - class GetExample : ExampleApp - { - import ocean.util.serialize.contiguous.Contiguous; - - override protected void example ( ) - { - // Assign a neo Get request. Note that the channel is copied inside - // the client -- the user does not need to maintain it after calling - // this method. - this.dht.neo.get("channel", 0x1234567812345678, &this.getNotifier); - } - - // Notifier which is called when something of interest happens to - // the Get request. See dhtproto.client.request.Get for - // details of the parameters of the notifier. - private void getNotifier ( DhtClient.Neo.Get.Notification info, - Const!(DhtClient.Neo.Get.Args) args ) - { - // Struct expected to be serialized in the received record value. - struct Record - { - mstring name; - hash_t id; - ulong[7] daily_totals; - } - - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case received: - this.log.trace(this.msg_buf); - - Contiguous!(Record) record; - auto deserialized = info.received.deserialize(record); - this.log.trace("Deserialized: {} / {} / {}", - deserialized.name, deserialized.id, - deserialized.daily_totals); - break; - - case no_record: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - case timed_out: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of Task-blocking neo Get request usage with a notifier -unittest -{ - class GetExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Get request. Note that the channel is - // copied inside the client -- the user does not need to maintain - // it after calling this method. - this.dht.blocking.get("channel", 0x1234567812345678, - &this.getNotifier); - } - - // Notifier which is called when something of interest happens to - // the Get request. See dhtproto.client.request.Get for - // details of the parameters of the notifier. - private void getNotifier ( DhtClient.Neo.Get.Notification info, - Const!(DhtClient.Neo.Get.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case received: - auto received_record = info.received.value; - goto case; - case no_record: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - case timed_out: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of simple Task-blocking neo Get request usage without a notifier -unittest -{ - class GetExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Get request and return a result struct - // indicating success/failure. Notes: - // 1. In a real application, you probably want more information than - // just success/failure and should use the task-blocking method - // with a notifier (see example above). - // 2. The channel is copied inside the client -- the user does not - // need to maintain it after calling this method. - void[] get_buf; - auto result = this.dht.blocking.get("channel", 0x1234567812345678, - get_buf); - if ( result.succeeded ) - { - if ( result.value.length ) - this.log.trace("Get succeeded: {}", result.value); - else - this.log.trace("Get succeeded: no record"); - } - else - this.log.error("Get failed"); - } - } -} - -/// Example of neo Exists request usage -unittest -{ - class ExistsExample : ExampleApp - { - override protected void example ( ) - { - // Assign a neo Exists request. Note that the channel is copied - // inside the client -- the user does not need to maintain it after - // calling this method. - this.dht.neo.exists("channel", 0x1234567812345678, - &this.existsNotifier); - } - - // Notifier which is called when something of interest happens to - // the Exists request. See dhtproto.client.request.Exists for - // details of the parameters of the notifier. - private void existsNotifier ( DhtClient.Neo.Exists.Notification info, - Const!(DhtClient.Neo.Exists.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case exists: - goto case; - case no_record: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of Task-blocking neo Exists request usage with a notifier -unittest -{ - class ExistsExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Exists request. Note that the channel is - // copied inside the client -- the user does not need to maintain - // it after calling this method. - this.dht.blocking.exists("channel", 0x1234567812345678, - &this.existsNotifier); - } - - // Notifier which is called when something of interest happens to - // the Exists request. See dhtproto.client.request.Exists for - // details of the parameters of the notifier. - private void existsNotifier ( DhtClient.Neo.Exists.Notification info, - Const!(DhtClient.Neo.Exists.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case exists: - goto case; - case no_record: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of simple Task-blocking neo Exists request usage without a notifier -unittest -{ - class ExistsExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Exists request and return a result struct - // indicating success/failure. Notes: - // 1. In a real application, you probably want more information than - // just success/failure and should use the task-blocking method - // with a notifier (see example above). - // 2. The channel is copied inside the client -- the user does not - // need to maintain it after calling this method. - auto result = this.dht.blocking.exists("channel", 0x1234567812345678); - if ( result.succeeded ) - { - if ( result.exists ) - this.log.trace("Exists succeeded: record exists"); - else - this.log.trace("Exists succeeded: no record"); - } - else - this.log.error("Exists failed"); - } - } -} - -/// Example of neo Remove request usage -unittest -{ - class RemoveExample : ExampleApp - { - override protected void example ( ) - { - // Assign a neo Remove request. Note that the channel is copied - // inside the client -- the user does not need to maintain it after - // calling this method. - this.dht.neo.remove("channel", 0x1234567812345678, - &this.removeNotifier); - } - - // Notifier which is called when something of interest happens to - // the Remove request. See dhtproto.client.request.Remove for - // details of the parameters of the notifier. - private void removeNotifier ( DhtClient.Neo.Remove.Notification info, - Const!(DhtClient.Neo.Remove.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case removed: - case no_record: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of Task-blocking neo Remove request usage with a notifier -unittest -{ - class RemoveExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Remove request. Note that the channel is - // copied inside the client -- the user does not need to maintain - // it after calling this method. - this.dht.blocking.remove("channel", 0x1234567812345678, - &this.removeNotifier); - } - - // Notifier which is called when something of interest happens to - // the Remove request. See dhtproto.client.request.Remove for - // details of the parameters of the notifier. - private void removeNotifier ( DhtClient.Neo.Remove.Notification info, - Const!(DhtClient.Neo.Remove.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case removed: - case no_record: - this.log.trace(this.msg_buf); - break; - - case failure: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of simple Task-blocking neo Remove request usage without a notifier -unittest -{ - class RemoveExample : ExampleApp - { - override protected void example ( ) - { - // Perform a blocking neo Remove request and return a result struct - // indicating success/failure. Notes: - // 1. In a real application, you probably want more information than - // just success/failure and should use the task-blocking method - // with a notifier (see example above). - // 2. The channel is copied inside the client -- the user does not - // need to maintain it after calling this method. - auto result = this.dht.blocking.remove("channel", 0x1234567812345678); - if ( result.succeeded ) - { - if ( result.existed ) - this.log.trace("Remove succeeded; record removed"); - else - this.log.trace("Remove succeeded; record did not exist"); - } - else - this.log.error("Remove failed"); - } - } -} - -/// Example of neo Mirror request usage -unittest -{ - class MirrorExample : ExampleApp - { - // Id of the running Mirror request - private DhtClient.Neo.RequestId rq_id; - - override protected void example ( ) - { - // Optional mirror settings. This example just sets the default - // value for each field, but different behaviour can be configured - // by setting different values. See dhtproto.client.request.Mirror. - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = true; - mirror_settings.periodic_refresh_s = 360; - - // Assign a neo Mirror request. Note that the channel is copied - // inside the client -- the user does not need to maintain it after - // calling this method. - this.rq_id = this.dht.neo.mirror("channel", &this.mirrorNotifier, - mirror_settings); - } - - // Notifier which is called when something of interest happens to - // the Mirror request. See dhtproto.client.request.Mirror for - // details of the parameters of the notifier. - private void mirrorNotifier ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case updated: - this.log.trace(this.msg_buf); - - // Here we use the controller to cleanly end the request - // after a while - static ubyte count; - if ( ++count >= 10 ) - this.stop(); - break; - - case started: - case refreshed: - case deleted: - case channel_removed: - case stopped: - case suspended: - case resumed: - this.log.trace(this.msg_buf); - break; - - case updates_lost: - this.log.warn(this.msg_buf); - break; - - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - // Method which is called from the `updated` case of the notifier - // (above). Sends a message to the DHT to cleanly stop handling this - // request. - private void stop ( ) - { - // The control() method of the client allows you to get access - // to an interface providing methods which control the state of - // a request, while it's in progress. The Mirror request - // controller interface is in dhtproto.client.request.Mirror. - // Not all requests can be controlled in this way. - this.dht.neo.control(this.rq_id, - ( DhtClient.Neo.Mirror.IController mirror ) - { - // We tell the request to stop. This will cause a - // message to be sent to all DHT nodes, telling them to - // end the Mirror. More updates may be received while - // this is happening, but the notifier will be called as - // soon as all nodes have stopped. (There are also - // controller methods to suspend and resume the request - // on the node-side.) - mirror.stop(); - } - ); - } - } -} - -/// Example of neo GetAll request usage -unittest -{ - class GetAllExample : ExampleApp - { - // Id of the running GetAll request - private DhtClient.Neo.RequestId rq_id; - - override protected void example ( ) - { - // Assign a neo GetAll request. Note that the channel is copied - // inside the client -- the user does not need to maintain it after - // calling this method. - this.rq_id = this.dht.neo.getAll("channel", &this.getAllNotifier); - } - - // Notifier which is called when something of interest happens to - // the GetAll request. See dhtproto.client.request.GetAll for - // details of the parameters of the notifier. - private void getAllNotifier ( DhtClient.Neo.GetAll.Notification info, - Const!(DhtClient.Neo.GetAll.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case started: - case received: - case received_key: - case finished: - case stopped: - case suspended: - case resumed: - this.log.trace(this.msg_buf); - break; - - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of Task-blocking neo GetAll request usage -unittest -{ - class GetAllExample : ExampleApp - { - override protected void example ( ) - { - void[] buf; - - // Assign a neo GetAll request. Note that the channel is copied - // inside the client -- the user does not need to maintain it after - // calling this method. - foreach ( k, v; this.dht.blocking.getAll("channel", buf) ) - { - log.trace("GetAll received 0x{:x16}:{}", k, v); - } - } - } -} - -/// Example of Task-blocking neo GetChannels request usage -unittest -{ - class GetChannelsExample : ExampleApp - { - override protected void example ( ) - { - mstring buf; - - // Assign a neo GetChannels request. - foreach ( channel_name; this.dht.blocking.getChannels(buf) ) - { - log.trace("GetChannels received {}", channel_name); - } - } - } -} - -/// Example of neo Update request usage -unittest -{ - class UpdateExample : ExampleApp - { - import ocean.core.array.Mutation : copy; - - override protected void example ( ) - { - // Assign a neo Update request. Note that the channel is copied - // inside the client -- the user does not need to maintain it after - // calling this method. - this.dht.neo.update("channel", 0x1234567812345678, &this.updateNotifier); - } - - // Notifier which is called when something of interest happens to - // the Update request. See dhtproto.client.request.Update for - // details of the parameters of the notifier. - private void updateNotifier ( DhtClient.Neo.Update.Notification info, - Const!(DhtClient.Neo.Update.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case received: - // Set updated value. In this simple example, we simply - // copy the original value, without change. - auto received_record = info.received.value; - (*info.received.updated_value).copy(info.received.value); - goto case; - - case succeeded: // Updated successfully. - case no_record: // Record not in DHT. Use Put to write a new record. - this.log.trace(this.msg_buf); - break; - - case conflict: // Another client updated the same record. Try again. - this.log.warn(this.msg_buf); - break; - - case error: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of neo Update request usage, including record de/serialization -unittest -{ - class UpdateExample : ExampleApp - { - import ocean.util.serialize.contiguous.Contiguous; - - override protected void example ( ) - { - // Assign a neo Update request. Note that the channel is copied - // inside the client -- the user does not need to maintain it after - // calling this method. - this.dht.neo.update("channel", 0x1234567812345678, &this.updateNotifier); - } - - // Notifier which is called when something of interest happens to - // the Update request. See dhtproto.client.request.Update for - // details of the parameters of the notifier. - private void updateNotifier ( DhtClient.Neo.Update.Notification info, - Const!(DhtClient.Neo.Update.Args) args ) - { - // Struct expected to be serialized in the received record value. - struct Record - { - mstring name; - hash_t id; - ulong[7] daily_totals; - } - - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case received: - // Deserialize the received record. - Contiguous!(Record) record; - auto deserialized = info.received.deserialize(record); - this.log.trace("Deserialized: {} / {} / {}", - deserialized.name, deserialized.id, - deserialized.daily_totals); - - // Update the record. - deserialized.daily_totals[0]++; - - // Serialize the updated record. - info.received.serialize(*deserialized); - goto case; - - case succeeded: // Updated successfully. - case no_record: // Record not in DHT. Use Put to write a new record. - this.log.trace(this.msg_buf); - break; - - case conflict: // Another client updated the same record. Try again. - this.log.warn(this.msg_buf); - break; - - case error: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of Task-blocking neo Update request usage -unittest -{ - class UpdateExample : ExampleApp - { - import ocean.core.array.Mutation : copy; - - override protected void example ( ) - { - // Perform a blocking neo Update request. Note that the channel is - // copied inside the client -- the user does not need to maintain - // it after calling this method. - this.dht.blocking.update("channel", 0x1234567812345678, - &this.updateNotifier); - } - - // Notifier which is called when something of interest happens to - // the Update request. See dhtproto.client.request.Update for - // details of the parameters of the notifier. - private void updateNotifier ( DhtClient.Neo.Update.Notification info, - Const!(DhtClient.Neo.Update.Args) args ) - { - formatNotification(info, this.msg_buf); - - with ( info.Active ) final switch ( info.active ) - { - case received: - // Set updated value. In this simple example, we simply - // copy the original value, without change. - auto received_record = info.received.value; - (*info.received.updated_value).copy(info.received.value); - goto case; - - case succeeded: // Updated successfully. - case no_record: // Record not in DHT. Use Put to write a new record. - this.log.trace(this.msg_buf); - break; - - case conflict: // Another client updated the same record. Try again. - this.log.warn(this.msg_buf); - break; - - case error: - case no_node: - case node_disconnected: - this.log.error(this.msg_buf); - break; - - // Fatal errors. Retrying will almost certainly result in the - // same error. - case node_error: - case wrong_node: - case unsupported: - this.log.error(this.msg_buf); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - } -} - -/// Example of using the DHT client's stats APIs -unittest -{ - class StatsExample : ExampleApp - { - // Log on-demand stats. - override protected void example ( ) - { - // See DhtStats in dhtproto.client.mixins.NeoSupport and Stats in - // swarm.neo.client.mixins.ClientCore - auto stats = this.dht.neo.new DhtStats; - - // Connection stats. - this.log.info("DHT nodes registered with client: {}", - stats.num_registered_nodes); - this.log.info("DHT nodes in initial connection establishment state: {}", - stats.num_initializing_nodes); - this.log.info("Current fraction of DHT nodes in initial connection establishment state: {}", - stats.initializing_nodes_fraction); - this.log.info("DHT nodes connected: {}", - stats.num_connected_nodes); - this.log.info("DHT nodes with hash ranges queried: {}", - stats.num_nodes_known_hash_range); - this.log.info("All DHT nodes connected?: {}", - stats.all_nodes_connected); - this.log.info("Current fraction of DHT nodes connected: {}", - stats.connected_nodes_fraction); - - // Connection I/O stats. - { - size_t i; - foreach ( conn_sender_io, conn_receiver_io; stats.connection_io ) - { - // See swarm.neo.protocol.socket.IOStats - this.log.info("Total bytes sent/received over connection {}: {} / {}", - i++, conn_sender_io.socket.total, conn_receiver_io.socket.total); - } - } - - // Connection send queue stats. - { - size_t i; - foreach ( send_queue; stats.connection_send_queue ) - { - // See swarm.neo.util.TreeQueue - this.log.info("Total time messages waited in send queue of connection {}: {}μs", - i++, send_queue.time_histogram.total_time_micros); - } - } - - // Request pool stats. - this.log.info("Requests currently active: {}", - stats.num_active_requests); - this.log.info("Maximum active requests allowed: {}", - stats.max_active_requests); - this.log.info("Current fraction of maximum active requests: {}", - stats.active_requests_fraction); - - // Per-request stats. - auto rq_stats = this.dht.neo.new RequestStats; - foreach ( name, stats; rq_stats.allRequests() ) - { - // See swarm.neo.client.requests.Stats - this.log.info("{} {} requests handled, mean time: {}μs", - stats.count, name, stats.mean_handled_time_micros); - } - } - - // Pass the neo client's stats to the stats logger. - override protected void onStatsTimer ( ) - { - // See DhtStats in dhtproto.client.mixins.NeoSupport and Stats in - // swarm.neo.client.mixins.ClientCore - auto stats = this.dht.neo.new DhtStats; - - // Create a struct like this to contain all the stats you want to log. - // The field names and types must match the names and return values of - // getter methods in DhtStats (or its base class Stats). - struct StatsAggregate - { - size_t num_active_requests; - size_t max_active_requests; - double active_requests_fraction; - size_t num_registered_nodes; - size_t num_initializing_nodes; - double initializing_nodes_fraction; - size_t num_connected_nodes; - bool all_nodes_connected; - double connected_nodes_fraction; - } - - // Fills in an instance of StatsAggregate by calling the corresponding - // getters from `stats` and passes the struct to the logger. - this.dht.neo.logStatsFromAggregate!(StatsAggregate)(stats, - this.stats_ext.stats_log); - - // Logs all per-request stats. - auto rq_stats = this.dht.neo.new RequestStats; - rq_stats.log(this.stats_ext.stats_log); - } - } -} diff --git a/src/dhtproto/client/internal/NodeHashRanges.d b/src/dhtproto/client/internal/NodeHashRanges.d deleted file mode 100644 index 7639632e..00000000 --- a/src/dhtproto/client/internal/NodeHashRanges.d +++ /dev/null @@ -1,514 +0,0 @@ -/******************************************************************************* - - Class which tracks the hash range associated with a set of nodes, with - methods to update and query this information. The update method hooks into - the client's ConnectionSet and adds new connections, as required. - - The order in which the node hash range info is updated is also tracked, - allowing queries to determine, in the case of a hash range overlap, which is - the new and which the old node covering this hash range. This information is - required during DHT redistributions, when nodes may have overlapping hash - ranges. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.internal.NodeHashRanges; - -import ocean.transition; -import swarm.neo.util.VoidBufferAsArrayOf; - -/// ditto -public final class NodeHashRanges : NodeHashRangesBase -{ - import swarm.neo.client.ConnectionSet; - import swarm.neo.client.RequestOnConn; - import swarm.neo.client.RequestHandlers : UseNodeDg; - import swarm.neo.AddrPort; - - /// Set of connections to nodes (one per node) - private ConnectionSet connections; - - /// Alias for a delegate which receives info about a new node hash-range. - private alias void delegate ( AddrPort addr, hash_t min, hash_t max ) - NewNodeNotifier; - - /// Delegate called when info about a new node hash-range is available. - private NewNodeNotifier new_node_dg; - - /*************************************************************************** - - Constructor. - - Params: - connections = ConnectionSet of the client; to be updated by - updateNodeHashRange(), when applicable - new_node_dg = delegate called when info about a new node hash-range - is available - - ***************************************************************************/ - - public this ( ConnectionSet connections, NewNodeNotifier new_node_dg ) - { - this.connections = connections; - this.new_node_dg = new_node_dg; - } - - /*************************************************************************** - - Helper encapsulating the node-selection logic required by requests that - get a single record from the DHT. Namely: - - During a data redistribution, more than one node may be responsible - for a given key. In this case, the node that was most recently - reported as being responsible for the key is queried first, followed - by others (in order) until the record is located, an error occurs, - or no node has the record. - - TODO: test the logic for retrying the request on other nodes which - previously covered the hash. This will require a full neo implementation - of the Redistribute request. See - https://github.com/sociomantic-tsunami/dhtnode/issues/21 - - Params: - h = hash to query - node_hash_ranges = buffer to receive hash range information of - nodes which cover the specified hash - use_node = delegate to call to gain access to a request-on-conn to - communicate with the selected node - get = delegate to communicate with the selected node (over the - request-on-conn provided by use_node). Should return true if the - get succeeded or false to try the next node - - ***************************************************************************/ - - public void getFromNode ( hash_t h, - VoidBufferAsArrayOf!(NodeHashRange) node_hash_ranges, UseNodeDg use_node, - bool delegate ( RequestOnConn.EventDispatcher ) get ) - { - auto nodes = this.getNodesForHash(h, node_hash_ranges); - if ( nodes.length == 0 ) - return; - - foreach ( node_hash_range; nodes.array() ) - { - bool try_next_node; - scope conn_dg = - ( RequestOnConn.EventDispatcher conn ) - { - try_next_node = get(conn); - }; - use_node(node_hash_range.addr, conn_dg); - - // If we got the record or an error occurred, don't try more nodes - if ( !try_next_node ) - break; - } - } - - /*************************************************************************** - - Helper encapsulating the node-selection logic required by requests that - remove a single record from the DHT. Namely: - - During a data redistribution, more than one node may be responsible - for a given key. In this case, the node that was least recently - reported as being responsible for the key is queried first, followed - by others (in order) until the record is either removed from or does - not exist on all nodes, or an error occurs. The reason for removing - from the *least* recently responsible nodes first is to avoid - getting into inconsistent states if an error occurs while removing. - (If an error occurred when removing the record from the most - recently responsible node first, subsequent read requests would - fetch the removed record from older nodes, and the old value could - be forwarded from an older node, undoing the removal.) - - TODO: test the logic for retrying the request on other nodes which - previously covered the hash. This will require a full neo implementation - of the Redistribute request. See - https://github.com/sociomantic-tsunami/dhtnode/issues/21 - - Params: - h = hash to query - node_hash_ranges = buffer to receive hash range information of - nodes which cover the specified hash - use_node = delegate to call to gain access to a request-on-conn to - communicate with the selected node - remove = delegate to communicate with the selected node (over the - request-on-conn provided by use_node). Should return true if the - removal succeeded or false on error - - ***************************************************************************/ - - public void removeFromNodes ( hash_t h, - VoidBufferAsArrayOf!(NodeHashRange) node_hash_ranges, UseNodeDg use_node, - bool delegate ( RequestOnConn.EventDispatcher ) remove ) - { - auto nodes = this.getNodesForHash(h, node_hash_ranges); - if ( nodes.length == 0 ) - return; - - foreach_reverse ( node_hash_range; nodes.array() ) - { - bool continue_to_next_node; - scope conn_dg = - ( RequestOnConn.EventDispatcher conn ) - { - continue_to_next_node = remove(conn); - }; - use_node(node_hash_range.addr, conn_dg); - - // If an error occurred, don't try more nodes - if ( !continue_to_next_node ) - break; - } - } - - /*************************************************************************** - - Helper encapsulating the node-selection logic required by requests that - put a single record to the DHT. Namely: - - During a data redistribution, more than one node may be responsible - for a given key. In this case, the record is written to the node - that was most recently reported as being responsible for the key. - - Params: - h = hash to query - node_hash_ranges = buffer to receive hash range information of - nodes which cover the specified hash - use_node = delegate to call to gain access to a request-on-conn to - communicate with the selected node - put = delegate to communicate with the selected node (over the - request-on-conn provided by use_node) - - ***************************************************************************/ - - public void putToNode ( hash_t h, - VoidBufferAsArrayOf!(NodeHashRange) node_hash_ranges, UseNodeDg use_node, - void delegate ( RequestOnConn.EventDispatcher ) put ) - { - auto nodes = this.getNodesForHash(h, node_hash_ranges); - if ( nodes.length == 0 ) - return; - - scope conn_dg = - ( RequestOnConn.EventDispatcher conn ) - { - put(conn); - }; - use_node(nodes.array()[0].addr, conn_dg); - } - - /*************************************************************************** - - Adds a new node to the node hash range set and the ConnectionSet. - - Params: - addr = address & port of new node - min = minimum hash of new node - max = maximum hash of new node - - ***************************************************************************/ - - override protected void newNode ( AddrPort addr, hash_t min, hash_t max ) - { - super.newNode(addr, min, max); - - // Also add the new connection to the ConnectionSet, if it's new. - if ( this.connections.get(addr) is null ) - this.connections.start(addr); - - if ( this.new_node_dg !is null ) - { - this.new_node_dg(addr, min, max); - } - } -} - -/******************************************************************************* - - Struct containing information about the hash range of a single node. - -*******************************************************************************/ - -public struct NodeHashRange -{ - import ocean.math.Range; - import swarm.neo.AddrPort; - - /// Convenience alias for a range of hash_t. - public alias Range!(hash_t) HashRange; - - /// Address & port of node. - public AddrPort addr; - - /// Range of hashes covered by node (hash_range.is_empty() may be true). - public HashRange hash_range; - - /// Ordering integer. Higher numbers were updated more recently. - public ulong order; -} - -/******************************************************************************* - - Base class for NodeHashRanges, without a dependence on the ConnectionSet. - Purely exists for the sake of unittesting. - -*******************************************************************************/ - -private class NodeHashRangesBase -{ - import ocean.core.Traits : ReturnTypeOf; - import swarm.neo.AddrPort; - import swarm.neo.client.ConnectionSet; - import swarm.util.Hash : isWithinNodeResponsibility; - import ocean.core.array.Mutation : sort; - import ocean.core.Verify; - - /// Value of the next created NodeHashRange's order field. - private static ulong order_counter; - - /// Convenience alias for the comparison type of AddrPort. - private alias ReturnTypeOf!(AddrPort.cmp_id) Addr; - - /// Map of IP address -> node hash range. - private NodeHashRange[Addr] node_hash_ranges; - - /*************************************************************************** - - Returns: - the number of nodes about which hash-range info is known - - ***************************************************************************/ - - public size_t length ( ) - { - return this.node_hash_ranges.length; - } - - /*************************************************************************** - - Adds or modifies the hash range associated with the specified node. If - the node is not already in the set, it is added and also added to the - client's ConnectionSet (this.connections). - - Params: - addr = address & port of node - min = miniumum hash which the node is responsible for - max = maxiumum hash which the node is responsible for - - ***************************************************************************/ - - public void updateNodeHashRange ( AddrPort addr, hash_t min, hash_t max ) - { - // Update an existing node. - if ( auto nhr = addr.cmp_id in this.node_hash_ranges ) - { - verify(nhr.addr == addr); - nhr.hash_range = NodeHashRange.HashRange(min, max); - nhr.order = order_counter++; - } - // Or add a new node. - else - this.newNode(addr, min, max); - } - - /*************************************************************************** - - Gets the list of nodes (along with hash range and ordering information) - which cover the specified hash, sorted (descending) by the values of - their `order` fields. (This means that the node most recently reported - as covering the specified hash will appear first in the list.) - - Params: - h = hash to query - node_hash_ranges = buffer to receive hash range information of - nodes which cover the specified hash - - Returns: - hash range information of nodes which cover the specified hash (a - slice of node_hash_ranges) - - ***************************************************************************/ - - public VoidBufferAsArrayOf!(NodeHashRange) getNodesForHash ( hash_t h, - VoidBufferAsArrayOf!(NodeHashRange) node_hash_ranges ) - { - node_hash_ranges.length = 0; - - foreach ( nhr; this.node_hash_ranges ) - { - if ( isWithinNodeResponsibility(h, - nhr.hash_range.min, nhr.hash_range.max) ) - { - node_hash_ranges ~= nhr; - } - } - - bool sortPred ( NodeHashRange e1, NodeHashRange e2 ) - { - verify(e1.order != e2.order); - return e1.order > e2.order; - } - - node_hash_ranges.array().sort(&sortPred); - - return node_hash_ranges; - } - - /*************************************************************************** - - Adds a new node to the set. (This method is protected as derived classes - may wish to add extra behaviour when adding a new node.) - - Params: - addr = address & port of new node - min = minimum hash of new node - max = maximum hash of new node - - ***************************************************************************/ - - protected void newNode ( AddrPort addr, hash_t min, hash_t max ) - { - this.node_hash_ranges[addr.cmp_id] = NodeHashRange( - addr, NodeHashRange.HashRange(min, max), order_counter++); - } -} - -version ( UnitTest ) -{ - import ocean.core.Test; - import Integer = ocean.text.convert.Integer_tango; - import swarm.neo.AddrPort; - import ocean.core.BitManip : bitswap; - - alias NodeHashRange.HashRange HR; - - void checkTestCase ( VoidBufferAsArrayOf!(NodeHashRange) r1, - NodeHashRange[] r2, long line_num = __LINE__ ) - { - auto t = new NamedTest(idup("Test at line " ~ Integer.toString(line_num))); - t.test!("==")(r1.length, r2.length); - - foreach ( i, e; r1.array() ) - { - t.test!("==")(e.addr, r2[i].addr); - t.test!("==")(e.hash_range.min, r2[i].hash_range.min); - t.test!("==")(e.hash_range.max, r2[i].hash_range.max); - t.test!("==")(e.order, r2[i].order); - } - } -} - -// Tests for hash range overlaps and gaps -unittest -{ - auto addr1 = AddrPort(1, 1); - auto addr2 = AddrPort(2, 2); - - void[] backing; - auto ranges = VoidBufferAsArrayOf!(NodeHashRange)(&backing); - auto hr = new NodeHashRangesBase; - - // Initially empty - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, []); - - // One node covering the whole range - hr.updateNodeHashRange(addr1, hash_t.min, hash_t.max); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr1, HR(hash_t.min, hash_t.max), 0)]); - - // A second node covering half the range (overlap) - hr.updateNodeHashRange(addr2, 0x8000000000000000, hash_t.max); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr1, HR(hash_t.min, hash_t.max), 0)]); - hr.getNodesForHash(hash_t.max, ranges); - checkTestCase(ranges, - [NodeHashRange(addr2, HR(0x8000000000000000, hash_t.max), 1), - NodeHashRange(addr1, HR(hash_t.min, hash_t.max), 0)]); - - // Change the range of the first node to cover the other half of the range - hr.updateNodeHashRange(addr1, hash_t.min, 0x7fffffffffffffff); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr1, HR(hash_t.min, 0x7fffffffffffffff), 2)]); - hr.getNodesForHash(hash_t.max, ranges); - checkTestCase(ranges, - [NodeHashRange(addr2, HR(0x8000000000000000, hash_t.max), 1)]); - - // Change the range of the second node to create a gap - // (Note that bitswap is used to mimic the internal behaviour of - // isWithinNodeResponsibility, used in getNodesForHash.) - hr.updateNodeHashRange(addr2, 0x9000000000000000, hash_t.max); - hr.getNodesForHash(bitswap(0x8000000000000000), ranges); - checkTestCase(ranges, []); - hr.getNodesForHash(hash_t.max, ranges); - checkTestCase(ranges, - [NodeHashRange(addr2, HR(0x9000000000000000, hash_t.max), 3)]); -} - -// Tests for hash range ordering -unittest -{ - // Reset ordering counter, as it was modified in other unittests - NodeHashRangesBase.order_counter = 0; - - auto addr1 = AddrPort(1, 1); - auto addr2 = AddrPort(2, 2); - auto addr3 = AddrPort(3, 3); - auto range = HR(hash_t.min, hash_t.max); - - void[] backing; - auto ranges = VoidBufferAsArrayOf!(NodeHashRange)(&backing); - auto hr = new NodeHashRangesBase; - - // Initially empty - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, []); - - // One node - hr.updateNodeHashRange(addr1, range.min, range.max); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr1, range, 0)]); - - // Add a second node - hr.updateNodeHashRange(addr2, range.min, range.max); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr2, range, 1), - NodeHashRange(addr1, range, 0)]); - - // Add a third node - hr.updateNodeHashRange(addr3, range.min, range.max); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr3, range, 2), - NodeHashRange(addr2, range, 1), - NodeHashRange(addr1, range, 0)]); - - // Modify the first node - hr.updateNodeHashRange(addr1, range.min, range.max); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr1, range, 3), - NodeHashRange(addr3, range, 2), - NodeHashRange(addr2, range, 1)]); - - // Modify the third node - hr.updateNodeHashRange(addr3, range.min, range.max); - hr.getNodesForHash(0, ranges); - checkTestCase(ranges, - [NodeHashRange(addr3, range, 4), - NodeHashRange(addr1, range, 3), - NodeHashRange(addr2, range, 1)]); -} diff --git a/src/dhtproto/client/internal/SharedResources.d b/src/dhtproto/client/internal/SharedResources.d deleted file mode 100644 index 07c0d089..00000000 --- a/src/dhtproto/client/internal/SharedResources.d +++ /dev/null @@ -1,282 +0,0 @@ -/******************************************************************************* - - Neo DHT client shared resources, available to all request handlers. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.internal.SharedResources; - -import ocean.transition; - -/// ditto -public final class SharedResources -{ - import swarm.neo.util.AcquiredResources; - import dhtproto.client.internal.NodeHashRanges; - import swarm.neo.client.ConnectionSet; - import ocean.util.container.pool.FreeList; - import ocean.core.TypeConvert : downcast; - import ocean.core.Verify; - import ocean.io.compress.Lzo; - import swarm.neo.util.MessageFiber; - import swarm.neo.util.VoidBufferAsArrayOf; - import swarm.util.RecordBatcher; - - /// Global NodeHashRanges instance - private NodeHashRanges node_hash_ranges_; - - /// Free list of recycled buffers - private FreeList!(ubyte[]) buffers; - - /// Free list of MessageFiber instances - private FreeList!(MessageFiber) fibers; - - /// Free list of RecordBatch instances - private FreeList!(RecordBatch) record_batches; - - /// Lzo instance shared by all record batches (newed on demand) - private Lzo lzo_; - - /*************************************************************************** - - A SharedResources instance is stored in the ConnectionSet as an Object. - This helper function safely casts from this Object to a correctly-typed - instance. - - Params: - obj = object to cast from - - Returns: - obj cast to SharedResources - - ***************************************************************************/ - - public static SharedResources fromObject ( Object obj ) - { - auto shared_resources = downcast!(SharedResources)(obj); - verify(shared_resources !is null); - return shared_resources; - } - - /*************************************************************************** - - Constructor. - - ***************************************************************************/ - - this ( ) - { - this.buffers = new FreeList!(ubyte[]); - this.fibers = new FreeList!(MessageFiber); - this.record_batches = new FreeList!(RecordBatch); - } - - /*************************************************************************** - - Sets the shared NodeHashRanges instance. (This cannot be done in a ctor, - because the NodeHashRanges instance requires a ConnectionSet, which is - owned by the Neo object, which requires a SharedResources instance to be - passed to its ctor.) - - Params: - node_hash_ranges = NodeHashRanges owned by the client - - ***************************************************************************/ - - public void setNodeHashRanges ( NodeHashRanges node_hash_ranges ) - { - this.node_hash_ranges_ = node_hash_ranges; - } - - /*************************************************************************** - - Returns: - shared NodeHashRanges instance - - ***************************************************************************/ - - public NodeHashRanges node_hash_ranges ( ) - { - return this.node_hash_ranges_; - } - - /*************************************************************************** - - Returns: - shared Lzo instance - - ***************************************************************************/ - - private Lzo lzo ( ) - { - if ( this.lzo_ is null ) - this.lzo_ = new Lzo; - - return this.lzo_; - } - - /*************************************************************************** - - Class to track the resources acquired by a request and relinquish them - (recylcing them into the shared resources pool) when the request - finishes. An instance should be newed as a request is started and - destroyed as it finishes. Newing an instance as `scope` is the most - convenient way. - - ***************************************************************************/ - - public class RequestResources - { - /// Set of acquired buffers - private AcquiredArraysOf!(void) acquired_buffers; - - /// Set of acquired arrays of buffer slices - private AcquiredArraysOf!(void[]) acquired_buffer_lists; - - /// Set of acquired buffers of NodeHashRange - private AcquiredArraysOf!(NodeHashRange) acquired_node_hash_range_buffers; - - /// Set of acquired fibers - private Acquired!(MessageFiber) acquired_fibers; - - /// Set of acquired record batches - private Acquired!(RecordBatch) acquired_record_batches; - - /*********************************************************************** - - Constructor. - - ***********************************************************************/ - - this ( ) - { - this.acquired_buffers.initialise(this.outer.buffers); - this.acquired_buffer_lists.initialise(this.outer.buffers); - this.acquired_node_hash_range_buffers.initialise(this.outer.buffers); - this.acquired_fibers.initialise(this.outer.buffers, - this.outer.fibers); - this.acquired_record_batches.initialise(this.outer.buffers, - this.outer.record_batches); - } - - /*********************************************************************** - - Destructor. Relinquishes any acquired resources. - - ***********************************************************************/ - - ~this ( ) - { - this.acquired_buffers.relinquishAll(); - this.acquired_buffer_lists.relinquishAll(); - this.acquired_node_hash_range_buffers.relinquishAll(); - this.acquired_fibers.relinquishAll(); - this.acquired_record_batches.relinquishAll(); - } - - /*********************************************************************** - - Returns: - a shared LZO instance - - ***********************************************************************/ - - public Lzo lzo ( ) - { - return this.outer.lzo; - } - - /*********************************************************************** - - Returns: - a pointer to a new chunk of memory (a void[]) to use during the - request's lifetime - - ***********************************************************************/ - - public void[]* getVoidBuffer ( ) - { - return this.acquired_buffers.acquire(); - } - - /*********************************************************************** - - Returns: - a void[] wrapped as an array of void[] (slices). The individual - slices that are added to the array should be acquired using - getVoidBuffer(). - - ***********************************************************************/ - - public VoidBufferAsArrayOf!(void[]) getBufferList ( ) - { - return this.acquired_buffer_lists.acquireWrapped(); - } - - /*********************************************************************** - - Returns: - a new NodeHashRange buffer acquired from the shared resources - pools - - ***********************************************************************/ - - public VoidBufferAsArrayOf!(NodeHashRange) getNodeHashRangeBuffer ( ) - { - return this.acquired_node_hash_range_buffers.acquireWrapped(); - } - - /*********************************************************************** - - Gets a fiber from the shared resources pool and assigns the provided - delegate as its entry point. - - Params: - fiber_method = entry point to assign to acquired fiber - - Returns: - a new MessageFiber acquired from the shared resources pools - - ***********************************************************************/ - - public MessageFiber getFiber ( void delegate ( ) fiber_method ) - { - bool new_fiber = false; - - MessageFiber newFiber ( ) - { - new_fiber = true; - return new MessageFiber(fiber_method, 64 * 1024); - } - - auto fiber = this.acquired_fibers.acquire(newFiber()); - if (!new_fiber) - fiber.reset(fiber_method); - - return fiber; - } - - /*********************************************************************** - - Gets a record batch from the shared resources pool. - - Returns: - a new record batch acquired from the shared resources pools - - ***********************************************************************/ - - public RecordBatch getRecordBatch ( ) - { - auto batch = this.acquired_record_batches.acquire( - new RecordBatch(this.outer.lzo())); - batch.clear(); - return batch; - } - } -} diff --git a/src/dhtproto/client/legacy/DhtConst.d b/src/dhtproto/client/legacy/DhtConst.d index 68ba4b93..c7f08754 100644 --- a/src/dhtproto/client/legacy/DhtConst.d +++ b/src/dhtproto/client/legacy/DhtConst.d @@ -66,7 +66,7 @@ static: ***************************************************************************/ - public const RecordSizeLimit = RecordBatcher.DefaultMaxBatchSize; + public const RecordSizeLimit = RecordBatch.DefaultMaxBatchSize; /*************************************************************************** diff --git a/src/dhtproto/client/legacy/internal/connection/DhtRequestConnection.d b/src/dhtproto/client/legacy/internal/connection/DhtRequestConnection.d index 838f69d3..7f6ebb3d 100644 --- a/src/dhtproto/client/legacy/internal/connection/DhtRequestConnection.d +++ b/src/dhtproto/client/legacy/internal/connection/DhtRequestConnection.d @@ -47,8 +47,6 @@ import swarm.client.request.GetChannelSizeRequest; import swarm.client.request.GetSizeRequest; import swarm.client.request.RemoveChannelRequest; -import swarm.util.RecordBatcher; - import dhtproto.client.legacy.internal.request.model.IDhtRequestResources; import dhtproto.client.legacy.internal.request.GetVersionRequest; import dhtproto.client.legacy.internal.request.GetResponsibleRangeRequest; @@ -231,7 +229,7 @@ public class DhtRequestConnection : override protected mstring new_batch_buffer ( ) { - return new char[RecordBatcher.DefaultMaxBatchSize]; + return new char[RecordBatch.DefaultMaxBatchSize]; } diff --git a/src/dhtproto/client/mixins/NeoSupport.d b/src/dhtproto/client/mixins/NeoSupport.d deleted file mode 100644 index a7f55ec6..00000000 --- a/src/dhtproto/client/mixins/NeoSupport.d +++ /dev/null @@ -1,1995 +0,0 @@ -/******************************************************************************* - - Neo protocol support for DhtClient. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.mixins.NeoSupport; - -/******************************************************************************* - - Template wrapping access to all "neo" features. Mix this class into a - DhtClient-derived class and construct the `neo` and 'blocking' objects in - your constructor. - -*******************************************************************************/ - -template NeoSupport ( ) -{ - import dhtproto.client.internal.SharedResources; - import dhtproto.client.internal.NodeHashRanges; - import swarm.neo.AddrPort; - import Hmac = swarm.neo.authentication.HmacDef; - import swarm.neo.client.NotifierTypes; - - /*************************************************************************** - - Class wrapping access to all "neo" features. (When the old protocol is - removed, the contents of this class will be moved into the top level of - the client class.) - - Usage example: - see the documented unittest, after the class definition - - ***************************************************************************/ - - public class Neo - { - import swarm.neo.client.mixins.ClientCore; - import swarm.neo.client.mixins.Controllers; - import swarm.neo.client.request_options.RequestOptions; - import ocean.core.SmartUnion; - import ocean.core.Verify; - - /*********************************************************************** - - Union of connection notification types. - - ***********************************************************************/ - - private union DhtConnNotificationUnion - { - NodeInfo connected; - - NodeInfo hash_range_queried; - - NodeExceptionInfo connection_error; - } - - /*********************************************************************** - - Smart-union of connection notification types. - - ***********************************************************************/ - - public alias SmartUnion!(DhtConnNotificationUnion) DhtConnNotification; - - /*********************************************************************** - - Alias for a delegate which receives a DhtConnNotification. - - ***********************************************************************/ - - public alias void delegate ( DhtConnNotification ) DhtConnectionNotifier; - - /*********************************************************************** - - Public imports of the request API modules, for the convenience of - user code. - - ***********************************************************************/ - - public import Put = dhtproto.client.request.Put; - public import Get = dhtproto.client.request.Get; - public import Remove = dhtproto.client.request.Remove; - public import Mirror = dhtproto.client.request.Mirror; - public import GetAll = dhtproto.client.request.GetAll; - public import GetChannels = dhtproto.client.request.GetChannels; - public import Exists = dhtproto.client.request.Exists; - public import RemoveChannel = dhtproto.client.request.RemoveChannel; - public import Update = dhtproto.client.request.Update; - - /*********************************************************************** - - Private imports of the request implementation modules. - - ***********************************************************************/ - - private struct Internals - { - import dhtproto.client.request.internal.GetHashRange; - import dhtproto.client.request.internal.Put; - import dhtproto.client.request.internal.Get; - import dhtproto.client.request.internal.Remove; - import dhtproto.client.request.internal.Mirror; - import dhtproto.client.request.internal.GetAll; - import dhtproto.client.request.internal.GetChannels; - import dhtproto.client.request.internal.Exists; - import dhtproto.client.request.internal.RemoveChannel; - import dhtproto.client.request.internal.Update; - } - - /*********************************************************************** - - Mixin core client internals (see - swarm.neo.client.mixins.ClientCore). - - ***********************************************************************/ - - mixin ClientCore!(); - - /*********************************************************************** - - Mixin `Controller` and `Suspendable` helper class templates (see - swarm.neo.client.mixins.Controllers). - - ***********************************************************************/ - - mixin Controllers!(); - - /*********************************************************************** - - Test instantiating the `Controller` and `Suspendable` class - templates. - - ***********************************************************************/ - - unittest - { - alias Controller!(Mirror.IController) MirrorController; - alias Suspendable!(Mirror.IController) MirrorSuspendable; - - alias Controller!(GetAll.IController) GetAllController; - alias Suspendable!(GetAll.IController) GetAllSuspendable; - } - - /*********************************************************************** - - DMQ request stats class. New an instance of this class to access - per-request stats. - - ***********************************************************************/ - - public alias RequestStatsTemplate!("Get", "Put", "Mirror", "GetAll", - "GetChannels", "Exists", "RemoveChannel") RequestStats; - - /*********************************************************************** - - DHT stats class. Extends the Stats class defined in ClientCore - with additional, DHT-specific stats. New an instance of this class - to access client-global stats. - - ***********************************************************************/ - - public class DhtStats : Stats - { - /******************************************************************* - - Returns: - the number of nodes for which the hash-range is known - - *******************************************************************/ - - public size_t num_nodes_known_hash_range ( ) - { - return this.outer.outer.shared_resources.node_hash_ranges.length; - } - } - - /*********************************************************************** - - Assigns a Put request, writing a record to the specified channel. - See $(LINK2 dhtproto/client/request/Put.html, dhtproto.client.request.Put) - for detailed documentation. - - Params: - channel = name of the channel to write to - key = hash of the record to write - value = record value to write (will be copied internally) - notifier = notifier delegate - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - TODO: allow optional settings to be specified via varargs - - ***********************************************************************/ - - public RequestId put ( cstring channel, hash_t key, Const!(void)[] value, - Put.Notifier notifier ) - { - auto params = Const!(Internals.Put.UserSpecifiedParams)( - Const!(Put.Args)(channel, key, value), notifier); - - auto id = this.assign!(Internals.Put)(params); - return id; - } - - /*********************************************************************** - - Assigns a Get request, reading a record from the specified channel. - See $(LINK2 dhtproto/client/request/Get.html, dhtproto.client.request.Get) - for detailed documentation. - - Params: - Options = tuple of types of additional arguments - channel = name of the channel to read from - key = hash of the record to read - notifier = notifier delegate - options = additional arguments. The following are supported: - dhtproto.client.request.Get.Timeout: sets the request to - time out after a specified time - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId get ( Options ... ) ( cstring channel, hash_t key, - Get.Notifier notifier, Options options ) - { - auto params = Const!(Internals.Get.UserSpecifiedParams)( - Const!(Get.Args)(channel, key), notifier); - - auto id = this.assign!(Internals.Get)(params); - - scope args_visitor = - ( Get.Timeout timeout ) - { - this.connections.request_set.setRequestTimeout( - id, timeout.ms); - }; - setupOptionalArgs!(options.length)(options, args_visitor); - - return id; - } - - /*********************************************************************** - - Assigns an Exists request, checking for the presence of a record in - the specified channel. - See $(LINK2 dhtproto/client/request/Exists.html, dhtproto.client.request.Exists) - for detailed documentation. - - Params: - channel = name of the channel to check in - key = hash of the record to check for - notifier = notifier delegate - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId exists ( cstring channel, hash_t key, - Exists.Notifier notifier ) - { - auto params = Const!(Internals.Exists.UserSpecifiedParams)( - Const!(Exists.Args)(channel, key), notifier); - - auto id = this.assign!(Internals.Exists)(params); - return id; - } - - /*********************************************************************** - - Assigns a Remove request, removing a record from the specified - channel. - See $(LINK2 dhtproto/client/request/Remove.html, dhtproto.client.request.Remove) - for detailed documentation. - - Params: - channel = name of the channel to remove a record from - key = hash of the record to remove - notifier = notifier delegate - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId remove ( cstring channel, hash_t key, - Remove.Notifier notifier ) - { - auto params = Const!(Internals.Remove.UserSpecifiedParams)( - Const!(Remove.Args)(channel, key), notifier); - - auto id = this.assign!(Internals.Remove)(params); - return id; - } - - /*********************************************************************** - - Assigns an Update request, fetching a record and replacing it with - an updated version. - See $(LINK2 dhtproto/client/request/Update.html, dhtproto.client.request.Update) - for detailed documentation. - - Params: - channel = name of the channel to update a record in - key = hash of the record to update - notifier = notifier delegate - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId update ( cstring channel, hash_t key, - Update.Notifier notifier ) - { - auto params = Const!(Internals.Update.UserSpecifiedParams)( - Const!(Update.Args)(channel, key), notifier); - - auto id = this.assign!(Internals.Update)(params); - return id; - } - - /*********************************************************************** - - Assigns a Mirror request, reading updates from the specified - channel. - See $(LINK2 dhtproto/client/request/Mirror.html, dhtproto.client.request.Mirror) - for detailed documentation. - - Params: - Options = tuple of types of additional arguments - channel = name of the channel to receive updates from - notifier = notifier delegate - options = additional arguments. The following are supported: - dhtproto.client.request.Mirror.Settings: Mirror behaviour - configuration - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId mirror ( Options ... ) ( cstring channel, - Mirror.Notifier notifier, Options options ) - { - Mirror.Settings settings; - - scope args_visitor = - ( Mirror.Settings user_settings ) - { - settings = user_settings; - }; - setupOptionalArgs!(options.length)(options, args_visitor); - - auto params = Const!(Internals.Mirror.UserSpecifiedParams)( - Const!(Mirror.Args)( - channel, - settings - ), notifier); - - auto id = this.assign!(Internals.Mirror)(params); - return id; - } - - /*********************************************************************** - - Assigns a GetAll request, reading all records from the specified - channel. - See $(LINK2 dhtproto/client/request/GetAll.html, dhtproto.client.request.GetAll) - for detailed documentation. - - Params: - Options = tuple of types of additional arguments - channel = name of the channel to read from - notifier = notifier delegate - options = additional arguments. The following are supported: - dhtproto.client.request.GetAll.Settings: GetAll behaviour - configuration - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId getAll ( Options ... ) ( cstring channel, - GetAll.Notifier notifier, Options options ) - { - GetAll.Settings settings; - - scope args_visitor = - ( GetAll.Settings user_settings ) - { - settings = user_settings; - }; - setupOptionalArgs!(options.length)(options, args_visitor); - - auto params = Const!(Internals.GetAll.UserSpecifiedParams)( - Const!(GetAll.Args)( - channel, - settings - ), notifier); - - auto id = this.assign!(Internals.GetAll)(params); - return id; - } - - /*********************************************************************** - - Assigns a GetChannels request, reading the names of all channels in - the DHT. - See $(LINK2 dhtproto/client/request/GetChannels.html, dhtproto.client.request.GetChannels) - for detailed documentation. - - Params: - notifier = notifier delegate - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId getChannels ( GetChannels.Notifier notifier ) - { - auto params = Const!(Internals.GetChannels.UserSpecifiedParams)( - Const!(GetChannels.Args)(), notifier); - - auto id = this.assign!(Internals.GetChannels)(params); - return id; - } - - /*********************************************************************** - - Assigns a RemoveChannel request to delete the specified channel from - the DHT. - - See $(LINK2 dhtproto/client/request/RemoveChannel.html, dhtproto.client.request.RemoveChannel) - for detailed documentation. - - Params: - channel = name of channel to remove - notifier = notifier delegate - - Returns: - id of newly assigned request - - Throws: - NoMoreRequests if the pool of active requests is full - - ***********************************************************************/ - - public RequestId removeChannel ( cstring channel, - RemoveChannel.Notifier notifier ) - { - auto params = Const!(Internals.RemoveChannel.UserSpecifiedParams)( - Const!(RemoveChannel.Args)(channel), notifier); - - auto id = this.assign!(Internals.RemoveChannel)(params); - return id; - } - - /*********************************************************************** - - Gets the type of the wrapper struct of the request associated with - the specified controller interface. - - Params: - I = type of controller interface - - Evaluates to: - the type of the request wrapper struct which contains an - implementation of the interface I - - ***********************************************************************/ - - private template Request ( I ) - { - static if ( is(I == Mirror.IController ) ) - { - alias Internals.Mirror Request; - } - else static if ( is(I == GetAll.IController ) ) - { - alias Internals.GetAll Request; - } - else - { - static assert(false, I.stringof ~ " does not match any request " - ~ "controller"); - } - } - - /*********************************************************************** - - Gets access to a controller for the specified request. If the - request is still active, the controller is passed to the provided - delegate for use. - - Important usage notes: - 1. The controller is newed on the stack. This means that user - code should never store references to it -- it must only be - used within the scope of the delegate. - 2. As the id which identifies the request is only known at run- - time, it is not possible to statically enforce that the - specified ControllerInterface type matches the request. This - is asserted at run-time, though (see - RequestSet.getRequestController()). - - Params: - ControllerInterface = type of the controller interface (should - be inferred by the compiler) - id = id of request to get a controller for (the return value of - the method which assigned your request) - dg = delegate which is called with the controller, if the - request is still active - - Returns: - false if the specified request no longer exists; true if the - controller delegate was called - - ***********************************************************************/ - - public bool control ( ControllerInterface ) ( RequestId id, - void delegate ( ControllerInterface ) dg ) - { - alias Request!(ControllerInterface) R; - - return this.controlImpl!(R)(id, dg); - } - - /*********************************************************************** - - Test instantiating the `control` function template. - - ***********************************************************************/ - - unittest - { - alias control!(Mirror.IController) mirrorControl; - alias control!(GetAll.IController) getAllControl; - } - - /*********************************************************************** - - Assigns a GetHashRange request. This request has no public API and - cannot be assigned by the user. It is assigned automatically in the - `neoInit` method of the outer class. - - ***********************************************************************/ - - private void assignGetHashRange ( ) - { - Internals.GetHashRange.UserSpecifiedParams params; - this.assign!(Internals.GetHashRange)(params); - } - } - - /*************************************************************************** - - Class wrapping access to all task-blocking "neo" features. (This - functionality is separated from the main neo functionality as it - implements methods with the same names and arguments (e.g. a callback- - based Put request and a task-blocking Put request).) - - ***************************************************************************/ - - public class TaskBlocking - { - import swarm.neo.client.mixins.TaskBlockingCore; - import ocean.core.Array : copy; - import ocean.task.Task; - - mixin TaskBlockingCore!(); - - /*********************************************************************** - - Suspends the current Task until a connection has been established to - the node listening on the specified address and port and its - hash-range queried. - - Params: - addr = address of node to connect to - port = port of node to connect to - - ***********************************************************************/ - - public void connect ( cstring addr, ushort port ) - { - this.outer.neo.addNode(addr, port); - this.waitAllHashRangesKnown(); - } - - /*********************************************************************** - - Suspends the current Task until a connection has been established to - all nodes listed in the specified config file and their hash-ranges - queried. The config file is expected to be in the format accepted by - swarm.neo.client.mixins.ClientCore.addNodes(). - - Params: - nodes_file = name of config file to read - - ***********************************************************************/ - - public void connect ( cstring nodes_file ) - { - this.outer.neo.addNodes(nodes_file); - this.waitAllHashRangesKnown(); - } - - /*********************************************************************** - - Suspends the current Task until a connection has been established to - all known nodes and their hash-ranges queried. - - ***********************************************************************/ - - public void waitAllHashRangesKnown ( ) - { - scope stats = this.outer.neo.new DhtStats; - - bool finished ( ) - { - return stats.num_nodes_known_hash_range == - stats.num_registered_nodes; - } - - this.waitHashRangeQuery(&finished); - } - - /*********************************************************************** - - Task class which ensures that the hash ranges of all nodes are - known. Intended for use with Scheduler.await() or - Scheduler.awaitResult(). - - ***********************************************************************/ - - public class AllHashRangesKnown : Task - { - /******************************************************************* - - Task main method. Exits when the client has received the hash - ranges of all registered nodes. - - *******************************************************************/ - - public override void run ( ) - { - this.outer.waitAllHashRangesKnown(); - } - } - - /*********************************************************************** - - Task class which ensures that the hash ranges of at least the - specified number of nodes are known. Intended for use with - Scheduler.await() or Scheduler.awaitResult(). - - ***********************************************************************/ - - public class HashRangesKnown : Task - { - /// When the task exits, holds the number of nodes whose hash ranges - /// are known. - public size_t result; - - /// The minimum number of node hash ranges to wait for. - private size_t minimum_hr_known; - - /******************************************************************* - - Constructor. - - Params: - minimum_hr_known = the minimum number of node hash ranges to - wait for - - *******************************************************************/ - - public this ( size_t minimum_hr_known ) - { - this.minimum_hr_known = minimum_hr_known; - } - - /******************************************************************* - - Task main method. Exits only when the client has received the - hash ranges for at least this.minimum_hr_known nodes. - - *******************************************************************/ - - public override void run ( ) - { - scope stats = this.outer.outer.neo.new DhtStats; - - bool finished ( ) - { - return stats.num_nodes_known_hash_range >= - this.minimum_hr_known; - } - - this.outer.waitHashRangeQuery(&finished); - - this.result = stats.num_nodes_known_hash_range; - } - } - - /*********************************************************************** - - Suspends the current task until the specified finished condition is - satisifed. - - Params: - finished = delegate specifying the condition under which the - method will return. The delegate is called once when the - method is called, and then again every time a hash-range - query event occurs. - - ***********************************************************************/ - - private void waitHashRangeQuery ( bool delegate ( ) finished ) - { - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - auto old_user_conn_notifier = this.outer.user_conn_notifier; - - scope conn_notifier = - ( Neo.DhtConnNotification info ) - { - old_user_conn_notifier(info); - - with ( info.Active ) switch ( info.active ) - { - case hash_range_queried: - if ( task.suspended() ) - task.resume(); - break; - - case connected: - case connection_error: - break; - - default: assert(false); - } - }; - - this.outer.user_conn_notifier = conn_notifier; - scope ( exit ) - this.outer.user_conn_notifier = old_user_conn_notifier; - - while ( !finished() ) - task.suspend(); - } - - /*********************************************************************** - - Assigns a Put request and blocks the current Task until the request - is completed. See - $(LINK2 dhtproto/client/request/Put.html, dhtproto.client.request.Put) - for detailed documentation. - - Params: - channel = name of the channel to write to - key = hash of the record to write - value = record value to write (will be copied internally) - user_notifier = notifier delegate - - ***********************************************************************/ - - public void put ( cstring channel, hash_t key, Const!(void)[] value, - Neo.Put.Notifier user_notifier ) - { - verify(user_notifier !is null); - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - bool finished; - - void notifier ( Neo.Put.Notification info, Const!(Neo.Put.Args) args ) - { - user_notifier(info, args); - - // All Put notifications indicate that the request has finished - // (no need to switch) - finished = true; - if ( task.suspended ) - task.resume(); - } - - this.outer.neo.put(channel, key, value, ¬ifier); - if ( !finished ) // if request not completed, suspend - task.suspend(); - verify(finished); - } - - /*********************************************************************** - - Struct returned after a Put request has finished. - - ***********************************************************************/ - - private struct PutResult - { - /******************************************************************* - - Set to true if the record was written to the DHT or false if an - error occurred. - - *******************************************************************/ - - bool succeeded; - } - - /*********************************************************************** - - Assigns a Put request and blocks the current Task until the request - is completed. See - $(LINK2 dhtproto/client/request/Put.html, dhtproto.client.request.Put) - for detailed documentation. - - Note that the API of this method is intentionally minimal (e.g. it - provides no detailed feedback about errors to the user). If you need - more control, use the method above which works via a notifier - callback. - - Params: - channel = name of the channel to write to - key = hash of the record to write - value = record value to write (will be copied internally) - - Returns: - PutResult struct, indicating the result of the request - - ***********************************************************************/ - - public PutResult put ( cstring channel, hash_t key, Const!(void)[] value ) - { - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - enum FinishedStatus - { - None, - Succeeded, - Failed - } - - FinishedStatus state; - - void notifier ( Neo.Put.Notification info, Const!(Neo.Put.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case success: - state = state.Succeeded; - if ( task.suspended ) - task.resume(); - break; - - case value_too_big: - case node_disconnected: - case node_error: - case unsupported: - case no_node: - case wrong_node: - case failure: - state = state.Failed; - if ( task.suspended ) - task.resume(); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - this.outer.neo.put(channel, key, value, ¬ifier); - if ( state == state.None ) // if request not completed, suspend - task.suspend(); - verify(state != state.None); - - PutResult res; - res.succeeded = state == state.Succeeded; - return res; - } - - /*********************************************************************** - - Assigns a Get request and blocks the current Task until the request - is completed. See - $(LINK2 dhtproto/client/request/Get.html, dhtproto.client.request.Get) - for detailed documentation. - - Params: - channel = name of the channel to read from - key = hash of the record to read - user_notifier = notifier delegate - - ***********************************************************************/ - - public void get ( cstring channel, hash_t key, - Neo.Get.Notifier user_notifier ) - { - verify(user_notifier !is null); - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - bool finished; - - void notifier ( Neo.Get.Notification info, Const!(Neo.Get.Args) args ) - { - user_notifier(info, args); - - // All Get notifications indicate that the request has finished - // (no need to switch) - finished = true; - if ( task.suspended ) - task.resume(); - } - - this.outer.neo.get(channel, key, ¬ifier); - if ( !finished ) // if request not completed, suspend - task.suspend(); - verify(finished); - } - - /*********************************************************************** - - Struct returned after a Get request has finished. - - ***********************************************************************/ - - private struct GetResult - { - /******************************************************************* - - Set to true if no error occurred. - - *******************************************************************/ - - bool succeeded; - - /******************************************************************* - - The value read from the channel or an empty array, if no record - exists in the channel for the specified key or an error - occurred. - - *******************************************************************/ - - void[] value; - } - - /*********************************************************************** - - Assigns a Get request and blocks the current Task until the request - is completed. See - $(LINK2 dhtproto/client/request/Get.html, dhtproto.client.request.Get) - for detailed documentation. - - Note that the API of this method is intentionally minimal (e.g. it - provides no detailed feedback about errors to the user). If you need - more control, use the method above which works via a notifier - callback. - - Params: - channel = name of the channel to read from - key = hash of the record to read - value = buffer to receive the read value (will be set to length - 0, if key does not exists in the specified channel) - - Returns: - GetResult struct, indicating the result of the request - - ***********************************************************************/ - - public GetResult get ( cstring channel, hash_t key, ref void[] value ) - { - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - enum FinishedStatus - { - None, - Succeeded, - Failed - } - - GetResult res; - FinishedStatus state; - - void notifier ( Neo.Get.Notification info, Const!(Neo.Get.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case received: - value.copy(info.received.value); - res.value = value; - goto case no_record; - - case no_record: - state = state.Succeeded; - if ( task.suspended ) - task.resume(); - break; - - case node_disconnected: - case node_error: - case unsupported: - case no_node: - case wrong_node: - case timed_out: - case failure: - state = state.Failed; - if ( task.suspended ) - task.resume(); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - this.outer.neo.get(channel, key, ¬ifier); - if ( state == state.None ) // if request not completed, suspend - task.suspend(); - verify(state != state.None); - - res.succeeded = state == state.Succeeded; - return res; - } - - /*********************************************************************** - - Assigns an Update request and blocks the current Task until the - request is completed. See - $(LINK2 dhtproto/client/request/Update.html, dhtproto.client.request.Update) - for detailed documentation. - - Params: - channel = name of the channel to update a record in - key = hash of the record to update - user_notifier = notifier delegate - - ***********************************************************************/ - - public void update ( cstring channel, hash_t key, - Neo.Update.Notifier user_notifier ) - { - verify(user_notifier !is null); - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - bool finished; - - void notifier ( Neo.Update.Notification info, - Const!(Neo.Update.Args) args ) - { - user_notifier(info, args); - - with ( info.Active ) switch ( info.active ) - { - case succeeded: // Updated successfully. - case no_record: // Record not in DHT. Use Put to write a new record. - case conflict: // Another client updated the same record. Try again. - case error: - case no_node: - finished = true; - if ( task.suspended ) - task.resume(); - break; - - default: - break; - } - } - - this.outer.neo.update(channel, key, ¬ifier); - if ( !finished ) // if request not completed, suspend - task.suspend(); - verify(finished); - } - - /*********************************************************************** - - Assigns an Exists request and blocks the current Task until the - request is completed. See - $(LINK2 dhtproto/client/request/Exists.html, dhtproto.client.request.Exists) - for detailed documentation. - - Params: - channel = name of the channel to check in - key = hash of the record to check for - user_notifier = notifier delegate - - ***********************************************************************/ - - public void exists ( cstring channel, hash_t key, - Neo.Exists.Notifier user_notifier ) - { - verify(user_notifier !is null); - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - bool finished; - - void notifier ( Neo.Exists.Notification info, - Const!(Neo.Exists.Args) args ) - { - user_notifier(info, args); - - // All Exists notifications indicate that the request has - // finished (no need to switch) - finished = true; - if ( task.suspended ) - task.resume(); - } - - this.outer.neo.exists(channel, key, ¬ifier); - if ( !finished ) // if request not completed, suspend - task.suspend(); - verify(finished); - } - - /*********************************************************************** - - Struct returned after an Exists request has finished. - - ***********************************************************************/ - - private struct ExistsResult - { - /******************************************************************* - - Set to true if no error occurred. - - *******************************************************************/ - - bool succeeded; - - /******************************************************************* - - If the request succeeded, stores the result of the request. - - *******************************************************************/ - - bool exists; - } - - /*********************************************************************** - - Assigns an Exists request and blocks the current Task until the - request is completed. See - $(LINK2 dhtproto/client/request/Exists.html, dhtproto.client.request.Exists) - for detailed documentation. - - Note that the API of this method is intentionally minimal (e.g. it - provides no detailed feedback about errors to the user). If you need - more control, use the method above which works via a notifier - callback. - - Params: - channel = name of the channel to check in - key = hash of the record to check for - - Returns: - ExistsResult struct, indicating the result of the request - - ***********************************************************************/ - - public ExistsResult exists ( cstring channel, hash_t key ) - { - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - bool finished; - ExistsResult res; - - void notifier ( Neo.Exists.Notification info, Const!(Neo.Exists.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case exists: - res.exists = true; - res.succeeded = true; - break; - - case no_record: - res.exists = false; - res.succeeded = true; - break; - - case node_disconnected: - case node_error: - case unsupported: - case no_node: - case wrong_node: - case failure: - res.succeeded = false; - break; - - mixin(typeof(info).handleInvalidCases); - } - - // All Exists notifications indicate that the request has - // finished. - finished = true; - if ( task.suspended ) - task.resume(); - } - - this.outer.neo.exists(channel, key, ¬ifier); - if ( !finished ) // if request not completed, suspend - task.suspend(); - verify(finished); - - return res; - } - - /*********************************************************************** - - Assigns a Remove request and blocks the current Task until the - request is completed. See - $(LINK2 dhtproto/client/request/Remove.html, dhtproto.client.request.Remove) - for detailed documentation. - - Params: - channel = name of the channel to remove a record from - key = hash of the record to remove - user_notifier = notifier delegate - - ***********************************************************************/ - - public void remove ( cstring channel, hash_t key, - Neo.Remove.Notifier user_notifier ) - { - verify(user_notifier !is null); - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - bool finished; - - void notifier ( Neo.Remove.Notification info, - Const!(Neo.Remove.Args) args ) - { - user_notifier(info, args); - - // All Remove notifications indicate that the request has - // finished (no need to switch) - finished = true; - if ( task.suspended ) - task.resume(); - } - - this.outer.neo.remove(channel, key, ¬ifier); - if ( !finished ) // if request not completed, suspend - task.suspend(); - verify(finished); - } - - /*********************************************************************** - - Struct returned after a Remove request has finished. - - ***********************************************************************/ - - private struct RemoveResult - { - /******************************************************************* - - Set to true if no error occurred. - - *******************************************************************/ - - bool succeeded; - - /******************************************************************* - - Set to true if the record existed. - - *******************************************************************/ - - bool existed; - } - - /*********************************************************************** - - Assigns a Remove request and blocks the current Task until the - request is completed. See - $(LINK2 dhtproto/client/request/Remove.html, dhtproto.client.request.Remove) - for detailed documentation. - - Note that the API of this method is intentionally minimal (e.g. it - provides no detailed feedback about errors to the user). If you need - more control, use the method above which works via a notifier - callback. - - Params: - channel = name of the channel to remove a record from - key = hash of the record to remove - - Returns: - RemoveResult struct, indicating the result of the request - - ***********************************************************************/ - - public RemoveResult remove ( cstring channel, hash_t key ) - { - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - enum FinishedStatus - { - None, - Succeeded, - Failed - } - - RemoveResult res; - FinishedStatus state; - - void notifier ( Neo.Remove.Notification info, - Const!(Neo.Remove.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case removed: - res.existed = true; - goto case no_record; - - case no_record: - state = state.Succeeded; - if ( task.suspended ) - task.resume(); - break; - - case node_disconnected: - case node_error: - case unsupported: - case no_node: - case wrong_node: - case failure: - state = state.Failed; - if ( task.suspended ) - task.resume(); - break; - - default: assert(false); - } - } - - this.outer.neo.remove(channel, key, ¬ifier); - if ( state == state.None ) // if request not completed, suspend - task.suspend(); - verify(state != state.None); - - res.succeeded = state == state.Succeeded; - return res; - } - - /*********************************************************************** - - Struct to provide an opApply for a task-blocking GetAll. - - Node that the Task-blocking GetAll consciously provides a - simplistic, clean API, without any of the more advanced features of - the request (e.g. suspending). If you need these features, please - use the standard, callback-based version of the request. - - ***********************************************************************/ - - public struct GetAllFruct - { - import ocean.core.array.Mutation: copy; - - /// User task to resume/suspend. - private Task task; - - /// Key of the current record. - private hash_t record_key; - - /// Value of the current record. - private void[]* record_value; - - /// Possible states of the request. - private enum State - { - /// The request is running. - Running, - - /// The user has stopped this request by breaking from foreach - /// (the request may still be running for some time, but all - /// records will be ignored). - Stopped, - - /// The request has finished on all nodes. - Finished - } - - /// Indicator of the request's state. - private State state; - - /// Channel to iterate over. - private cstring channel; - - /// Neo instance to assign the request with. - private Neo neo; - - /// Error indicator. - public bool error; - - /// Request id (used internally). - private Neo.RequestId rq_id; - - /******************************************************************* - - Notifier used to set the local values and resume the task. - - Params: - info = information and payload about the event user has - been notified about - args = arguments passed by the user when starting request - - *******************************************************************/ - - private void notifier ( Neo.GetAll.Notification info, - Const!(Neo.GetAll.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case received: - // Ignore all received value on user break. - if (this.state == State.Stopped) - break; - - // Store the received value. - this.record_key = info.received.key; - - copy(*this.record_value, info.received.value); - - if (this.task.suspended()) - { - this.task.resume(); - } - break; - - case stopped: - case finished: - // Even if the user has requested stopping, - // but finished arrived, we will just finish and exit. - this.state = State.Finished; - this.task.resume(); - break; - - case suspended: // Unexepected (unsupported by blocking API) - case resumed: // Unexepected (unsupported by blocking API) - case node_disconnected: - case node_error: - case unsupported: - // Ignore all errors on user break. - if (this.state == State.Stopped) - break; - - // Otherwise flag an error and allow the request to - // finish normally. - this.error = true; - break; - - case received_key: - // Not yet supported by blocking GetAll. - break; - - case started: - // Irrelevant. - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - /******************************************************************* - - Task-blocking opApply iteration over GetAll. - - *******************************************************************/ - - public int opApply ( int delegate ( ref hash_t key, - ref void[] value ) dg ) - { - int ret; - - this.rq_id = this.neo.getAll(this.channel, &this.notifier); - - while (this.state != State.Finished) - { - Task.getThis().suspend(); - - // No more records. - if (this.state == State.Finished - || this.state == State.Stopped - || this.error) - break; - - ret = dg(this.record_key, *this.record_value); - - if (ret) - { - this.state = State.Stopped; - - this.neo.control(this.rq_id, - ( Neo.GetAll.IController get_all ) - { - get_all.stop(); - }); - - // Wait for the request to finish. - Task.getThis().suspend(); - break; - } - } - - return ret; - } - } - - /*********************************************************************** - - Assigns a task blocking GetAll request, getting the values from - the specified channel and range. This method provides nothing but - the most basic usage (no request context, no way to control the - request (stop/resume/suspend)), so if that is needed, please use the - non-task blocking getAll. - - Breaking out of the iteration stops the GetAll request. - - Params: - channel = name of the channel to get the records from - record_buffer = reusable buffer to store the current record's - values into - - Returns: - GetAllFruct structure, whose opApply should be used - - ***********************************************************************/ - - public GetAllFruct getAll ( cstring channel, ref void[] record_buffer ) - { - auto task = Task.getThis(); - verify(task !is null, - "This method may only be called from inside a Task"); - - GetAllFruct res; - res.task = task; - res.neo = this.outer.neo; - res.record_value = &record_buffer; - res.channel = channel; - - return res; - } - - /*********************************************************************** - - Struct to provide an opApply for a task-blocking GetChannels. - - ***********************************************************************/ - - public struct GetChannelsFruct - { - import ocean.core.array.Mutation: copy; - - /// User task to resume/suspend. - private Task task; - - /// Name of the current channel. - private mstring* channel_name; - - /// Possible states of the request. - private enum State - { - /// The request is running. - Running, - - /// The user has stopped this request by breaking from foreach - /// (the request may still be running for some time, but all - /// channel names will be ignored). - Stopped, - - /// The request has finished on all nodes. - Finished - } - - /// Indicator of the request's state. - private State state; - - /// Neo instance to assign the request with. - private Neo neo; - - /// Error indicator. - public bool error; - - /******************************************************************* - - Notifier used to set the local values and resume the task. - - Params: - info = information and payload about the event user has - been notified about - args = arguments passed by the user when starting request - - *******************************************************************/ - - private void notifier ( Neo.GetChannels.Notification info, - Const!(Neo.GetChannels.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case received: - // Ignore all received value on user break. - if (this.state == State.Stopped) - break; - - copy(*this.channel_name, - cast(cstring)info.received.value); - - if (this.task.suspended()) - { - this.task.resume(); - } - break; - - case finished: - this.state = State.Finished; - this.task.resume(); - break; - - case node_disconnected: - case node_error: - case unsupported: - // Ignore all errors on user break. - if (this.state == State.Stopped) - break; - - // Otherwise flag an error and allow the request to - // finish normally. - this.error = true; - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - /******************************************************************* - - Task-blocking opApply iteration over GetChannels. - - *******************************************************************/ - - public int opApply ( int delegate ( ref mstring channel_name ) dg ) - { - int ret; - - this.neo.getChannels(&this.notifier); - - while (this.state != State.Finished) - { - Task.getThis().suspend(); - - // No more records. - if (this.state == State.Finished - || this.state == State.Stopped - || this.error) - break; - - ret = dg(*this.channel_name); - - if (ret) - { - this.state = State.Stopped; - - // Wait for the request to finish. - Task.getThis().suspend(); - break; - } - } - - return ret; - } - } - - /*********************************************************************** - - Assigns a task blocking GetChannels request, getting the channel - names from the DHT. - - Params: - channel_buffer = reusable buffer to store the current channel's - name in - - Returns: - GetChannelsFruct structure, whose opApply should be used - - ***********************************************************************/ - - public GetChannelsFruct getChannels ( ref mstring channel_buffer ) - { - auto task = Task.getThis(); - verify(task !is null, - "This method may only be called from inside a Task"); - - GetChannelsFruct res; - res.task = task; - res.neo = this.outer.neo; - res.channel_name = &channel_buffer; - - return res; - } - - /*********************************************************************** - - Struct returned after a RemoveChannel request has finished. - - ***********************************************************************/ - - private struct RemoveChannelResult - { - /******************************************************************* - - Set to true if the channel was removed from the DHT or did not - exist. False if an error occurred. - - *******************************************************************/ - - bool succeeded; - } - - /*********************************************************************** - - Assigns a RemoveChannel request and blocks the current Task until - the request is completed. See - $(LINK2 dhtproto/client/request/RemoveChannel.html, dhtproto.client.request.RemoveChannel) - for detailed documentation. - - Note that the API of this method is intentionally minimal (e.g. it - provides no detailed feedback about errors to the user). If you need - more control, use the method above which works via a notifier - callback. - - Params: - channel = name of the channel to remove - - Returns: - RemoveChannelResult struct, indicating the result of the request - - ***********************************************************************/ - - public RemoveChannelResult removeChannel ( cstring channel ) - { - auto task = Task.getThis(); - verify(task !is null, "This method may only be called from inside a Task"); - - enum FinishedStatus - { - None, - Succeeded, - Failed - } - - FinishedStatus state; - - void notifier ( Neo.RemoveChannel.Notification info, - Neo.RemoveChannel.Args args ) - { - with ( info.Active ) final switch ( info.active ) - { - case finished: - state = state.Succeeded; - if ( task.suspended ) - task.resume(); - break; - - case not_permitted: - case node_disconnected: - case node_error: - case unsupported: - state = state.Failed; - if ( task.suspended ) - task.resume(); - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - this.outer.neo.removeChannel(channel, ¬ifier); - if ( state == state.None ) // if request not completed, suspend - task.suspend(); - verify(state != state.None); - - RemoveChannelResult res; - res.succeeded = state == state.Succeeded; - return res; - } - } - - /*************************************************************************** - - Object containing all neo functionality. - - ***************************************************************************/ - - public Neo neo; - - /*************************************************************************** - - Object containing all neo task-blocking functionality. - - ***************************************************************************/ - - public TaskBlocking blocking; - - /*************************************************************************** - - Global resources required by all requests. Passed to the ConnectionSet. - - ***************************************************************************/ - - private SharedResources shared_resources; - - /*************************************************************************** - - Callback for notifying the user about connection success / failure and - hash-range queries. - - ***************************************************************************/ - - private Neo.DhtConnectionNotifier user_conn_notifier; - - /*************************************************************************** - - Helper function to initialise neo components. Automatically calls - addNodes() with the node definition files specified in the Config - instance. - - Params: - config = swarm.client.model.IClient.Config instance. (The Config - class is designed to be read from an application's config.ini - file via ocean.util.config.ConfigFiller.) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established) and when the hash-range for a node is first - queried. Of type: - void delegate ( Neo.DhtConnNotification ) - - ***************************************************************************/ - - private void neoInit ( Neo.Config config, - Neo.DhtConnectionNotifier user_conn_notifier ) - { - verify(user_conn_notifier !is null); - this.user_conn_notifier = user_conn_notifier; - - this.shared_resources = new SharedResources; - this.neo = new Neo(config, - Neo.Settings(&this.connNotifier, this.shared_resources)); - auto node_hash_ranges = new NodeHashRanges(this.neo.connections, - &this.hashRangeNotifier); - this.shared_resources.setNodeHashRanges(node_hash_ranges); - this.blocking = new TaskBlocking; - - this.neo.assignGetHashRange(); - } - - /*************************************************************************** - - Helper function to initialise neo components. This initialiser that - accepts all arguments manually (i.e. not read from config files) is - mostly of use in tests. - - Params: - auth_name = client name for authorisation - auth_key = client key (password) for authorisation. This should be a - properly generated random number which only the client and the - nodes know. See `swarm/README_client_neo.rst` for suggestions. - The key must be of the length defined in - swarm.neo.authentication.HmacDef (128 bytes) - conn_notifier = delegate which is called when a connection attempt - succeeds or fails (including when a connection is - re-established) and when the hash-range for a node is first - queried. Of type: - void delegate ( Neo.DhtConnNotification ) - - ***************************************************************************/ - - private void neoInit ( cstring auth_name, ubyte[] auth_key, - Neo.DhtConnectionNotifier user_conn_notifier ) - { - verify(user_conn_notifier !is null); - this.user_conn_notifier = user_conn_notifier; - - this.shared_resources = new SharedResources; - this.neo = new Neo(auth_name, auth_key, - Neo.Settings(&this.connNotifier, this.shared_resources)); - auto node_hash_ranges = new NodeHashRanges(this.neo.connections, - &this.hashRangeNotifier); - this.shared_resources.setNodeHashRanges(node_hash_ranges); - this.blocking = new TaskBlocking; - - this.neo.assignGetHashRange(); - } - - /*************************************************************************** - - Neo client connection notifier. Calls this.user_conn_notifier as - appropriate. - - Params: - addr = address of node for which connection succeeded / failed - e = exception. If null, the connection succeeded. Otherwise, - indicates the error which prevented connection - - ***************************************************************************/ - - private void connNotifier ( Neo.ConnNotification info ) - { - Neo.DhtConnNotification n; - with ( info.Active ) final switch ( info.active ) - { - case connected: - n.connected = NodeInfo(info.connected.node_addr); - break; - case error_while_connecting: - n.connection_error = NodeExceptionInfo( - info.error_while_connecting.node_addr, - info.error_while_connecting.e); - break; - mixin(typeof(info).handleInvalidCases); - } - - this.user_conn_notifier(n); - } - - /*************************************************************************** - - NodeHashRanges new node notifier. Calls this.user_conn_notifier as - appropriate. - - Params: - addr = address of node for which hash-range info has been queried - min = minimum hash covered by node - max = maximum hash covered by node - - ***************************************************************************/ - - private void hashRangeNotifier ( AddrPort addr, hash_t min, hash_t max ) - { - Neo.DhtConnNotification n; - n.hash_range_queried = NodeInfo(addr); - - this.user_conn_notifier(n); - } -} diff --git a/src/dhtproto/client/request/Exists.d b/src/dhtproto/client/request/Exists.d deleted file mode 100644 index 962421e4..00000000 --- a/src/dhtproto/client/request/Exists.d +++ /dev/null @@ -1,102 +0,0 @@ -/******************************************************************************* - - Client DHT Exists request definitions / handler. - - The Exists request checks for the existence of a record in the specified DHT - channel. This works as follows: - 1. The client selects a connected DHT node from its registry, based on - the key of the record to check for. - 2. A request is sent to the selected node, asking for the existence of - the specified record in the specified channel to be checked. - 3. The request ends when either the record is confirmed to exist or not - or the node could not handle the request due to an error. - - During a data redistribution, more than one node may be responsible for a - given key. In this case, the node which was most recently reported as being - responsible for the key is queried first, followed by others (in order) - until the record is located, an error occurs, or no node has the record. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.Exists; - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - mstring channel; - hash_t key; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - * wrong_node - -*******************************************************************************/ - -private union NotificationUnion -{ - /// The request succeeded and the record exists. - RequestInfo exists; - - /// The request succeeded, but no record exists with the specified key. - RequestInfo no_record; - - /// The request failed due to a connection error. - RequestNodeExceptionInfo node_disconnected; - - /// The request failed due to an internal node error. - RequestNodeInfo node_error; - - /// The request failed because it is unsupported. - RequestNodeUnsupportedInfo unsupported; - - /// The DHT node to which the request was sent is not responsible for the - /// record's key. This is a sanity check performed within the node in order - /// to avoid data inconsistency. - RequestNodeInfo wrong_node; - - /// No DHT node is known to cover the hash of the request. Note that this - /// may be because the client has just been started and has not received - /// hash range information from the DHT yet. - RequestInfo no_node; - - /// Internal error in client. - RequestInfo failure; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notifcation delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; diff --git a/src/dhtproto/client/request/Get.d b/src/dhtproto/client/request/Get.d deleted file mode 100644 index 196313dd..00000000 --- a/src/dhtproto/client/request/Get.d +++ /dev/null @@ -1,126 +0,0 @@ -/******************************************************************************* - - Client DHT Get request definitions / handler. - - The Get request attempts to read one record from the specified DHT channel. - This works as follows: - 1. The client selects a connected DHT node from its registry, based on - the key of the record to be read. - 2. A request is sent to the selected node, asking for the specified - record in the specified channel to be returned. - 3. The request ends when either the record is read from the node, is - found to not exist, or the node could not handle the request due to - an error. - - During a data redistribution, more than one node may be responsible for a - given key. In this case, the node which was most recently reported as being - responsible for the key is queried first, followed by others (in order) - until the record is located, an error occurs, or no node has the record. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.Get; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - mstring channel; - hash_t key; -} - -/******************************************************************************* - - Optional struct to be passed when assigning a Get request. Specifies that - the request should time out after the specified period, if it has not - completed. - -*******************************************************************************/ - -public struct Timeout -{ - /// Timeout in milliseconds. - uint ms; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - * wrong_node - -*******************************************************************************/ - -private union NotificationUnion -{ - /// The request succeeded, but no record exists with the specified key. - RequestInfo no_record; - - /// The request succeeded and the value was read. - RequestDataInfo received; - - /// The request timed out before it completed. - RequestInfo timed_out; - - /// The request failed due to a connection error. - RequestNodeExceptionInfo node_disconnected; - - /// The request failed due to an internal node error. - RequestNodeInfo node_error; - - /// The request failed because it is unsupported. - RequestNodeUnsupportedInfo unsupported; - - /// The DHT node to which the request was sent is not responsible for the - /// record's key. This is a sanity check performed within the node in order - /// to avoid data inconsistency. - RequestNodeInfo wrong_node; - - /// No DHT node is known to cover the hash of the request. Note that this - /// may be because the client has just been started and has not received - /// hash range information from the DHT yet. - RequestInfo no_node; - - /// Internal error in client. - RequestInfo failure; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notifcation delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; diff --git a/src/dhtproto/client/request/GetAll.d b/src/dhtproto/client/request/GetAll.d deleted file mode 100644 index 77d787d5..00000000 --- a/src/dhtproto/client/request/GetAll.d +++ /dev/null @@ -1,177 +0,0 @@ -/******************************************************************************* - - Client DHT GetAll request definitions / handler. - - The GetAll request communicates with all DHT nodes to fetch all records in a - channel. If a connection error occurs, the request will be restarted once - the connection has been re-established and will continue from where it left - off. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.GetAll; - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; -public import dhtproto.client.NotifierTypes; - -/******************************************************************************* - - GetAll behaviour settings. May be optionally passed to the getAll() method - of the DHT client, to modify the default behaviour. - -*******************************************************************************/ - -public struct Settings -{ - /// If true, only record keys will be fetched, no values. - bool keys_only; - - /// If non-empty, only records whose values contain these bytes as a - /// "substring" will be fetched. - void[] value_filter; -} - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - /// Name of the channel to fetch. - mstring channel; - - /// Settings for the behaviour of the request. - Settings settings; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - -*******************************************************************************/ - -private union NotificationUnion -{ - /// The request is now in a state where it can be suspended / resumed / - /// stopped via the controller. (This means that all known nodes have either - /// started handling the request or are not currently connected.) Note that - /// records may be received from some nodes before this notification occurs. - RequestInfo started; - - /// A record has been received from the DHT. - RequestRecordInfo received; - - /// A record key has been received from the DHT. (Only used when in - /// keys-only mode. See Settings.) - RequestKeyInfo received_key; - - /// The connection to a node has disconnected; the request will - /// automatically continue after reconnection. - RequestNodeExceptionInfo node_disconnected; - - /// A node returned a non-OK status code. The request cannot be handled by - /// this node. - RequestNodeInfo node_error; - - /// The request was tried on a node and failed because it is unsupported. - /// The request cannot be handled by this node. - RequestNodeUnsupportedInfo unsupported; - - /// All known nodes have either suspended the request (as requested by the - /// user, via the controller) or are not currently connected. - RequestInfo suspended; - - /// All known nodes have either resumed the request (as requested by the - /// user, via the controller) or are not currently connected. - RequestInfo resumed; - - /// All known nodes have either stopped the request (as requested by the - /// user, via the controller) or are not currently connected. The request is - /// now finished. (A `finished` notification will not occur as well.) - RequestInfo stopped; - - /// All known nodes have either finished the request or are not currently - /// connected. The request is now finished. - RequestInfo finished; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notification delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; - -/******************************************************************************* - - Request controller, accessible via the client's `control()` method. - - Note that only one control change message can be "in-flight" to the nodes at - a time. If the controller is used when a control change message is already - in-flight, the method will return false. The notifier is called when a - requested control change is carried through. - -*******************************************************************************/ - -public interface IController -{ - /*************************************************************************** - - Tells the nodes to stop sending data to this request. - - Returns: - false if the controller cannot be used because a control change is - already in progress - - ***************************************************************************/ - - bool suspend ( ); - - /*************************************************************************** - - Tells the nodes to resume sending data to this request. - - Returns: - false if the controller cannot be used because a control change is - already in progress - - ***************************************************************************/ - - bool resume ( ); - - /*************************************************************************** - - Tells the nodes to cleanly end the request. - - Returns: - false if the controller cannot be used because a control change is - already in progress - - ***************************************************************************/ - - bool stop ( ); -} diff --git a/src/dhtproto/client/request/GetChannels.d b/src/dhtproto/client/request/GetChannels.d deleted file mode 100644 index 0d92a2c0..00000000 --- a/src/dhtproto/client/request/GetChannels.d +++ /dev/null @@ -1,81 +0,0 @@ -/******************************************************************************* - - Client DHT GetChannels request definitions / handler. - - The GetChannels request communicates with all DHT nodes to fetch the names - of all channels in the DHT. If a connection error occurs, the request will - be restarted once the connection has been re-established. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.GetChannels; - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - // Nothing. -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - -*******************************************************************************/ - -private union NotificationUnion -{ - /// A channel name has been received from the DHT. - RequestDataInfo received; - - /// The connection to a node has disconnected; the request will - /// automatically restart after reconnection. - RequestNodeExceptionInfo node_disconnected; - - /// A node returned a non-OK status code. The request cannot be handled by - /// this node. - RequestNodeInfo node_error; - - /// The request was tried on a node and failed because it is unsupported. - /// The request cannot be handled by this node. - RequestNodeUnsupportedInfo unsupported; - - /// All known nodes have either finished the request or are not currently - /// connected. The request is now finished. - RequestInfo finished; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notification delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; diff --git a/src/dhtproto/client/request/Mirror.d b/src/dhtproto/client/request/Mirror.d deleted file mode 100644 index 16099672..00000000 --- a/src/dhtproto/client/request/Mirror.d +++ /dev/null @@ -1,210 +0,0 @@ -/******************************************************************************* - - Client DHT Mirror request definitions / handler. - - The Mirror request communicates with all DHT nodes to fetch records from a - whole channel (i.e. the request creates a "mirror" of the channel in the - client). When a record is modified in the channel, an update is immediately - sent to the client via the Mirror request. The request can also be - configured to periodically perform a "refresh" on all records -- that is, - fetch a copy of all records in the mirrored channel, irrespective of whether - they've changed recently or not. Two options are possible here (usable in - any combination): - 1. Initial refresh: when the request starts, all records in the channel - are sent to the client via the Mirror request. - 2. Periodic refreshes: every N seconds (as specified by the user), all - records in the channel are sent to the client via the Mirror request. - - The most common ways of using a Mirror request are: - a. To get an initial dump of all records in a channel, followed by a - stream of updates to records in the channel. (Refresh mode 1.) - b. To get an initial dump of all records in a channel, followed by a - stream of updates to records in the channel plus a periodic fresh - dump of the values of all records (the latter as a fall-back, in case - any errors occurred in transferring updates via the stream). (Refresh - modes 1 and 2.) (This is the default behaviour of the request, if no - settings are configured by the user.) - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.Mirror; - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; -public import dhtproto.client.NotifierTypes; - -/******************************************************************************* - - Mirror behaviour settings. May be optionally passed to the mirror() method - of the DHT client, to modify the default behaviour. - - The Mirror request provides the possibility of two types of refresh which - can be combined in various ways: - 1. Initial refresh: when the request starts, all records in the channel - are sent to the client via the Mirror request. - 2. Periodic refreshes: every N seconds (as specified by the user), all - records in the channel are sent to the client via the Mirror request. - -*******************************************************************************/ - -public struct Settings -{ - /// If true, the request, upon starting, will immediately send all records - /// in the channel to the client. - bool initial_refresh = true; - - /// If non-zero, the request will repeatedly send all records in the channel - /// to the client after every specified period. If zero, no periodic - /// refreshes will occur. - uint periodic_refresh_s = 360; -} - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - /// Name of the channel to mirror. - mstring channel; - - /// Settings for the behaviour of the request. - Settings settings; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - -*******************************************************************************/ - -private union NotificationUnion -{ - /// The request is now in a state where it can be suspended / resumed / - /// stopped via the controller. (This means that all known nodes have either - /// started handling the request or are not currently connected.) Note that - /// updates may be received from some nodes before this notification occurs. - RequestInfo started; - - /// A record's value has been updated. - RequestRecordInfo updated; - - /// A record's value has been resent to the client during a refresh cycle. - RequestRecordInfo refreshed; - - /// A record has been removed. - RequestKeyInfo deleted; - - /// The connection to a node disconnected; the request will automatically - /// continue after reconnection. - RequestNodeExceptionInfo node_disconnected; - - /// A node returned a non-OK status code. The request cannot be handled by - /// this node. - RequestNodeInfo node_error; - - /// The request was tried on a node and failed because it is unsupported. - /// The request cannot be handled by this node. - RequestNodeUnsupportedInfo unsupported; - - /// All known nodes have either suspended the request (as requested by the - /// user, via the controller) or are not currently connected. - RequestInfo suspended; - - /// All known nodes have either resumed the request (as requested by the - /// user, via the controller) or are not currently connected. - RequestInfo resumed; - - /// All known nodes have either stopped the request (as requested by the - /// user, via the controller) or are not currently connected. The request is - /// now finished. (A `finished` notification will not occur as well.) - RequestInfo stopped; - - /// The channel being mirrored has been removed. The request is now - /// finished for this node. - RequestNodeInfo channel_removed; - - /// The queue of updates to be sent to the client from the specified node - /// has overflowed. At least one update has been discarded. - RequestNodeInfo updates_lost; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notification delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; - -/******************************************************************************* - - Request controller, accessible via the client's `control()` method. - - Note that only one control change message can be "in-flight" to the nodes at - a time. If the controller is used when a control change message is already - in-flight, the method will return false. The notifier is called when a - requested control change is carried through. - -*******************************************************************************/ - -public interface IController -{ - /*************************************************************************** - - Tells the nodes to stop sending data to this request. - - Returns: - false if the controller cannot be used because a control change is - already in progress - - ***************************************************************************/ - - bool suspend ( ); - - /*************************************************************************** - - Tells the nodes to resume sending data to this request. - - Returns: - false if the controller cannot be used because a control change is - already in progress - - ***************************************************************************/ - - bool resume ( ); - - /*************************************************************************** - - Tells the nodes to cleanly end the request. - - Returns: - false if the controller cannot be used because a control change is - already in progress - - ***************************************************************************/ - - bool stop ( ); -} diff --git a/src/dhtproto/client/request/Put.d b/src/dhtproto/client/request/Put.d deleted file mode 100644 index cc7e0e5c..00000000 --- a/src/dhtproto/client/request/Put.d +++ /dev/null @@ -1,125 +0,0 @@ -/******************************************************************************* - - Client DHT Put request definitions / handler. - - The Put request attempts to write one record to the specified DHT channel. - This works as follows: - 1. The client selects a connected DHT node from its registry, based on - the key of the record to put. - 2. A request is sent to the selected node, asking for the specified - record to be added to the specified channel. - 3. The request ends when either the record is pushed to the node or the - node could not handle the request due to an error. - - Note that this request enforces a size limit on record values (see - MaxRecordSize constant, below). - - During a data redistribution, more than one node may be responsible for a - given key. In this case, the record is written to the node which was most - recently reported as being responsible for the key. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.Put; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; -import swarm.util.RecordBatcher; - -/// Maximum allowed size of record values. The maximum record value size depends -/// on the size of the batch used by GetAll requests. It must not be possible to -/// store a record in the DHT that cannot be returned by a GetAll request. -// TODO: ideally, the GetAll batch size should be defined in terms of the -// maximum record size, not the other way around. However, support for easily -// calculating the required size of a record batch based on the size of a record -// that can fit in it is lacking. -// See https://github.com/sociomantic-tsunami/swarm/issues/142 -public const MaxRecordSize = - RecordBatcher.DefaultMaxBatchSize - hash_t.sizeof - (size_t.sizeof * 2); - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - mstring channel; - hash_t key; - void[] value; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - * wrong_node - * value_too_big - -*******************************************************************************/ - -private union NotificationUnion -{ - /// The request succeeded. - RequestInfo success; - - /// The request failed due to a connection error. - RequestNodeExceptionInfo node_disconnected; - - /// The request failed due to an internal node error. - RequestNodeInfo node_error; - - /// The request failed because it is unsupported. - RequestNodeUnsupportedInfo unsupported; - - /// The length of the provided record value exceeds the MaxRecordSize - /// constant (see above). - RequestInfo value_too_big; - - /// The DHT node to which the request was sent is not responsible for the - /// record's key. This is a sanity check performed within the node in order - /// to avoid data inconsistency. - RequestNodeInfo wrong_node; - - /// No DHT node is known to cover the hash of the request. Note that this - /// may be because the client has just been started and has not received - /// hash range information from the DHT yet. - RequestInfo no_node; - - /// Internal error in client. - RequestInfo failure; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notifcation delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; diff --git a/src/dhtproto/client/request/Remove.d b/src/dhtproto/client/request/Remove.d deleted file mode 100644 index d8d7de77..00000000 --- a/src/dhtproto/client/request/Remove.d +++ /dev/null @@ -1,109 +0,0 @@ -/******************************************************************************* - - Client DHT Remove request definitions / handler. - - The Remove request attempts to remove one record from the specified DHT - channel. This works as follows: - 1. The client selects a connected DHT node from its registry, based on - the key of the record to be read. - 2. A request is sent to the selected node, asking for the specified - record in the specified channel to be removed. - 3. The request ends when either the record is removed from the node, is - found to not exist, or the node could not handle the request due to - an error. - - During a data redistribution, more than one node may be responsible for a - given key. In this case, the node that was least recently reported as being - responsible for the key is queried first, followed by others (in order) - until the record is either removed from or does not exist on all nodes, or - an error occurs. The reason for removing from the *least* recently - responsible nodes first is to avoid getting into inconsistent states if an - error occurs while removing. (If an error occurred when removing the record - from the most recently responsible node first, subsequent read requests - would fetch the removed record from older nodes, and the old value could be - forwarded from an older node, undoing the removal.) - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.Remove; - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - mstring channel; - hash_t key; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - * wrong_node - -*******************************************************************************/ - -private union NotificationUnion -{ - /// The request succeeded; no record existed with the specified key. - RequestInfo no_record; - - /// The request succeeded and the record was removed. - RequestInfo removed; - - /// The request failed due to a connection error. - RequestNodeExceptionInfo node_disconnected; - - /// The request failed due to an internal node error. - RequestNodeInfo node_error; - - /// The request failed because it is unsupported. - RequestNodeUnsupportedInfo unsupported; - - /// The DHT node to which the request was sent is not responsible for the - /// record's key. This is a sanity check performed within the node in order - /// to avoid data inconsistency. - RequestNodeInfo wrong_node; - - /// No DHT node is known to cover the hash of the request. Note that this - /// may be because the client has just been started and has not received - /// hash range information from the DHT yet. - RequestInfo no_node; - - /// Internal error in client. - RequestInfo failure; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notifcation delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; diff --git a/src/dhtproto/client/request/RemoveChannel.d b/src/dhtproto/client/request/RemoveChannel.d deleted file mode 100644 index fe22191c..00000000 --- a/src/dhtproto/client/request/RemoveChannel.d +++ /dev/null @@ -1,84 +0,0 @@ -/******************************************************************************* - - Client DHT RemoveChannel request definitions / handler. - - The RemoveChannel request instructs all DHT nodes to remove the named - channel. As removing a channel is a major operation, the node only allows - clients named "admin" (naturally, with the appropriate authentication) to - perform the request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.RemoveChannel; - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - mstring channel; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - * not_permitted - -*******************************************************************************/ - -private union NotificationUnion -{ - /// All known nodes have either handled the request or are not currently - /// connected. The request is now finished. - RequestInfo finished; - - /// One or more nodes refused to handle the request because the client does - /// not have admin permissions. Get permission before retrying. - RequestInfo not_permitted; - - /// The connection to a node has disconnected; the request will - /// automatically continue on this node after reconnection. - RequestNodeExceptionInfo node_disconnected; - - /// A node returned a non-OK status code. The request cannot be handled by - /// this node. - RequestNodeInfo node_error; - - /// The request was tried on a node and failed because it is unsupported. - /// The request cannot be handled by this node. - RequestNodeUnsupportedInfo unsupported; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notification delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Args ) Notifier; diff --git a/src/dhtproto/client/request/Update.d b/src/dhtproto/client/request/Update.d deleted file mode 100644 index f444471a..00000000 --- a/src/dhtproto/client/request/Update.d +++ /dev/null @@ -1,135 +0,0 @@ -/******************************************************************************* - - Client DHT Update request definitions / handler. - - The Update request attempts to read one record from the specified DHT - channel and replace its value in the DHT with a modified version, specified - by the user. This works as follows: - 1. The client selects a connected DHT node from its registry, based on - the key of the record to be updated. - 2. A request is sent to the selected node, asking for the specified - record in the specified channel to be returned. - 3. If the record exists, its value is passed to the user, who may - provide a modified version of the value or decide to leave the value - as it is. - 4. The modified version of the value is then sent back to the DHT. One - of the following then happens: - 4a. The record has been modified by another request, in the - meantime. The Update request is rejected and has no effect. (The - user may retry the request.) - 4b. The record has either been removed of has not been modified, in - the meantime. The new value is written to the DHT. - 5. The request ends when either the record does not exist, is - successfully updated in the node, or the node could not handle the - request due to an error. - - During a data redistribution, more than one node may be responsible for a - given key. In this case, the following logic is used: - * Reading: the node that was most recently reported as being responsible - for the key is queried first, followed by others (in order) until the - record is located, an error occurs, or no node has the record. - * Updating: the modified record is written to the node that was most - recently reported as being responsible for the key. If this is - different to the node that the record was read from, the source node - is instructed to remove the record. - - Copyright: - Copyright (c) 2018 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.Update; - -import ocean.transition; -import ocean.core.SmartUnion; -public import swarm.neo.client.NotifierTypes; -public import dhtproto.client.NotifierTypes; - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -public struct Args -{ - mstring channel; - hash_t key; -} - -/******************************************************************************* - - Union of possible notifications. - - The following notifications are considered fatal (i.e. the request will - almost certainly get the same error if retried): - * node_error - * unsupported - * wrong_node - -*******************************************************************************/ - -private union NotificationUnion -{ - // Notifications that indicate the end of the request: - - /// The request succeeded (either the value was updated or the original - /// value was left, as specified by the user). - RequestInfo succeeded; - - /// The specified record does not exist in the DHT so cannot be updated. - /// You probably want to Put a new record. - RequestInfo no_record; - - /// The specified record has been updated by another client. Try again. - RequestInfo conflict; - - /// No DHT node is known to cover the hash of the request. Note that this - /// may be because the client has just been started and has not received - /// hash range information from the DHT yet. - RequestInfo no_node; - - /// An I/O, node, or internal client error occurred while handling the - /// request. - RequestInfo error; - - // Notifications about communication with an individual DHT node: - - /// The record was retrieved from the DHT. The updated value to be sent back - /// should be copied into `received.value_buffer`. (If the buffer is left - /// empty, the DHT will be told to leave the record as it is.) - RequestDataUpdateInfo received; - - /// The request failed due to a connection error. - RequestNodeExceptionInfo node_disconnected; - - /// The request failed due to an internal node error. - RequestNodeInfo node_error; - - /// The request failed because it is unsupported. - RequestNodeUnsupportedInfo unsupported; - - /// The DHT node to which the request was sent is not responsible for the - /// record's key. This is a sanity check performed within the node in order - /// to avoid data inconsistency. - RequestNodeInfo wrong_node; -} - -/******************************************************************************* - - Notification smart union. - -*******************************************************************************/ - -public alias SmartUnion!(NotificationUnion) Notification; - -/******************************************************************************* - - Type of notifcation delegate. - -*******************************************************************************/ - -public alias void delegate ( Notification, Const!(Args) ) Notifier; diff --git a/src/dhtproto/client/request/internal/Exists.d b/src/dhtproto/client/request/internal/Exists.d deleted file mode 100644 index e51e263c..00000000 --- a/src/dhtproto/client/request/internal/Exists.d +++ /dev/null @@ -1,300 +0,0 @@ -/******************************************************************************* - - Client DHT Exists v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.Exists; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import ocean.transition; -import ocean.core.VersionCheck; -import ocean.util.log.Logger; - -/******************************************************************************* - - Module logger - -*******************************************************************************/ - -static private Logger log; -static this ( ) -{ - log = Log.lookup("dhtproto.client.request.internal.Exists"); -} - -/******************************************************************************* - - Exists request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct Exists -{ - import dhtproto.common.Exists; - import dhtproto.client.request.Exists; - import dhtproto.common.RequestCodes; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import swarm.neo.request.Command; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Enum indicating the ways in which the request may end. - public enum Result - { - Failure, // Default value; unknown error (presumably in client) - NoNode, // No node responsible for key - Error, // Node or I/O error - NoRecord, // Record not found - RecordExists // Record exists - } - - /// The way in which the request ended. Used by the finished notifier to - /// decide what kind of notification (if any) to send to the user. - Result result; - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.SingleNode, RequestCode.Exists, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - FIXME: Note that the logic for retrying the request on other nodes which - previously covered the hash has not been properly tested. This will - require a full neo implementation of the Redistribute request. See - https://github.com/sociomantic-tsunami/dhtnode/issues/21 - - Params: - use_node = delegate to get an EventDispatcher for the node with the - specified address - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( UseNodeDg use_node, void[] context_blob ) - { - auto context = Exists.getContext(context_blob); - context.shared_working.result = SharedWorking.Result.Failure; - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - // Try getting the record from the nodes responsible for the hash, - // querying them in newest -> oldest order - bool query_node_called; - shared_resources.node_hash_ranges.getFromNode( - context.user_params.args.key, - acquired_resources.getNodeHashRangeBuffer(), use_node, - ( RequestOnConn.EventDispatcher conn ) - { - query_node_called = true; - return queryNode(conn, context); - } - ); - - if ( !query_node_called ) - // No node is responsible for the key. - context.shared_working.result = SharedWorking.Result.NoNode; - } - - /*************************************************************************** - - Asks the specified node if it has the record. - - Params: - conn = event dispatcher for the connection to send the record to - context = deserialized request context, including record/value - - Returns: - true to try another node, false if finished (the record was fetched - or an error occurred). All error cases abort the request (return - false), as it is not possible to know if the node where the error - occurred has the record or not. - - ***************************************************************************/ - - private static bool queryNode ( RequestOnConn.EventDispatcher conn, - Exists.Context* context ) - { - try - { - // Send request info to node - conn.send( - ( conn.Payload payload ) - { - payload.add(Exists.cmd.code); - payload.add(Exists.cmd.ver); - payload.addArray(context.user_params.args.channel); - payload.add(context.user_params.args.key); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Receive supported code from node - auto supported = conn.receiveValue!(SupportedStatus)(); - if ( !Exists.handleSupportedCodes(supported, context, - conn.remote_address) ) - { - // Request not supported; abort further handling. - context.shared_working.result = SharedWorking.Result.Error; - return false; - } - else - { - // Request supported; read result code from node. - auto result = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( result ) - { - case RecordExists: - context.shared_working.result = - SharedWorking.Result.RecordExists; - return false; - - case NoRecord: - context.shared_working.result = - SharedWorking.Result.NoRecord; - return true; - - case WrongNode: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node is not reponsible for the key. Notify the user. - Notification n; - n.wrong_node = - RequestNodeInfo(context.request_id,conn.remote_address); - Exists.notify(context.user_params, n); - return false; - - case Error: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node returned an error code. Notify the user. - Notification n; - n.node_error = RequestNodeInfo(context.request_id, - conn.remote_address); - Exists.notify(context.user_params, n); - return false; - - default: - log.warn("Received unknown message code {} from node " - ~ "in response to Exists request. Treating as " - ~ "Error.", result); - goto case Error; - } - } - - assert(false); - } - catch ( IOError e ) - { - context.shared_working.result = SharedWorking.Result.Error; - - // A connection error occurred. Notify the user. - Notification n; - n.node_disconnected = RequestNodeExceptionInfo(context.request_id, - conn.remote_address, e); - Exists.notify(context.user_params, n); - - return false; - } - - assert(false); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = Exists.getContext(context_blob); - - Notification n; - - with ( SharedWorking.Result ) switch ( context.shared_working.result ) - { - case RecordExists: - n.exists = RequestInfo(context.request_id); - break; - case NoRecord: - n.no_record = RequestInfo(context.request_id); - break; - case NoNode: - n.no_node = RequestInfo(context.request_id); - break; - case Failure: - n.failure = RequestInfo(context.request_id); - break; - case Error: - // Error notification was already handled in queryNode(), - // where we have access to the node's address &/ exception. - return; - default: - assert(false); - } - - Exists.notify(context.user_params, n); - } -} diff --git a/src/dhtproto/client/request/internal/Get.d b/src/dhtproto/client/request/internal/Get.d deleted file mode 100644 index af5275c4..00000000 --- a/src/dhtproto/client/request/internal/Get.d +++ /dev/null @@ -1,341 +0,0 @@ -/******************************************************************************* - - Client DHT Get v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.Get; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import ocean.transition; -import ocean.core.VersionCheck; -import ocean.util.log.Logger; - -/******************************************************************************* - - Module logger - -*******************************************************************************/ - -static private Logger log; -static this ( ) -{ - log = Log.lookup("dhtproto.client.request.internal.Get"); -} - -/******************************************************************************* - - Get request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct Get -{ - import dhtproto.common.Get; - import dhtproto.client.request.Get; - import dhtproto.common.RequestCodes; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import swarm.neo.request.Command; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Enum indicating the ways in which the request may end. - public enum Result - { - Failure, // Default value; unknown error (presumably in client) - Timeout, // Request timed out in client before completion - NoNode, // No node responsible for key - Error, // Node or I/O error - NoRecord, // Record not found - Got // Got record - } - - /// The way in which the request ended. Used by the finished notifier to - /// decide what kind of notification (if any) to send to the user. - Result result; - } - - /*************************************************************************** - - Data which each request-on-conn needs while it is progress. An instance - of this struct is stored per connection on which the request runs and is - passed to the request handler. - - ***************************************************************************/ - - private static struct Working - { - // Dummy (not required by this request) - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.SingleNode, RequestCode.Get, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - FIXME: Note that the logic for retrying the request on other nodes which - previously covered the hash has not been properly tested. This will - require a full neo implementation of the Redistribute request. See - https://github.com/sociomantic-tsunami/dhtnode/issues/21 - - Params: - use_node = delegate to get an EventDispatcher for the node with the - specified address - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( UseNodeDg use_node, void[] context_blob ) - { - auto context = Get.getContext(context_blob); - context.shared_working.result = SharedWorking.Result.Failure; - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - // Try getting the record from the nodes responsible for the hash, - // querying them in newest -> oldest order - bool get_called; - shared_resources.node_hash_ranges.getFromNode( - context.user_params.args.key, - acquired_resources.getNodeHashRangeBuffer(), use_node, - ( RequestOnConn.EventDispatcher conn ) - { - get_called = true; - return getFromNode(conn, context); - } - ); - - if ( !get_called ) - { - // Bail out if no nodes cover the record's hash - context.shared_working.result = SharedWorking.Result.NoNode; - return; - } - } - - /*************************************************************************** - - Tries to gets the record from the specified node. - - Params: - conn = event dispatcher for the connection to send the record to - context = deserialized request context, including record/value - - Returns: - true to try another node, false if finished (the record was fetched - or an error occurred). All error cases abort the request (return - false), as it is not possible to know if the node where the error - occurred has the record or not. - - ***************************************************************************/ - - private static bool getFromNode ( RequestOnConn.EventDispatcher conn, - Get.Context* context ) - { - try - { - // Send request info to node - conn.send( - ( conn.Payload payload ) - { - payload.add(Get.cmd.code); - payload.add(Get.cmd.ver); - payload.addArray(context.user_params.args.channel); - payload.add(context.user_params.args.key); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Receive supported code from node - auto supported = conn.receiveValue!(SupportedStatus)(); - if ( !Get.handleSupportedCodes(supported, context, - conn.remote_address) ) - { - // Request not supported; abort further handling. - context.shared_working.result = SharedWorking.Result.Error; - return false; - } - else - { - // Request supported; read result code from node. - auto result = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( result ) - { - case Got: - context.shared_working.result = - SharedWorking.Result.Got; - - // Receive record value from node. - conn.receive( - ( in void[] const_payload ) - { - Const!(void)[] payload = const_payload; - auto value = - conn.message_parser.getArray!(void)(payload); - - Notification n; - n.received = RequestDataInfo(context.request_id, - value); - Get.notify(context.user_params, n); - } - ); - return false; - - case NoRecord: - context.shared_working.result = - SharedWorking.Result.NoRecord; - return true; - - case WrongNode: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node is not reponsible for the key. Notify the user. - Notification n; - n.wrong_node = RequestNodeInfo(context.request_id,conn.remote_address); - Get.notify(context.user_params, n); - return false; - - case Error: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node returned an error code. Notify the user. - Notification n; - n.node_error = RequestNodeInfo(context.request_id, - conn.remote_address); - Get.notify(context.user_params, n); - return false; - - default: - log.warn("Received unknown message code {} from node " - ~ "in response to Get request. Treating as " - ~ "Error.", result); - goto case Error; - } - } - - assert(false); - } - catch ( RequestOnConn.AbortException e ) - { - context.shared_working.result = SharedWorking.Result.Timeout; - throw e; - } - catch ( IOError e ) - { - context.shared_working.result = SharedWorking.Result.Error; - - // A connection error occurred. Notify the user. - auto info = RequestNodeExceptionInfo(context.request_id, - conn.remote_address, e); - - Notification n; - n.node_disconnected = info; - Get.notify(context.user_params, n); - - return false; - } - - assert(false); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = Get.getContext(context_blob); - - Notification n; - - with ( SharedWorking.Result ) switch ( context.shared_working.result ) - { - case NoRecord: - n.no_record = RequestInfo(context.request_id); - break; - case Timeout: - n.timed_out = RequestInfo(context.request_id); - break; - case NoNode: - n.no_node = RequestInfo(context.request_id); - break; - case Failure: - n.failure = RequestInfo(context.request_id); - break; - case Got: - // Got notification was already handled in getFromNode(), where - // the value received from the node is available. - case Error: - // Error notification was already handled in getFromNode(), - // where we have access to the node's address &/ exception. - return; - default: - assert(false); - } - - Get.notify(context.user_params, n); - } -} diff --git a/src/dhtproto/client/request/internal/GetAll.d b/src/dhtproto/client/request/internal/GetAll.d deleted file mode 100644 index 77e666b1..00000000 --- a/src/dhtproto/client/request/internal/GetAll.d +++ /dev/null @@ -1,558 +0,0 @@ -/******************************************************************************* - - Client DHT GetAll v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.GetAll; - -import ocean.transition; -import ocean.util.log.Logger; -import ocean.core.Verify; - -/******************************************************************************* - - GetAll request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct GetAll -{ - import dhtproto.common.GetAll; - import dhtproto.client.request.GetAll; - import dhtproto.common.RequestCodes; - import dhtproto.client.NotifierTypes; - import swarm.util.RecordBatcher; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.client.mixins.SuspendableRequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Request controller, accessible to the user via the client's `control()` - method. - - ***************************************************************************/ - - mixin SuspendableController!(GetAll, IController, MessageType); - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Shared working data required for core all-nodes request behaviour. - AllNodesRequestSharedWorkingData all_nodes; - - /// Shared working data required for core suspendable behaviour. - SuspendableRequestSharedWorkingData suspendable_control; - } - - /*************************************************************************** - - Data which each request-on-conn needs while it is progress. An instance - of this struct is stored per connection on which the request runs and is - passed to the request handler. - - ***************************************************************************/ - - private static struct Working - { - // Dummy struct. - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.AllNodes, RequestCode.GetAll, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - Params: - conn = request-on-conn event dispatcher - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( RequestOnConn.EventDispatcherAllNodes conn, - void[] context_blob ) - { - auto context = GetAll.getContext(context_blob); - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - scope handler = new GetAllHandler(conn, context, acquired_resources); - handler.run(); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = GetAll.getContext(context_blob); - - if ( !context.shared_working.suspendable_control.stopped_notification_done ) - { - Notification n; - n.finished = RequestInfo(context.request_id); - GetAll.notify(context.user_params, n); - } - } -} - -/******************************************************************************* - - Client GetAll v0 request handler. - -*******************************************************************************/ - -private scope class GetAllHandler -{ - import swarm.neo.client.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.client.mixins.SuspendableRequestCore; - import swarm.neo.request.Command; - import swarm.neo.request.RequestEventDispatcher; - import swarm.neo.util.MessageFiber; - import swarm.util.RecordBatcher; - - import dhtproto.common.GetAll; - import dhtproto.client.request.GetAll; - import dhtproto.client.internal.SharedResources; - - /// Request-on-conn event dispatcher. - private RequestOnConn.EventDispatcherAllNodes conn; - - /// Request context. - private GetAll.Context* context; - - /// Request resource acquirer. - private SharedResources.RequestResources resources; - - /// Request event dispatcher. - private RequestEventDispatcher request_event_dispatcher; - - /// Reader fiber instance. - private Reader reader; - - /// Controller fiber instance. - private Controller controller; - - /// Record batch received from the node. - private RecordBatch batch; - - /// Flag indicating that the request has successfully received some data. - private bool received_a_batch; - - /// Flag indicating that the request should continue where it left off, - /// after a disconnection. - private bool continuing; - - /// Key of last record received. - private hash_t last_key; - - /*************************************************************************** - - Constructor. - - Params: - conn = request-on-conn event dispatcher to communicate with node - context = deserialised request context - resources = request resource acquirer - - ***************************************************************************/ - - public this ( RequestOnConn.EventDispatcherAllNodes conn, - GetAll.Context* context, SharedResources.RequestResources resources ) - { - this.conn = conn; - this.context = context; - this.resources = resources; - - this.batch = resources.getRecordBatch(); - } - - /*************************************************************************** - - Main request handling entry point. - - ***************************************************************************/ - - public void run ( ) - { - auto request = createSuspendableRequest!(GetAll)(this.conn, this.context, - &this.connect, &this.disconnected, &this.fillPayload, - &this.handle); - request.run(); - } - - /*************************************************************************** - - Connect policy, called from AllNodesRequest template to ensure the - connection to the node is up. - - Returns: - true to continue handling the request; false to abort - - ***************************************************************************/ - - private bool connect ( ) - { - auto r = suspendableRequestConnector(this.conn, - &this.context.shared_working.suspendable_control); - return r; - } - - /*************************************************************************** - - Disconnected policy, called from AllNodesRequest template when an I/O - error occurs on the connection. - - Params: - e = exception indicating error which occurred on the connection - - ***************************************************************************/ - - private void disconnected ( Exception e ) - { - // Notify the user of the disconnection. The user may use the - // controller, at this point, but as the request is not active - // on this connection, no special behaviour is needed. - GetAll.Notification notification; - notification.node_disconnected = - RequestNodeExceptionInfo(this.context.request_id, - this.conn.remote_address, e); - GetAll.notify(this.context.user_params, notification); - - // If at least one record was received before the connection error, - // set the `continuing` flag to inform the node (after the connection is - // re-established) to carry on from the last key received. - if ( this.received_a_batch ) - this.continuing = true; - } - - /*************************************************************************** - - FillPayload policy, called from SuspendableRequestInitialiser template - to add request-specific data to the initial message payload send to the - node to begin the request. - - Params: - payload = message payload to be filled - - ***************************************************************************/ - - private void fillPayload ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - this.context.shared_working.suspendable_control.fillPayload(payload); - payload.addArray(this.context.user_params.args.channel); - payload.add(this.continuing); - payload.add(this.last_key); // If not continuing, this field is ignored - payload.add(this.context.user_params.args.settings.keys_only); - payload.addArray(this.context.user_params.args.settings.value_filter); - } - - /*************************************************************************** - - Handler policy, called from AllNodesRequest template to run the - request's main handling logic. - - ***************************************************************************/ - - private void handle ( ) - { - // Handle initial started/error message from node. - auto msg = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( msg ) - { - case Started: - // Expected "request started" code. Continue handling request. - break; - - case Error: - // The node returned an error code. Notify the user and end the - // request. - GetAll.Notification n; - n.node_error = RequestNodeInfo( - this.context.request_id, conn.remote_address); - GetAll.notify(this.context.user_params, n); - return; - - default: - // Treat unknown/unexpected codes as node errors. - goto case Error; - } - - scope reader_ = new Reader; - scope controller_ = new Controller; - - this.request_event_dispatcher.initialise(&this.resources.getVoidBuffer); - - // Note: we store refs to the scope instances in class fields as a - // convenience to be able to access them from each other (e.g. the - // reader needs to access the controller and vice-versa). It's normally - // not safe to store refs to scope instances outside of the scope, so we - // need to be careful to only use them while they are in scope. - this.reader = reader_; - this.controller = controller_; - scope ( exit ) - { - this.reader = null; - this.controller = null; - } - - controller.fiber.start(); - reader.fiber.start(); - - // Handle initial 'started' notification (and potential state change - // requests in the notifier). - if ( this.context.shared_working.all_nodes.num_initialising == 0 ) - { - if ( this.context.shared_working.suspendable_control. - allInitialised!(GetAll)(this.context) ) - { - this.request_event_dispatcher.signal(this.conn, - SuspendableRequestSharedWorkingData.Signal.StateChangeRequested); - } - } - - this.request_event_dispatcher.eventLoop(this.conn); - - verify(controller.fiber.finished()); - verify(reader.fiber.finished()); - } - - /*************************************************************************** - - Fiber which handles: - 1. Receiving batches of records from the node and forwarding them to - the user. - 2. Receiving and ACKing messages from the node indicating that the - request has finished. - - ***************************************************************************/ - - private class Reader - { - /// Fiber. - private MessageFiber fiber; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - // Receive the incoming record stream. - bool finished; - do - { - auto msg = this.outer.request_event_dispatcher.receive( - this.fiber, - Message(MessageType.RecordBatch), - Message(MessageType.Finished)); - - with ( MessageType ) switch ( msg.type ) - { - case RecordBatch: - auto batch = this.outer.conn.message_parser. - getArray!(void)(msg.payload); - - this.receivedBatch(batch); - break; - - case Finished: - finished = true; - break; - - default: - assert(false); - } - } - while ( !finished ); - - // ACK Finished message - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.Ack); - } - ); - - // It's no longer valid to handle control messages. - this.outer.request_event_dispatcher.abort( - this.outer.controller.fiber); - } - - /*********************************************************************** - - Handles a received batch, decompressing the batch and sending the - contained records to the user. - - Params: - compressed_batch = compressed record batch received from the - node - - ***********************************************************************/ - - private void receivedBatch ( Const!(void)[] compressed_batch ) - { - this.outer.received_a_batch = true; - - auto suspendable_control = - &this.outer.context.shared_working.suspendable_control; - - this.outer.batch.decompress(cast(Const!(ubyte)[])compressed_batch); - - void notify ( GetAll.Notification notification ) - { - if ( suspendable_control.notifyAndCheckStateChange!(GetAll)( - this.outer.context, notification) ) - { - // The user used the controller in the notifier callback - this.outer.request_event_dispatcher.signal( - this.outer.conn, - suspendable_control.Signal.StateChangeRequested); - } - } - - if ( this.outer.context.user_params.args.settings.keys_only ) - { - foreach ( key; this.outer.batch ) - { - hash_t hash_key = *(cast(hash_t*)key.ptr); - - GetAll.Notification n; - n.received_key = RequestKeyInfo( - this.outer.context.request_id, hash_key); - notify(n); - - this.outer.last_key = hash_key; - } - } - else - { - foreach ( key, value; this.outer.batch ) - { - hash_t hash_key = *(cast(hash_t*)key.ptr); - - GetAll.Notification n; - n.received = RequestRecordInfo( - this.outer.context.request_id, hash_key, value); - notify(n); - - this.outer.last_key = hash_key; - } - } - } - } - - /*************************************************************************** - - Fiber which handles: - 1. Waiting for state change requests from the user (via the - controller). - 2. Sending the appropriate message to the node and handling the - returned ACK messages. - - ***************************************************************************/ - - private class Controller - { - /// Fiber. - private MessageFiber fiber; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - SuspendableRequestControllerFiber!(GetAll, MessageType) controller; - controller.handle(this.outer.conn, this.outer.context, - &this.outer.request_event_dispatcher, this.fiber); - - // Kill the reader fiber; the request is finished. - this.outer.request_event_dispatcher.abort( - this.outer.reader.fiber); - } - } -} diff --git a/src/dhtproto/client/request/internal/GetChannels.d b/src/dhtproto/client/request/internal/GetChannels.d deleted file mode 100644 index e9cb4b1e..00000000 --- a/src/dhtproto/client/request/internal/GetChannels.d +++ /dev/null @@ -1,358 +0,0 @@ -/******************************************************************************* - - Client DHT GetChannels v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.GetChannels; - -import ocean.transition; -import ocean.util.log.Logger; - -/******************************************************************************* - - GetChannels request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct GetChannels -{ - import dhtproto.common.GetChannels; - import dhtproto.client.request.GetChannels; - import dhtproto.common.RequestCodes; - import dhtproto.client.NotifierTypes; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - import ocean.core.array.Mutation : copy; - import ocean.core.array.Search : contains; - import swarm.neo.util.VoidBufferAsArrayOf; - - /// Shared working data required for core all-nodes request behaviour. - AllNodesRequestSharedWorkingData all_nodes; - - /// Pointer to a list of the channel names that the client has already - /// been notified of. - VoidBufferAsArrayOf!(void[]) notified_channels; - - /*********************************************************************** - - Informs the caller whether the user should be notified about the - specified channel name, ensuring that the user is told about each - channel only once. - - Params: - channel = name of the channel - resources = request resource acquirer - - Returns: - true if the notifier should be called - - ***********************************************************************/ - - bool shouldNotifyChannel ( Const!(void)[] channel, - SharedResources.RequestResources resources ) - { - bool already_notified = false; - - // Acquire the array of channel name slices that is shared by all - // RoCs, if it's not already acquired. - if ( this.notified_channels == typeof(this.notified_channels).init ) - this.notified_channels = resources.getBufferList(); - else - already_notified = - (this.notified_channels.array().contains(channel) == true); - - // If this channel name has not been seen before, copy it into a new - // buffer and add a slice to that buffer to the array of channel - // name slices (this.notified_channels). - if ( !already_notified ) - { - auto buf = resources.getVoidBuffer(); - auto void_channel = cast(void[])channel; - (*buf).copy(void_channel); - this.notified_channels ~= cast(char[])(*buf); - } - - return !already_notified; - } - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.AllNodes, RequestCode.GetChannels, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - Params: - conn = request-on-conn event dispatcher - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( RequestOnConn.EventDispatcherAllNodes conn, - void[] context_blob ) - { - auto context = GetChannels.getContext(context_blob); - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - scope handler = new GetChannelsHandler(conn, context, - acquired_resources); - handler.run(); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = GetChannels.getContext(context_blob); - - Notification n; - n.finished = RequestInfo(context.request_id); - GetChannels.notify(context.user_params, n); - } -} - -/******************************************************************************* - - Client GetChannels v0 request handler. - -*******************************************************************************/ - -private scope class GetChannelsHandler -{ - import swarm.neo.client.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.request.Command; - - import dhtproto.common.GetChannels; - import dhtproto.client.request.GetChannels; - import dhtproto.client.internal.SharedResources; - - /// Request-on-conn event dispatcher. - private RequestOnConn.EventDispatcherAllNodes conn; - - /// Request context. - private GetChannels.Context* context; - - /// Request resource acquirer. - private SharedResources.RequestResources resources; - - /*************************************************************************** - - Constructor. - - Params: - conn = request-on-conn event dispatcher to communicate with node - context = deserialised request context - resources = request resource acquirer - - ***************************************************************************/ - - public this ( RequestOnConn.EventDispatcherAllNodes conn, - GetChannels.Context* context, SharedResources.RequestResources resources ) - { - this.conn = conn; - this.context = context; - this.resources = resources; - } - - /*************************************************************************** - - Main request handling entry point. - - ***************************************************************************/ - - public void run ( ) - { - auto initialiser = createAllNodesRequestInitialiser!(GetChannels)( - this.conn, this.context, &this.fillPayload); - auto request = createAllNodesRequest!(GetChannels)(this.conn, this.context, - &this.connect, &this.disconnected, initialiser, &this.handle); - request.run(); - } - - /*************************************************************************** - - Connect policy, called from AllNodesRequest template to ensure the - connection to the node is up. - - Returns: - true to continue handling the request; false to abort - - ***************************************************************************/ - - private bool connect ( ) - { - return allNodesRequestConnector(this.conn); - } - - /*************************************************************************** - - Disconnected policy, called from AllNodesRequest template when an I/O - error occurs on the connection. - - Params: - e = exception indicating error which occurred on the connection - - ***************************************************************************/ - - private void disconnected ( Exception e ) - { - // Notify the user of the disconnection. - GetChannels.Notification notification; - notification.node_disconnected = - RequestNodeExceptionInfo(this.context.request_id, - this.conn.remote_address, e); - GetChannels.notify(this.context.user_params, notification); - } - - /*************************************************************************** - - FillPayload policy, called from AllNodesRequestInitialiser template - to add request-specific data to the initial message payload send to the - node to begin the request. - - Params: - payload = message payload to be filled - - ***************************************************************************/ - - private void fillPayload ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - // Nothing more to add. (Request code and version already added.) - } - - /*************************************************************************** - - Handler policy, called from AllNodesRequest template to run the - request's main handling logic. - - ***************************************************************************/ - - private void handle ( ) - { - // Receive the incoming record stream. - bool finished; - do - { - this.conn.receive( - ( in void[] payload ) - { - finished = this.handleMessage(payload); - } - ); - } - while ( !finished ); - } - - /*************************************************************************** - - Handles a single message received from the node. - - Params: - payload = message payload - - Returns: - true if the message indicates that the request is finished - - ***************************************************************************/ - - private bool handleMessage ( Const!(void)[] payload ) - { - auto msg_type = - *this.conn.message_parser.getValue!(MessageType)(payload); - with ( MessageType ) switch ( msg_type ) - { - case ChannelName: - Const!(void)[] channel; - this.conn.message_parser.parseBody(payload, channel); - - if ( this.context.shared_working.shouldNotifyChannel(channel, - this.resources) ) - { - Notification notification; - notification.received = - RequestDataInfo(context.request_id, channel); - GetChannels.notify(this.context.user_params, notification); - } - break; - - case Error: - // The node returned an error code. Notify the user and - // end the request. - GetChannels.Notification n; - n.node_error = RequestNodeInfo( - this.context.request_id, this.conn.remote_address); - GetChannels.notify(this.context.user_params, n); - return true; - - case Finished: - return true; - - default: - assert(false); - } - - return false; - } -} diff --git a/src/dhtproto/client/request/internal/GetHashRange.d b/src/dhtproto/client/request/internal/GetHashRange.d deleted file mode 100644 index f9c58858..00000000 --- a/src/dhtproto/client/request/internal/GetHashRange.d +++ /dev/null @@ -1,347 +0,0 @@ -/******************************************************************************* - - Client DHT GetHashRange v0 request handler. - - Note that this request is unusual in that there is no public API. It is not - possible for the user of the DHT client to assign this request; it is only - ever assigned internally by the client itself (upon construction). - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.GetHashRange; - -import ocean.transition; -import ocean.util.log.Logger; -import swarm.neo.client.NotifierTypes; - -/******************************************************************************* - - Module logger - -*******************************************************************************/ - -static private Logger log; -static this ( ) -{ - log = Log.lookup("dhtproto.client.request.internal.GetHashRange"); -} - -/******************************************************************************* - - Request-specific arguments provided by the user and passed to the notifier. - -*******************************************************************************/ - -private struct Args -{ - // Dummy (not required by this request) -} - -/******************************************************************************* - - Notification smart union (never used, but required by RequestCore). - -*******************************************************************************/ - -private struct Notification -{ - /// The request was tried on a node and failed because it is unsupported; - /// it will be retried on any remaining nodes. (Required by RequestCore.) - RequestNodeUnsupportedInfo unsupported; -} - -/******************************************************************************* - - Type of notifcation delegate (never used, but required by RequestCore). - -*******************************************************************************/ - -private alias void delegate ( Notification, Args ) Notifier; - -/******************************************************************************* - - GetHashRange request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct GetHashRange -{ - import dhtproto.common.RequestCodes; - import swarm.neo.client.RequestOnConn; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.client.Connection; - - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored in the request's context (which is passed to the - request handler). - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Shared working data required for core all-nodes request behaviour. - AllNodesRequestSharedWorkingData all_nodes; - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.AllNodes, RequestCode.GetHashRange, 0, - Args, SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). Never exits: - this request must remain active during the client's whole lifetime. - - Params: - conn = connection event dispatcher - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( RequestOnConn.EventDispatcherAllNodes conn, - void[] context_blob ) - out - { - assert(false); - } - body - { - auto context = GetHashRange.getContext(context_blob); - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope handler = new GetHashRangeHandler(conn, context, - shared_resources); - handler.run(); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). Under - normal circumstances, this will never happen. - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - log.error("GetHashRange request finished. This should never happen :("); - } -} - -/******************************************************************************* - - GetHashRange handler class instantiated inside the main handler() function, - above. - -*******************************************************************************/ - -private scope class GetHashRangeHandler -{ - import swarm.neo.request.Command; - import swarm.neo.client.Connection; - import swarm.neo.client.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.request.RequestEventDispatcher; - import swarm.neo.AddrPort; - - import dhtproto.common.GetHashRange; - import dhtproto.client.internal.SharedResources; - - /// Request-on-conn event dispatcher. - private RequestOnConn.EventDispatcherAllNodes conn; - - /// Request context. - private GetHashRange.Context* context; - - /// Client shared resources. (Note that, unlike many requests, this is not - /// a resource acquirer. This request does not acquire any resources.) - private SharedResources resources; - - /*************************************************************************** - - Constructor. - - Params: - conn = request-on-conn event dispatcher to communicate with node - context = deserialised request context - resources = client shared resources - - ***************************************************************************/ - - public this ( RequestOnConn.EventDispatcherAllNodes conn, - GetHashRange.Context* context, SharedResources resources ) - { - this.conn = conn; - this.context = context; - this.resources = resources; - } - - /*************************************************************************** - - Main request handling entry point. - - ***************************************************************************/ - - public void run ( ) - { - auto initialiser = createAllNodesRequestInitialiser!(GetHashRange)( - this.conn, this.context, &this.fillPayload); - auto request = createAllNodesRequest!(GetHashRange)(this.conn, - this.context, &this.connect, &this.disconnected, initialiser, - &this.handle); - request.run(); - } - - /*************************************************************************** - - Connect policy, called from AllNodesRequest template to ensure the - connection to the node is up. - - Returns: - true to continue handling the request; false to abort - - ***************************************************************************/ - - private bool connect ( ) - { - return allNodesRequestConnector(this.conn); - } - - /*************************************************************************** - - Disconnected policy, called from AllNodesRequest template when an I/O - error occurs on the connection. - - Params: - e = exception indicating error which occurred on the connection - - ***************************************************************************/ - - private void disconnected ( Exception e ) - { - // Log errors, except when the connection was deliberately shut down. - if ( cast(Connection.ConnectionClosedException)e is null ) - log.error("I/O error: {} @ {}:{}", e.message, e.file, e.line); - } - - /*************************************************************************** - - FillPayload policy, called from AllNodesRequestInitialiser template - to add request-specific data to the initial message payload send to the - node to begin the request. - - Params: - payload = message payload to be filled - - ***************************************************************************/ - - private void fillPayload ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - // Nothing more to add. (Request code and version already added.) - } - - /*************************************************************************** - - Handler policy, called from AllNodesRequest template to run the - request's main handling logic. - - ***************************************************************************/ - - private void handle ( ) - { - // Receive and store node's current hash range - hash_t min, max; - this.conn.receive( - ( in void[] payload ) - { - Const!(void)[] payload_slice = payload; - min = *this.conn.message_parser.getValue!(hash_t)(payload_slice); - max = *this.conn.message_parser.getValue!(hash_t)(payload_slice); - } - ); - this.resources.node_hash_ranges.updateNodeHashRange( - this.conn.remote_address, min, max); - - while ( true ) - { - this.conn.receive( - ( in void[] payload ) - { - Const!(void)[] payload_slice = payload; - auto msg_type = *this.conn.message_parser. - getValue!(MessageType)(payload_slice); - - with ( MessageType ) switch ( msg_type ) - { - case NewNode: - typeof(AddrPort.naddress) addr; - typeof(AddrPort.nport) port; - hash_t min, max; - this.conn.message_parser.parseBody(payload_slice, - addr, port, min, max); - - this.resources.node_hash_ranges.updateNodeHashRange( - AddrPort(addr, port), min, max); - break; - - case ChangeHashRange: - hash_t new_min, new_max; - this.conn.message_parser.parseBody(payload_slice, - new_min, new_max); - - this.resources.node_hash_ranges.updateNodeHashRange( - this.conn.remote_address, new_min, new_max); - break; - - default: - log.error("Unknown message code {} received", msg_type); - throw this.conn.shutdownWithProtocolError( - "Message parsing error"); - } - } - ); - } - } -} diff --git a/src/dhtproto/client/request/internal/Mirror.d b/src/dhtproto/client/request/internal/Mirror.d deleted file mode 100644 index 94191091..00000000 --- a/src/dhtproto/client/request/internal/Mirror.d +++ /dev/null @@ -1,569 +0,0 @@ -/******************************************************************************* - - Client DHT Mirror v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.Mirror; - -import ocean.transition; -import ocean.util.log.Logger; -import ocean.core.Verify; - -/******************************************************************************* - - Mirror request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct Mirror -{ - import dhtproto.common.Mirror; - import dhtproto.client.request.Mirror; - import dhtproto.common.RequestCodes; - import dhtproto.client.NotifierTypes; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.client.mixins.SuspendableRequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Request controller, accessible to the user via the client's `control()` - method. - - ***************************************************************************/ - - mixin SuspendableController!(Mirror, IController, MessageType); - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Shared working data required for core all-nodes request behaviour. - AllNodesRequestSharedWorkingData all_nodes; - - /// Shared working data required for core suspendable behaviour. - SuspendableRequestSharedWorkingData suspendable_control; - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.AllNodes, RequestCode.Mirror, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - Params: - use_node = delegate to get an EventDispatcher for the node with the - specified address - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( RequestOnConn.EventDispatcherAllNodes conn, - void[] context_blob ) - { - auto context = Mirror.getContext(context_blob); - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - scope handler = new MirrorHandler(conn, context, acquired_resources); - handler.run(); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - // Do nothing. The only way a Mirror request can end is if the user - // stops it. - } -} - -/******************************************************************************* - - Client Mirror v0 request handler. - -*******************************************************************************/ - -private scope class MirrorHandler -{ - import swarm.neo.client.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.client.mixins.SuspendableRequestCore; - import swarm.neo.request.RequestEventDispatcher; - import swarm.neo.util.MessageFiber; - import swarm.neo.request.Command; - - import dhtproto.common.Mirror; - import dhtproto.client.request.Mirror; - import dhtproto.client.internal.SharedResources; - - /// Request-on-conn event dispatcher. - private RequestOnConn.EventDispatcherAllNodes conn; - - /// Request context. - private Mirror.Context* context; - - /// Request resource acquirer. - private SharedResources.RequestResources resources; - - /// Request event dispatcher. - private RequestEventDispatcher request_event_dispatcher; - - /// Reader fiber instance. - private Reader reader; - - /// Controller fiber instance. - private Controller controller; - - /// Batch decompression buffer. - private void[]* decompress_buffer; - - /*************************************************************************** - - Constructor. - - Params: - conn = request-on-conn event dispatcher to communicate with node - context = deserialised request context - resources = request resource acquirer - - ***************************************************************************/ - - public this ( RequestOnConn.EventDispatcherAllNodes conn, - Mirror.Context* context, SharedResources.RequestResources resources ) - { - this.conn = conn; - this.context = context; - this.resources = resources; - } - - /*************************************************************************** - - Main request handling entry point. - - ***************************************************************************/ - - public void run ( ) - { - auto request = createSuspendableRequest!(Mirror)(this.conn, this.context, - &this.connect, &this.disconnected, &this.fillPayload, &this.handle); - request.run(); - } - - /*************************************************************************** - - Connect policy, called from AllNodesRequest template to ensure the - connection to the node is up. - - Returns: - true to continue handling the request; false to abort - - ***************************************************************************/ - - private bool connect ( ) - { - return suspendableRequestConnector(this.conn, - &this.context.shared_working.suspendable_control); - } - - /*************************************************************************** - - Disconnected policy, called from AllNodesRequest template when an I/O - error occurs on the connection. - - Params: - e = exception indicating error which occurred on the connection - - ***************************************************************************/ - - private void disconnected ( Exception e ) - { - // Notify the user of the disconnection. The user may use the - // controller, at this point, but as the request is not active - // on this connection, no special behaviour is needed. - Mirror.Notification notification; - notification.node_disconnected = - RequestNodeExceptionInfo(this.context.request_id, - this.conn.remote_address, e); - Mirror.notify(this.context.user_params, notification); - } - - /*************************************************************************** - - FillPayload policy, called from SuspendableRequestInitialiser template - to add request-specific data to the initial message payload send to the - node to begin the request. - - ***************************************************************************/ - - private void fillPayload ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - this.context.shared_working.suspendable_control.fillPayload(payload); - payload.addArray(context.user_params.args.channel); - payload.add(context.user_params.args.settings.initial_refresh); - payload.add(context.user_params.args.settings.periodic_refresh_s); - } - - /*************************************************************************** - - Handler policy, called from AllNodesRequest template to run the - request's main handling logic. - - ***************************************************************************/ - - private void handle ( ) - { - // Handle initial started/error message from node. - auto msg = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( msg ) - { - case Started: - // Expected "request started" code. Continue handling request. - break; - - case Error: - // The node returned an error code. Notify the user and end the - // request. - Mirror.Notification n; - n.node_error = RequestNodeInfo( - this.context.request_id, conn.remote_address); - Mirror.notify(this.context.user_params, n); - return; - - default: - // Treat unknown/unexpected codes as node errors. - goto case Error; - } - - this.decompress_buffer = this.resources.getVoidBuffer(); - - scope reader_ = new Reader; - scope controller_ = new Controller; - - this.request_event_dispatcher.initialise(&this.resources.getVoidBuffer); - - // Note: we store refs to the scope instances in class fields as a - // convenience to be able to access them from each other (e.g. the - // reader needs to access the controller and vice-versa). It's normally - // not safe to store refs to scope instances outside of the scope, so we - // need to be careful to only use them while they are in scope. - this.reader = reader_; - this.controller = controller_; - scope ( exit ) - { - this.reader = null; - this.controller = null; - } - - controller.fiber.start(); - reader.fiber.start(); - - // Handle initial 'started' notification (and potential state change - // requests in the notifier). - if ( this.context.shared_working.all_nodes.num_initialising == 0 ) - { - if ( this.context.shared_working.suspendable_control. - allInitialised!(Mirror)(this.context) ) - { - this.request_event_dispatcher.signal(this.conn, - SuspendableRequestSharedWorkingData.Signal.StateChangeRequested); - } - } - - this.request_event_dispatcher.eventLoop(this.conn); - - verify(controller.fiber.finished()); - verify(reader.fiber.finished()); - } - - /*************************************************************************** - - Fiber which handles: - 1. Receiving update messages from the node and forwarding them to - the user. - 2. Receiving and ACKing messages from the node indicating that the - request has finished. - - ***************************************************************************/ - - private class Reader - { - import swarm.neo.util.Batch; - - /// Fiber. - private MessageFiber fiber; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - // Receive the incoming record stream. - bool finished; - do - { - auto msg = this.outer.request_event_dispatcher.receive( - this.fiber, - Message(MessageType.RecordChanged), - Message(MessageType.RecordRefreshBatch), - Message(MessageType.RecordDeleted), - Message(MessageType.ChannelRemoved), - Message(MessageType.UpdateOverflow)); - - with ( MessageType ) switch ( msg.type ) - { - case RecordChanged: - auto key = *this.outer.conn.message_parser. - getValue!(hash_t)(msg.payload); - auto value = this.outer.conn.message_parser. - getArray!(void)(msg.payload); - - this.recordChanged(key, value); - break; - - case RecordDeleted: - auto key = *this.outer.conn.message_parser. - getValue!(hash_t)(msg.payload); - - this.recordDeleted(key); - break; - - case RecordRefreshBatch: - auto batch = this.outer.conn.message_parser. - getArray!(void)(msg.payload); - - this.recordRefreshedBatch(batch); - break; - - case ChannelRemoved: - finished = true; - - Mirror.Notification n; - n.channel_removed = RequestNodeInfo( - this.outer.context.request_id, - this.outer.conn.remote_address); - Mirror.notify(this.outer.context.user_params, n); - break; - - case UpdateOverflow: - Mirror.Notification n; - n.updates_lost = RequestNodeInfo( - this.outer.context.request_id, - this.outer.conn.remote_address); - Mirror.notify(this.outer.context.user_params, n); - break; - - default: - assert(false); - } - } - while ( !finished ); - - // ACK End message - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.Ack); - } - ); - - // It's no longer valid to handle control messages. - this.outer.request_event_dispatcher.abort( - this.outer.controller.fiber); - } - - /*********************************************************************** - - Notifies the user that a record value has changed. - - Params: - key = key of record which was updated - value = new value of record - - ***********************************************************************/ - - private void recordChanged ( hash_t key, Const!(void)[] value ) - { - Mirror.Notification n; - n.updated = RequestRecordInfo( - this.outer.context.request_id, key, value); - this.notify(n); - } - - /*********************************************************************** - - Notifies the user of a batch of refreshed records. - - Params: - batch_data = batch of refreshed records - - ***********************************************************************/ - - private void recordRefreshedBatch ( Const!(void)[] batch_data ) - { - scope batch = new BatchReader!(hash_t, Const!(void)[])( - this.outer.resources.lzo, batch_data, - *this.outer.decompress_buffer); - foreach ( key, value; batch ) - { - Mirror.Notification n; - n.refreshed = RequestRecordInfo( - this.outer.context.request_id, key, value); - this.notify(n); - } - } - - /*********************************************************************** - - Notifies the user that a record has been removed. - - Params: - key = key of record which was removed - - ***********************************************************************/ - - private void recordDeleted ( hash_t key ) - { - Mirror.Notification n; - n.deleted = RequestKeyInfo(this.outer.context.request_id, key); - this.notify(n); - } - - /*********************************************************************** - - User notification helper which handles the case where the user used - the controller in the notifier callback. - - Params: - n = notification to send to user - - ***********************************************************************/ - - private void notify ( Mirror.Notification n ) - { - auto suspendable_control = - &this.outer.context.shared_working.suspendable_control; - - if ( suspendable_control.notifyAndCheckStateChange!(Mirror)( - this.outer.context, n) ) - { - // The user used the controller in the notifier callback - this.outer.request_event_dispatcher.signal(this.outer.conn, - suspendable_control.Signal.StateChangeRequested); - } - } - } - - /*************************************************************************** - - Fiber which handles: - 1. Waiting for state change requests from the user (via the - controller). - 2. Sending the appropriate message to the node and handling the - returned ACK messages. - - ***************************************************************************/ - - private class Controller - { - /// Fiber. - private MessageFiber fiber; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - SuspendableRequestControllerFiber!(Mirror, MessageType) controller; - controller.handle(this.outer.conn, this.outer.context, - &this.outer.request_event_dispatcher, this.fiber); - - // Kill the reader fiber; the request is finished. - this.outer.request_event_dispatcher.abort( - this.outer.reader.fiber); - } - } -} diff --git a/src/dhtproto/client/request/internal/Put.d b/src/dhtproto/client/request/internal/Put.d deleted file mode 100644 index 47de28ef..00000000 --- a/src/dhtproto/client/request/internal/Put.d +++ /dev/null @@ -1,282 +0,0 @@ -/******************************************************************************* - - Client DHT Put v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.Put; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import ocean.transition; -import ocean.core.VersionCheck; -import ocean.util.log.Logger; - -/******************************************************************************* - - Module logger - -*******************************************************************************/ - -static private Logger log; -static this ( ) -{ - log = Log.lookup("dhtproto.client.request.internal.Put"); -} - -/******************************************************************************* - - Put request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct Put -{ - import dhtproto.common.Put; - import dhtproto.client.request.Put; - import dhtproto.common.RequestCodes; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import swarm.neo.request.Command; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - public enum Result - { - Failure, - ValueTooBig, - NoNode, - Error, - Success - } - - Result result; - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.SingleNode, RequestCode.Put, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - Params: - use_node = delegate to get an EventDispatcher for the node with the - specified address - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( UseNodeDg use_node, void[] context_blob ) - { - auto context = Put.getContext(context_blob); - context.shared_working.result = SharedWorking.Result.Failure; - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - if ( context.user_params.args.value.length > MaxRecordSize ) - { - context.shared_working.result = SharedWorking.Result.ValueTooBig; - return; - } - - // Try putting the record to the newest node responsible for the hash. - bool put_called; - shared_resources.node_hash_ranges.putToNode( - context.user_params.args.key, - acquired_resources.getNodeHashRangeBuffer(), use_node, - ( RequestOnConn.EventDispatcher conn ) - { - put_called = true; - putToNode(conn, context); - } - ); - - if ( !put_called ) - context.shared_working.result = SharedWorking.Result.NoNode; - } - - /*************************************************************************** - - Puts the record passed by the user to the specified node. - - Params: - conn = event dispatcher for the connection to send the record to - context = deserialized request context, including record/value - - ***************************************************************************/ - - private static void putToNode ( RequestOnConn.EventDispatcher conn, - Put.Context* context ) - { - try - { - // Send request info to node - conn.send( - ( conn.Payload payload ) - { - payload.add(Put.cmd.code); - payload.add(Put.cmd.ver); - payload.addArray(context.user_params.args.channel); - payload.add(context.user_params.args.key); - payload.addArray(context.user_params.args.value); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Receive supported code from node - auto supported = conn.receiveValue!(SupportedStatus)(); - if ( !Put.handleSupportedCodes(supported, context, - conn.remote_address) ) - { - // Request not supported; abort further handling. - context.shared_working.result = SharedWorking.Result.Error; - } - else - { - // Request supported; read result code from node. - auto result = conn.receiveValue!(MessageType)(); - switch ( result ) - { - case MessageType.Put: - context.shared_working.result = - SharedWorking.Result.Success; - break; - - case MessageType.WrongNode: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node is not reponsible for the key. Notify the user. - Notification n; - n.wrong_node = RequestNodeInfo(context.request_id, - conn.remote_address); - Put.notify(context.user_params, n); - break; - - case MessageType.Error: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node returned an error code. Notify the user. - Notification n; - n.node_error = RequestNodeInfo(context.request_id, - conn.remote_address); - Put.notify(context.user_params, n); - break; - - default: - log.warn("Received unknown message code {} from node " - ~ "in response to Put request. Treating as " - ~ "Error.", result); - goto case MessageType.Error; - } - } - } - catch ( IOError e ) - { - context.shared_working.result = - SharedWorking.Result.Error; - - // A connection error occurred. Notify the user. - auto info = RequestNodeExceptionInfo(context.request_id, - conn.remote_address, e); - - Notification n; - n.node_disconnected = info; - Put.notify(context.user_params, n); - } - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = Put.getContext(context_blob); - - Notification n; - - with ( SharedWorking.Result ) switch ( context.shared_working.result ) - { - case Success: - n.success = RequestInfo(context.request_id); - break; - case ValueTooBig: - n.value_too_big = RequestInfo(context.request_id); - break; - case NoNode: - n.no_node = RequestInfo(context.request_id); - break; - case Failure: - n.failure = RequestInfo(context.request_id); - break; - case Error: - // Error notification was already handled in putToNode(), - // where we have access to the node's address &/ exception. - return; - default: - assert(false); - } - - Put.notify(context.user_params, n); - } -} diff --git a/src/dhtproto/client/request/internal/Remove.d b/src/dhtproto/client/request/internal/Remove.d deleted file mode 100644 index c632a1af..00000000 --- a/src/dhtproto/client/request/internal/Remove.d +++ /dev/null @@ -1,297 +0,0 @@ -/******************************************************************************* - - Client DHT Remove v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.Remove; - -import ocean.transition; -import ocean.core.VersionCheck; -import ocean.util.log.Logger; - -/******************************************************************************* - - Module logger - -*******************************************************************************/ - -static private Logger log; -static this ( ) -{ - log = Log.lookup("dhtproto.client.request.internal.Remove"); -} - -/******************************************************************************* - - Remove request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct Remove -{ - import dhtproto.common.Remove; - import dhtproto.client.request.Remove; - import dhtproto.common.RequestCodes; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import swarm.neo.request.Command; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Enum indicating the ways in which the request may end. - public enum Result - { - Failure, // Default value; unknown error (presumably in client) - NoNode, // No node responsible for key - Error, // Node or I/O error - NoRecord, // Record not present - Removed // Removed record - } - - /// The way in which the request ended. Used by the finished notifier to - /// decide what kind of notification (if any) to send to the user. - Result result; - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.SingleNode, RequestCode.Remove, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - FIXME: Note that the logic for retrying the request on other nodes which - previously covered the hash has not been properly tested. This will - require a full neo implementation of the Redistribute request. See - https://github.com/sociomantic-tsunami/dhtnode/issues/21 - - Params: - use_node = delegate to get an EventDispatcher for the node with the - specified address - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( UseNodeDg use_node, void[] context_blob ) - { - auto context = Remove.getContext(context_blob); - context.shared_working.result = SharedWorking.Result.Failure; - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - // Remove the record from all nodes responsible for the hash, querying - // them in oldest -> newest order - bool dg_called; - shared_resources.node_hash_ranges.removeFromNodes( - context.user_params.args.key, - acquired_resources.getNodeHashRangeBuffer(), use_node, - ( RequestOnConn.EventDispatcher conn ) - { - dg_called = true; - return removeFromNode(conn, context); - } - ); - - if ( !dg_called ) - // No node covers the record's hash - context.shared_working.result = SharedWorking.Result.NoNode; - } - - /*************************************************************************** - - Tries to remove the record from the specified node. - - Params: - conn = event dispatcher for the connection to send to - context = deserialized request context - - Returns: - true to continue to the next node, if removal succeeded (either the - record was removed or it didn't exist in the node); false to abort - the request, if an error occurred. Error cases abort the request - (return false), because it is not possible to know if the node where - the error occurred now has the record or not. - - ***************************************************************************/ - - private static bool removeFromNode ( RequestOnConn.EventDispatcher conn, - Remove.Context* context ) - { - try - { - // Send request info to node - conn.send( - ( conn.Payload payload ) - { - payload.add(Remove.cmd.code); - payload.add(Remove.cmd.ver); - payload.addArray(context.user_params.args.channel); - payload.add(context.user_params.args.key); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Receive supported code from node - auto supported = conn.receiveValue!(SupportedStatus)(); - if ( !Remove.handleSupportedCodes(supported, context, - conn.remote_address) ) - { - // Request not supported; abort further handling. - context.shared_working.result = SharedWorking.Result.Error; - return false; - } - else - { - // Request supported; read result code from node. - auto result = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( result ) - { - case Removed: - context.shared_working.result = - SharedWorking.Result.Removed; - return true; - - case NoRecord: - context.shared_working.result = - SharedWorking.Result.NoRecord; - return true; - - case WrongNode: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node is not reponsible for the key. Notify the user. - Notification n; - n.wrong_node = RequestNodeInfo( - context.request_id,conn.remote_address); - Remove.notify(context.user_params, n); - return false; - - case Error: - context.shared_working.result = - SharedWorking.Result.Error; - - // The node returned an error code. Notify the user. - Notification n; - n.node_error = RequestNodeInfo(context.request_id, - conn.remote_address); - Remove.notify(context.user_params, n); - return false; - - default: - log.warn("Received unknown message code {} from node " - ~ "in response to Remove request. Treating as " - ~ "Error.", result); - goto case Error; - } - } - - assert(false); - } - catch ( IOError e ) - { - context.shared_working.result = SharedWorking.Result.Error; - - // A connection error occurred. Notify the user. - auto info = RequestNodeExceptionInfo(context.request_id, - conn.remote_address, e); - - Notification n; - n.node_disconnected = info; - Remove.notify(context.user_params, n); - - return false; - } - - assert(false); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = Remove.getContext(context_blob); - - Notification n; - - with ( SharedWorking.Result ) switch ( context.shared_working.result ) - { - case Failure: - n.failure = RequestInfo(context.request_id); - break; - case NoRecord: - n.no_record = RequestInfo(context.request_id); - break; - case NoNode: - n.no_node = RequestInfo(context.request_id); - break; - case Removed: - n.removed = RequestInfo(context.request_id); - break; - case Error: - // Error notification was already handled in getFromNode(), - // where we have access to the node's address &/ exception. - return; - default: - assert(false); - } - - Remove.notify(context.user_params, n); - } -} diff --git a/src/dhtproto/client/request/internal/RemoveChannel.d b/src/dhtproto/client/request/internal/RemoveChannel.d deleted file mode 100644 index 4f4d6315..00000000 --- a/src/dhtproto/client/request/internal/RemoveChannel.d +++ /dev/null @@ -1,279 +0,0 @@ -/******************************************************************************* - - Client DHT RemoveChannel v0 request handler. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.RemoveChannel; - -import ocean.transition; -import ocean.util.log.Logger; - -/******************************************************************************* - - RemoveChannel request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct RemoveChannel -{ - import dhtproto.common.RemoveChannel; - import dhtproto.client.request.RemoveChannel; - import dhtproto.common.RequestCodes; - import dhtproto.client.NotifierTypes; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import dhtproto.client.internal.SharedResources; - - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Shared working data required for core all-nodes request behaviour. - AllNodesRequestSharedWorkingData all_nodes; - - /// Flag indicating that one or more request-on-conns prevented the - /// request from being handled due to the wrong client permissions. - bool not_permitted; - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.AllNodes, RequestCode.RemoveChannel, 0, Args, - SharedWorking, Notification); - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - Params: - conn = request-on-conn event dispatcher - context_blob = untyped chunk of data containing the serialized - context of the request which is to be handled - - ***************************************************************************/ - - public static void handler ( RequestOnConn.EventDispatcherAllNodes conn, - void[] context_blob ) - { - auto context = RemoveChannel.getContext(context_blob); - - auto shared_resources = SharedResources.fromObject( - context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - scope handler = new RemoveChannelHandler(conn, context, - acquired_resources); - handler.run(); - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = RemoveChannel.getContext(context_blob); - Notification n; - - // Different nodes may have returned different status codes. - if ( context.shared_working.not_permitted ) - n.not_permitted = RequestInfo(context.request_id); - else - n.finished = RequestInfo(context.request_id); - - RemoveChannel.notify(context.user_params, n); - } -} - -/******************************************************************************* - - Client RemoveChannel v0 request handler. - -*******************************************************************************/ - -private scope class RemoveChannelHandler -{ - import swarm.neo.client.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import swarm.neo.client.mixins.AllNodesRequestCore; - import swarm.neo.request.Command; - - import dhtproto.common.RemoveChannel; - import dhtproto.client.request.RemoveChannel; - import dhtproto.client.internal.SharedResources; - - /// Request-on-conn event dispatcher. - private RequestOnConn.EventDispatcherAllNodes conn; - - /// Request context. - private RemoveChannel.Context* context; - - /// Request resource acquirer. - private SharedResources.RequestResources resources; - - /*************************************************************************** - - Constructor. - - Params: - conn = request-on-conn event dispatcher to communicate with node - context = deserialised request context - resources = request resource acquirer - - ***************************************************************************/ - - public this ( RequestOnConn.EventDispatcherAllNodes conn, - RemoveChannel.Context* context, SharedResources.RequestResources resources ) - { - this.conn = conn; - this.context = context; - this.resources = resources; - } - - /*************************************************************************** - - Main request handling entry point. - - ***************************************************************************/ - - public void run ( ) - { - auto initialiser = createAllNodesRequestInitialiser!(RemoveChannel)( - this.conn, this.context, &this.fillPayload); - auto request = createAllNodesRequest!(RemoveChannel)(this.conn, this.context, - &this.connect, &this.disconnected, initialiser, &this.handle); - request.run(); - } - - /*************************************************************************** - - Connect policy, called from AllNodesRequest template to ensure the - connection to the node is up. - - Returns: - true to continue handling the request; false to abort - - ***************************************************************************/ - - private bool connect ( ) - { - return allNodesRequestConnector(this.conn); - } - - /*************************************************************************** - - Disconnected policy, called from AllNodesRequest template when an I/O - error occurs on the connection. - - Params: - e = exception indicating error which occurred on the connection - - ***************************************************************************/ - - private void disconnected ( Exception e ) - { - // Notify the user of the disconnection. - RemoveChannel.Notification notification; - notification.node_disconnected = - RequestNodeExceptionInfo(this.context.request_id, - this.conn.remote_address, e); - RemoveChannel.notify(this.context.user_params, notification); - } - - /*************************************************************************** - - FillPayload policy, called from AllNodesRequestInitialiser template - to add request-specific data to the initial message payload send to the - node to begin the request. - - Params: - payload = message payload to be filled - - ***************************************************************************/ - - private void fillPayload ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addArray(this.context.user_params.args.channel); - } - - /*************************************************************************** - - Handler policy, called from AllNodesRequest template to run the - request's main handling logic. - - ***************************************************************************/ - - private void handle ( ) - { - auto result = conn.receiveValue!(MessageType)(); - - with ( MessageType ) switch ( result ) - { - case ChannelRemoved: - // Request succeeded. - break; - - case NotPermitted: - // The client is not "admin" and may not remove channels. - this.context.shared_working.not_permitted = true; - break; - - case Error: - // The node returned an error code. - RemoveChannel.Notification n; - n.node_error = RequestNodeInfo( - this.context.request_id, conn.remote_address); - RemoveChannel.notify(this.context.user_params, n); - break; - - default: - // Treat unknown codes as internal errors. - goto case Error; - } - } -} diff --git a/src/dhtproto/client/request/internal/Update.d b/src/dhtproto/client/request/internal/Update.d deleted file mode 100644 index c1d91989..00000000 --- a/src/dhtproto/client/request/internal/Update.d +++ /dev/null @@ -1,808 +0,0 @@ -/******************************************************************************* - - Client DHT Update v0 request handler. - - Copyright: - Copyright (c) 2018 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.client.request.internal.Update; - -import ocean.transition; -import ocean.core.VersionCheck; -import ocean.util.log.Logger; -import ocean.core.Verify; - -/******************************************************************************* - - Module logger - -*******************************************************************************/ - -static private Logger log; -static this ( ) -{ - log = Log.lookup("dhtproto.client.request.internal.Update"); -} - -/******************************************************************************* - - Update request implementation. - - Note that request structs act simply as namespaces for the collection of - symbols required to implement a request. They are never instantiated and - have no fields or non-static functions. - - The client expects several things to be present in a request struct: - 1. The static constants request_type and request_code - 2. The UserSpecifiedParams struct, containing all user-specified request - setup (including a notifier) - 3. The Notifier delegate type - 4. Optionally, the Controller type (if the request can be controlled, - after it has begun) - 5. The handler() function - 6. The all_finished_notifier() function - - The RequestCore mixin provides items 1 and 2. - -*******************************************************************************/ - -public struct Update -{ - import dhtproto.common.Update; - import dhtproto.common.RequestCodes; - import dhtproto.client.request.Update; - import swarm.neo.AddrPort; - import swarm.neo.client.mixins.RequestCore; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - - /*************************************************************************** - - Data which the request needs while it is progress. An instance of this - struct is stored per connection on which the request runs and is passed - to the request handler. - - ***************************************************************************/ - - private static struct SharedWorking - { - /// Address of the second node to contact, in the case where the record - /// is written back to a different node than the one it was read from. - AddrPort second_node_addr; - - /// Flag indicating whether the communication with the second node - /// succeeded or not. - bool second_node_failed; - - /// Reference to the original request-on-conn that handles reading the - /// record to be updated. Stored so that it can be resumed from the - /// second request-on-conn (if required). - RequestOnConn.EventDispatcher first_request_on_conn; - - /// Hash of the record value fetched from the node. Sent back to the - /// node along with the updated value. Used to confirm that no other - /// client has modified the value in the meantime. - hash_t original_hash; - - /// Updated record value. - void[]* updated_value; - - /// Enum indicating the ways in which the request may end. - public enum Result - { - Error, // Default value; client / I/O / node error - Succeeded, // Request succeeded. - Conflict, // Another client updated the record. - NoRecord, // Succeeded, but record not in DHT. - NoNode // No node responsible for key - } - - /// The way in which the request ended. Used by the finished notifier to - /// decide what kind of notification (if any) to send to the user. - Result result; - } - - /*************************************************************************** - - Request core. Mixes in the types `NotificationInfo`, `Notifier`, - `Params`, `Context` plus the static constants `request_type` and - `request_code`. - - ***************************************************************************/ - - mixin RequestCore!(RequestType.MultiNode, RequestCode.Update, 0, Args, - SharedWorking, Notification); - - /// Fiber resume code. - const SecondROCFinished = 1; - - /*************************************************************************** - - Request handler. Called from RequestOnConn.runHandler(). - - FIXME: Note that the logic for retrying the request on other nodes which - previously covered the hash has not been properly tested. This will - require a full neo implementation of the Redistribute request. See - https://github.com/sociomantic-tsunami/dhtnode/issues/21 - - Params: - use_node = delegate to be called from the handler to get access to - an `EventDispatcher` instance to communicate with the specified - node - new_request_on_conn = delegate to be called from the handler to - cause the handler to be called again in a new `RequestOnConn - instance - context_blob = packed request context struct - - ***************************************************************************/ - - public static void handler ( UseNodeDg use_node, - NewRequestOnConnDg new_request_on_conn, void[] context_blob ) - { - auto context = Update.getContext(context_blob); - context.shared_working.result = SharedWorking.Result.Error; - - // The logic to perform here depends which RoC we're running in. - if ( context.shared_working.first_request_on_conn is null ) - { - auto handler = - FirstROCHandler(use_node, new_request_on_conn, context); - handler.handle(); - } - else - { - // Whatever happens when communicating with the second node, we must - // always resume the first request-on-conn so that the request can - // finish cleanly. - scope ( exit ) - { - context.shared_working.first_request_on_conn.resumeFiber( - Update.SecondROCFinished); - } - - auto handler = SecondROCHandler(use_node, context); - handler.handle(); - } - } - - /*************************************************************************** - - Request finished notifier. Called from Request.handlerFinished(). - - Params: - context_blob = untyped chunk of data containing the serialized - context of the request which is finishing - - ***************************************************************************/ - - public static void all_finished_notifier ( void[] context_blob ) - { - auto context = Update.getContext(context_blob); - - Notification n; - auto info = RequestInfo(context.request_id); - - with ( SharedWorking.Result ) switch ( context.shared_working.result ) - { - case Succeeded: - n.succeeded = info; - break; - case NoRecord: - n.no_record = info; - break; - case Conflict: - n.conflict = info; - break; - case NoNode: - n.no_node = info; - break; - case Error: - n.error = info; - return; - default: - assert(false); - } - - Update.notify(context.user_params, n); - } -} - -/// Handler for the initial request-on-conn (which reads the record value from -/// the DHT). -private struct FirstROCHandler -{ - import ocean.io.digest.Fnv1; - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - import swarm.neo.AddrPort; - import swarm.neo.client.NotifierTypes; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import swarm.neo.request.Command; - import dhtproto.client.NotifierTypes; - import dhtproto.client.internal.NodeHashRanges; - import dhtproto.client.internal.SharedResources; - import dhtproto.common.Update; - - /// Enum defining three-state return values of some methods. - private enum Action - { - Abort, // An error occurred and the request should be aborted. - Retry, // The operation failed, but can be retried (on another node). - Success // The operation succeeded. - } - - /// Delegate to get access to an event dispatcher for a specific connection. - private UseNodeDg use_node; - - /// Delegate to spawn a new request-on-conn. When spawned, Update.handle - /// will be called in the new RoC fiber. - private NewRequestOnConnDg new_request_on_conn; - - /// Serialised request context. - private Update.Context* context; - - /*************************************************************************** - - Entry point for the first RoC's logic. - - ***************************************************************************/ - - public void handle ( ) - { - auto shared_resources = SharedResources.fromObject( - this.context.shared_resources); - scope acquired_resources = shared_resources.new RequestResources; - - // Get the list of nodes reponsible for this hash. - auto nodes = shared_resources.node_hash_ranges.getNodesForHash( - this.context.user_params.args.key, - acquired_resources.getNodeHashRangeBuffer()); - - // If no node covers the record's hash, exit. - if ( nodes.array.length == 0 ) - { - this.context.shared_working.result = - Update.SharedWorking.Result.NoNode; - return; - } - - // Iterate over the list of responsible nodes, in newest to oldest order - // of responsibility, trying to get and update the record. - bool success, error; - foreach ( node_hash_range; nodes.array() ) - { - scope conn_dg = - ( RequestOnConn.EventDispatcher conn ) - { - auto ret = this.tryConnection(conn, nodes.array[0].addr, - acquired_resources); - with ( Action ) final switch ( ret ) - { - case Success: - this.context.shared_working.result = - Update.SharedWorking.Result.Succeeded; - success = true; - break; - case Abort: - // shared_working.result may either be the default - // value (Error) or may have been set to a specific - // code. - error = true; - break; - case Retry: - // No record on this node, retry on the next one. - break; - version (D_Version2){} else { - default: assert(false); - } - } - }; - this.use_node(node_hash_range.addr, conn_dg); - - if ( error ) - break; - } - - // The specified record does not currently exist in the DHT. Do nothing. - if ( !success && !error ) - this.context.shared_working.result = - Update.SharedWorking.Result.NoRecord; - } - - /*************************************************************************** - - Tries to do the complete update process, reading from the specified - node. - - Params: - conn = connection to communicate over - newest_responsible_node_addr = address/port of the node that is most - recently responsible for this record. If a second node must be - contacted, this is the node that the updated record will be sent - to - acquired_resources = request resource acquirer - - Returns: - Success if updated; Retry if not on this node; Abort on error - - ***************************************************************************/ - - private Action tryConnection ( RequestOnConn.EventDispatcher conn, - AddrPort newest_responsible_node_addr, - SharedResources.RequestResources acquired_resources ) - { - try - { - // Start the request on this node. - auto ret = this.queryValueFromNode(conn); - if ( ret != Action.Success ) - return ret; - - // Receive response (possibly including a record value) from the - // node. - ret = Action.Abort; - conn.receive( - ( in void[] const_payload ) - { - ret = this.handleResponse(conn, const_payload, - acquired_resources); - } - ); - if ( ret != Action.Success ) - return ret; - - // If a value was received, send the updated value to the DHT. - bool success; - if ( this.context.shared_working.updated_value !is null ) - { - if ( conn.remote_address == newest_responsible_node_addr ) - success = this.updateOnNode(conn); - else - success = this.updateOnDifferentNode(conn, - newest_responsible_node_addr); - } - // If the user did not provide an updated value, tell the node to - // leave the record as is. - else - success = this.leaveRecordOnNode(conn); - - return success ? Action.Success : Action.Abort; - } - catch ( IOError e ) - { - // Notify user of connection error. - Update.Notification n; - n.node_disconnected = RequestNodeExceptionInfo( - this.context.request_id, conn.remote_address, e); - Update.notify(this.context.user_params, n); - return Action.Abort; - } - } - - /*************************************************************************** - - Asks the node to send the record value; handles the response code. - - Params: - conn = connection to communicate over - - Returns: - Success if request supported; Abort on unsupported - - ***************************************************************************/ - - private Action queryValueFromNode ( RequestOnConn.EventDispatcher conn ) - { - // Send request info to node. - conn.send( - ( conn.Payload payload ) - { - payload.add(Update.cmd.code); - payload.add(Update.cmd.ver); - payload.addCopy(MessageType.GetRecord); - payload.addArray(this.context.user_params.args.channel); - payload.add(this.context.user_params.args.key); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Receive supported code from node. - auto supported = conn.receiveValue!(SupportedStatus)(); - if ( !Update.handleSupportedCodes(supported, this.context, - conn.remote_address) ) - { - // Unsupported; abort request. - return Action.Abort; - } - - return Action.Success; - } - - /*************************************************************************** - - Reads the response from the node, parses and handles the message type. - If the message contains a record value, it is passed to the user's - notification delegate. - - Params: - conn = connection to communicate over - const_payload = message payload received from node - acquired_resources = request resource acquirer - - Returns: - Success if a value was received; Retry is the record does not exist - on this node; Abort on error - - ***************************************************************************/ - - private Action handleResponse ( RequestOnConn.EventDispatcher conn, - in void[] const_payload, - SharedResources.RequestResources acquired_resources ) - { - Const!(void)[] payload = const_payload; - auto result = conn.message_parser.getValue!(MessageType)(payload); - with ( MessageType ) switch ( *result ) - { - case RecordValue: - auto value = conn.message_parser.getArray!(void)(payload); - - this.context.shared_working.original_hash = Fnv1a(value); - verify(this.context.shared_working.updated_value is null); - this.context.shared_working.updated_value = - acquired_resources.getVoidBuffer(); - - Update.Notification n; - n.received = RequestDataUpdateInfo( - this.context.request_id, value, - this.context.shared_working.updated_value); - Update.notify(this.context.user_params, n); - - return Action.Success; - - case NoRecord: - // This node doesn't have the record. Try another. - return Action.Retry; - - case WrongNode: - // The node is not reponsible for the key. Notify the user. - Update.Notification n; - n.wrong_node = RequestNodeInfo(this.context.request_id, - conn.remote_address); - Update.notify(this.context.user_params, n); - return Action.Abort; - - case Error: - // The node returned an error code. Notify the user. - Update.Notification n; - n.node_error = RequestNodeInfo(this.context.request_id, - conn.remote_address); - Update.notify(this.context.user_params, n); - return Action.Abort; - - default: - log.warn("Received unknown or unexpected message code {} from " - ~ "node in response to GetRecord message. Treating as " - ~ "Error.", result); - goto case Error; - } - } - - /*************************************************************************** - - Sends the node the new record value; handles the response code. - - Params: - conn = connection to communicate over - - Returns: - true if the update succeeded, false if there was an error - - ***************************************************************************/ - - private bool updateOnNode ( RequestOnConn.EventDispatcher conn ) - { - // Send updated record to node. - conn.send( - ( conn.Payload payload ) - { - payload.addCopy(MessageType.UpdateRecord); - payload.add(this.context.shared_working.original_hash); - payload.addArray(*this.context.shared_working.updated_value); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Handle response message. - auto result = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( result ) - { - case Ok: - return true; - - case UpdateConflict: - this.context.shared_working.result = - Update.SharedWorking.Result.Conflict; - return false; - - case Error: - // The node returned an error code. Notify the user. - Update.Notification n; - n.node_error = RequestNodeInfo(this.context.request_id, - conn.remote_address); - Update.notify(this.context.user_params, n); - return false; - - default: - log.warn("Received unknown or unexpected message code {} from " - ~ "node in response to UpdateRecord message. Treating as " - ~ "Error.", result); - goto case Error; - } - - assert(false); - } - - /*************************************************************************** - - Starts a new request-on-conn to write the record value to another node. - Asks the node the record was read from to remove it; handles the - response code. - - Params: - conn = connection to communicate with the node that originally had - the record - second_node_addr = address/port of the node that the updated record - will be sent to - - Returns: - true if the update succeeded, false if there was an error - - ***************************************************************************/ - - private bool updateOnDifferentNode ( RequestOnConn.EventDispatcher conn, - AddrPort second_node_addr ) - { - this.context.shared_working.first_request_on_conn = conn; - this.context.shared_working.second_node_addr = second_node_addr; - - // Start a new request-on-conn to write to the other node. This will - // cause the static `handler()` function to be called again in a new - // RequestOnConn instance that can be attached to a different - // connection. The shared working data contains everything the second - // request-on-conn needs. - new_request_on_conn(); - - // Wait until the second request-on-conn finishes. (This is necessary - // because this ROC owns the acquired resources tracker, which contains - // the buffer for the updated value. If this ROC were to simply exit at - // this stage, this buffer would be relinquished.) - auto event = conn.nextEvent(conn.NextEventFlags.init); - verify(event.active == event.Active.resumed); - verify(event.resumed.code == Update.SecondROCFinished); - - // If the request on the second request-on-conn failed, the result code - // will already have been set in the shared working data. - if ( this.context.shared_working.second_node_failed ) - return false; - - // Send message to the node that was read from, informing it that the - // record was successfully updated on another node and may be deleted. - conn.send( - ( conn.Payload payload ) - { - payload.addCopy(MessageType.RemoveRecord); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Handle response message. - auto result = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( result ) - { - case Ok: - return true; - - case Error: - // The node returned an error code. Notify the user. - Update.Notification n; - n.node_error = RequestNodeInfo(this.context.request_id, - conn.remote_address); - Update.notify(this.context.user_params, n); - return false; - - default: - log.warn("Received unknown or unexpected message code {} from " - ~ "node in response to RemoveRecord message. Treating as " - ~ "Error.", result); - goto case Error; - } - - assert(false); - } - - /*************************************************************************** - - Asks the node the record was read from to leave it; handles the response - code. - - Params: - conn = connection to communicate over - second_node_addr = address/port of the node that the updated record - will be sent to - - Returns: - true if the request succeeded, false if there was an error - - ***************************************************************************/ - - private bool leaveRecordOnNode ( RequestOnConn.EventDispatcher conn ) - { - conn.send( - ( conn.Payload payload ) - { - payload.addCopy(MessageType.LeaveRecord); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - auto result = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( result ) - { - case Ok: - return true; - - case Error: - // The node returned an error code. Notify the user. - Update.Notification n; - n.node_error = RequestNodeInfo(this.context.request_id, - conn.remote_address); - Update.notify(this.context.user_params, n); - return false; - - default: - log.warn("Received unknown or unexpected message code {} from " - ~ "node in response to LeaveRecord message. Treating as " - ~ "Error.", result); - goto case Error; - } - } -} - -/// Handler for the second request-on-conn (which receives the updated record -/// value, in cases when this is different to the node which it was read from.) -private struct SecondROCHandler -{ - import ocean.io.select.protocol.generic.ErrnoIOException: IOError; - import swarm.neo.client.NotifierTypes; - import swarm.neo.client.RequestHandlers; - import swarm.neo.client.RequestOnConn; - import swarm.neo.request.Command; - import dhtproto.common.Update; - - /// Delegate to get access to an event dispatcher for a specific connection. - private UseNodeDg use_node; - - /// Serialised request context. - private Update.Context* context; - - /*************************************************************************** - - Entry point for the second RoC's logic. - - ***************************************************************************/ - - public void handle ( ) - { - scope conn_dg = - ( RequestOnConn.EventDispatcher conn ) - { - if ( !this.handleRequest(conn) ) - this.context.shared_working.second_node_failed = true; - }; - this.use_node(this.context.shared_working.second_node_addr, conn_dg); - } - - /*************************************************************************** - - Writes the updated record to the specified node; handles the response - code. - - Params: - conn = connection to communicate over - - Returns: - true if successful, false if node returned an error code - - ***************************************************************************/ - - private bool handleRequest ( RequestOnConn.EventDispatcher conn ) - { - try - { - // Send request info to node - conn.send( - ( conn.Payload payload ) - { - payload.add(Update.cmd.code); - payload.add(Update.cmd.ver); - payload.addCopy(MessageType.UpdateRecord); - payload.addArray(this.context.user_params.args.channel); - payload.add(this.context.user_params.args.key); - payload.add(this.context.shared_working.original_hash); - payload.addArray(*this.context.shared_working.updated_value); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - conn.flush(); - - // Receive supported code from node. - auto supported = conn.receiveValue!(SupportedStatus)(); - if ( !Update.handleSupportedCodes(supported, this.context, - conn.remote_address) ) - { - // Request not supported; abort further handling. - return false; - } - - // Handle response message. - auto result = conn.receiveValue!(MessageType)(); - with ( MessageType ) switch ( result ) - { - case Ok: - return true; - - case UpdateConflict: - this.context.shared_working.result = - Update.SharedWorking.Result.Conflict; - return false; - - case WrongNode: - // The node is not reponsible for the key. Notify the user. - Update.Notification n; - n.wrong_node = RequestNodeInfo(this.context.request_id, - conn.remote_address); - Update.notify(this.context.user_params, n); - return false; - - case Error: - // The node returned an error code. Notify the user. - Update.Notification n; - n.node_error = RequestNodeInfo(this.context.request_id, - conn.remote_address); - Update.notify(this.context.user_params, n); - return false; - - default: - log.warn("Received unknown message code {} from node " - ~ "in response to Update request. Treating as " - ~ "Error.", result); - goto case Error; - } - } - catch ( IOError e ) - { - // Notify user of connection error. - Update.Notification n; - n.node_disconnected = RequestNodeExceptionInfo( - this.context.request_id, conn.remote_address, e); - Update.notify(this.context.user_params, n); - return false; - } - - assert(false); - } -} diff --git a/src/dhtproto/common/Exists.d b/src/dhtproto/common/Exists.d deleted file mode 100644 index 8c37ce09..00000000 --- a/src/dhtproto/common/Exists.d +++ /dev/null @@ -1,37 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT Exists request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.Exists; - -/******************************************************************************* - - Status code enum. Sent from the node to the client. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - /// Invalid, default value - None, - - /// Record exists - RecordExists, - - /// Record does not exist - NoRecord, - - /// Node is not responsible for record key - WrongNode, - - /// Internal node error occurred - Error -} diff --git a/src/dhtproto/common/Get.d b/src/dhtproto/common/Get.d deleted file mode 100644 index e5f602f9..00000000 --- a/src/dhtproto/common/Get.d +++ /dev/null @@ -1,37 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT Get request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.Get; - -/******************************************************************************* - - Message type enum. Sent from the node to the client. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - /// Invalid, default value - None, - - /// Record does not exist - NoRecord, - - /// Value read from DHT - Got, - - /// Node is not responsible for record key - WrongNode, - - /// Internal node error occurred - Error -} diff --git a/src/dhtproto/common/GetAll.d b/src/dhtproto/common/GetAll.d deleted file mode 100644 index 3f3997a7..00000000 --- a/src/dhtproto/common/GetAll.d +++ /dev/null @@ -1,42 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT GetAll request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.GetAll; - -import swarm.neo.request.Command; - -/******************************************************************************* - - Message type enum. Each message sent between the client and the node as part - of a GetAll request is prepended by a type indicator. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - None, // Invalid, default value - - // Message types sent from the client to the node: - Suspend, // Sent when the client wants the node to stop sending records - Resume, // Sent when the client wants the node to resume sending records - Stop, // Sent when the client wants the node to cleanly end the request - - // Message types sent between the client and the node (in either direction): - Ack, // Sent by the node to acknowledge a state change message; - // sent by the client to acknowledge the request finishing - - // Message types sent from the node to the client: - Started, // Request handling beginning - Error, // Internal node error occurred; request not being handled - RecordBatch, // Sent by the node when it sends a batch of records - Finished // Send by the node when the iteration is finished -} diff --git a/src/dhtproto/common/GetChannels.d b/src/dhtproto/common/GetChannels.d deleted file mode 100644 index 5dbeda98..00000000 --- a/src/dhtproto/common/GetChannels.d +++ /dev/null @@ -1,32 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT GetChannels request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.GetChannels; - -import swarm.neo.request.Command; - -/******************************************************************************* - - Message type enum. Each message sent between the client and the node as part - of a GetChannels request is prepended by a type indicator. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - None, // Invalid, default value - - // Message types sent from the node to the client: - Error, // Internal node error - ChannelName, // Sent by the node when it sends a channel name - Finished // Send by the node when the request is finished -} diff --git a/src/dhtproto/common/GetHashRange.d b/src/dhtproto/common/GetHashRange.d deleted file mode 100644 index 41503372..00000000 --- a/src/dhtproto/common/GetHashRange.d +++ /dev/null @@ -1,39 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT GetHashRange request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.GetHashRange; - -import swarm.neo.request.Command; - -/******************************************************************************* - - Message type enum. Each message sent between the node and the client as part - of a GetHashRange request is prepended by a type indicator. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - /// Invalid, default value - None, - - /// The node handling the GetHashRange request has been informed about the - /// existence of another DHT node, which it forwards in this message. The - /// other node may be previously unknown to the client (in which case, an - /// entry is added to the node hash range registry) or may be already known - /// to the client (in which case, its entry in the registry is updated). - NewNode, - - /// The node handling the GetHashRange request has changed its hash range. - /// An entry is added to the node hash range registry. - ChangeHashRange -} diff --git a/src/dhtproto/common/Mirror.d b/src/dhtproto/common/Mirror.d deleted file mode 100644 index c3c2df37..00000000 --- a/src/dhtproto/common/Mirror.d +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT Mirror request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.Mirror; - -import swarm.neo.request.Command; - -/******************************************************************************* - - Message type enum. Each message sent between the client and the node as part - of a Mirror request is prepended by a type indicator. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - None, // Invalid, default value - - // Message types sent from the client to the node: - Suspend, // Sent when the client wants the node to stop sending records - Resume, // Sent when the client wants the node to resume sending records - Stop, // Sent when the client wants the node to cleanly end the request - - // Message types sent between the client and the node (in either direction): - Ack, // Sent by the node to acknowledge a state change message; - // sent by the client to acknowledge the request finishing - - // Message types sent from the node to the client: - Started, // Request handling beginning - Error, // Internal node error occurred; request not being handled - RecordChanged, // Sent by the node when it sends a record value - RecordDeleted, // Sent by the node when it sends the key of a deleted record - RecordRefreshBatch, // Sent by the node when it sends a batch of refreshed - // records - ChannelRemoved, // Sent when the channel being consumed is removed - UpdateOverflow // Sent when the queue of updates overflows -} diff --git a/src/dhtproto/common/Put.d b/src/dhtproto/common/Put.d deleted file mode 100644 index 6fc49d5c..00000000 --- a/src/dhtproto/common/Put.d +++ /dev/null @@ -1,34 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT Put request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.Put; - -/******************************************************************************* - - Message type enum. Sent from the node to the client. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - /// Invalid, default value - None, - - /// Value written to DHT - Put, - - /// Node is not responsible for record key - WrongNode, - - /// Internal node error occurred - Error -} diff --git a/src/dhtproto/common/Remove.d b/src/dhtproto/common/Remove.d deleted file mode 100644 index 9cda44d2..00000000 --- a/src/dhtproto/common/Remove.d +++ /dev/null @@ -1,37 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT Remove request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.Remove; - -/******************************************************************************* - - Status code enum. Sent from the node to the client. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - /// Invalid, default value - None, - - /// Record does not exist - NoRecord, - - /// Value removed from DHT - Removed, - - /// Node is not responsible for record key - WrongNode, - - /// Internal node error occurred - Error -} diff --git a/src/dhtproto/common/RemoveChannel.d b/src/dhtproto/common/RemoveChannel.d deleted file mode 100644 index cd3058b1..00000000 --- a/src/dhtproto/common/RemoveChannel.d +++ /dev/null @@ -1,34 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT RemoveChannel request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.RemoveChannel; - -/******************************************************************************* - - Status code enum. Sent from the node to the client. - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - /// Invalid, default value - None, - - /// RemoveChannel succeeded - ChannelRemoved, - - /// Channel cannot be removed as the client does not have admin permissions - NotPermitted, - - /// Internal node error occurred - Error -} diff --git a/src/dhtproto/common/RequestCodes.d b/src/dhtproto/common/RequestCodes.d deleted file mode 100644 index 1724ac44..00000000 --- a/src/dhtproto/common/RequestCodes.d +++ /dev/null @@ -1,28 +0,0 @@ -/******************************************************************************* - - DHT neo request codes. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.RequestCodes; - -public enum RequestCode : ubyte -{ - None, - GetHashRange, - Put, - Get, - Mirror, - GetAll, - GetChannels, - Exists, - Remove, - RemoveChannel, - Update -} diff --git a/src/dhtproto/common/Update.d b/src/dhtproto/common/Update.d deleted file mode 100644 index 16a9c236..00000000 --- a/src/dhtproto/common/Update.d +++ /dev/null @@ -1,52 +0,0 @@ -/******************************************************************************* - - Protocol definition of the DHT Update request. - - Copyright: - Copyright (c) 2018 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.common.Update; - -/******************************************************************************* - - Message code enum - -*******************************************************************************/ - -public enum MessageType : ubyte -{ - /// Invalid, default value - None, - - // Sent from the client to the node: - - /// Client requesting record value from a node - GetRecord, - /// Client requesting to update record value on a node - UpdateRecord, - /// Client informing node that record was updated on another node and can be - /// removed - RemoveRecord, - /// Client informing node that record should be left as it is - LeaveRecord, - - // Sent from the node to the client: - - /// Record value sent from node - RecordValue, - /// Record value updated / left / removed, per client's request - Ok, - /// Record does not exist in node - NoRecord, - /// Record has been updated by another client. The request may be retried - UpdateConflict, - /// Node is not responsible for record key - WrongNode, - /// Internal node error occurred - Error -} diff --git a/src/dhtproto/node/neo/request/Exists.d b/src/dhtproto/node/neo/request/Exists.d deleted file mode 100644 index 08b69d62..00000000 --- a/src/dhtproto/node/neo/request/Exists.d +++ /dev/null @@ -1,130 +0,0 @@ -/******************************************************************************* - - Exists request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.Exists; - -import ocean.core.VersionCheck; -import swarm.neo.node.IRequestHandler; - -/******************************************************************************* - - v0 Exists request protocol. - -*******************************************************************************/ - -public abstract scope class ExistsProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import dhtproto.common.Exists; - import dhtproto.node.neo.request.core.Mixins; - - import ocean.transition; - - /// Mixin the initialiser and the connection and resources members. - mixin IRequestHandlerRequestCore!(); - - /// Response to client. - private MessageType response; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - auto parser = this.connection.event_dispatcher.message_parser(); - - auto channel = parser.getArray!(char)(init_payload); - auto key = *parser.getValue!(hash_t)(init_payload); - - // Check record key and read from channel, if ok. - if ( this.responsibleForKey(key) ) - { - bool found; - if ( this.exists(channel, key, found) ) - this.response = found - ? MessageType.RecordExists : MessageType.NoRecord; - else - this.response = MessageType.Error; - } - else - this.response = MessageType.WrongNode; - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - auto ed = this.connection.event_dispatcher(); - - // Send status code - ed.send( - ( ed.Payload payload ) - { - payload.add(this.response); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - } - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - abstract protected bool responsibleForKey ( hash_t key ); - - /*************************************************************************** - - Checks whether a single record exists in the storage engine. - - Params: - channel = channel to check in - key = key of record to check - found = out value, set to true if the record exists - - Returns: - true if the operation succeeded; false if an error occurred - - ***************************************************************************/ - - abstract protected bool exists ( cstring channel, hash_t key, out bool found ); -} diff --git a/src/dhtproto/node/neo/request/Get.d b/src/dhtproto/node/neo/request/Get.d deleted file mode 100644 index 581309fe..00000000 --- a/src/dhtproto/node/neo/request/Get.d +++ /dev/null @@ -1,167 +0,0 @@ -/******************************************************************************* - - Get request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.Get; - -import ocean.core.VersionCheck; -import swarm.neo.node.IRequestHandler; - -/******************************************************************************* - - v0 Get request protocol. - -*******************************************************************************/ - -public abstract class GetProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import dhtproto.common.Get; - import dhtproto.node.neo.request.core.Mixins; - - import ocean.transition; - import ocean.core.Verify; - import ocean.core.array.Mutation : copy; - - /*************************************************************************** - - Mixin the initialiser and the connection and resources members. - - ***************************************************************************/ - - mixin IRequestHandlerRequestCore!(); - - /// Name of channel to read from. - private void[]* channel; - - /// Key to be read. - private hash_t key; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - auto ed = this.connection.event_dispatcher(); - - this.channel = this.resources.getVoidBuffer(); - (*this.channel).copy(ed.message_parser.getArray!(char)(init_payload)); - enableStomping(*this.channel); - this.key = *ed.message_parser.getValue!(hash_t)(init_payload); - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - auto ed = this.connection.event_dispatcher(); - - void sendResponse ( MessageType status_code, - void delegate ( ed.Payload ) extra = null ) - { - ed.send( - ( ed.Payload payload ) - { - payload.add(status_code); - } - ); - - // TODO: this could be sent in the same message as the status code, - // above. (Client would need to be adaptated.) - if ( extra !is null ) - ed.send(extra); - } - - // Check record key and read from channel, if ok. - if ( this.responsibleForKey(this.key) ) - { - bool sent; - auto ok = this.get(cast(char[])*this.channel, this.key, - ( Const!(void)[] value ) - { - verify(value !is null); - verify(value.length > 0); - - sendResponse(MessageType.Got, - ( ed.Payload payload ) - { - payload.addArray(value); - } - ); - - sent = true; - } - ); - - if ( !sent ) - sendResponse(ok ? MessageType.NoRecord - : MessageType.Error); - } - else - sendResponse(MessageType.WrongNode); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - } - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - abstract protected bool responsibleForKey ( hash_t key ); - - /*************************************************************************** - - Gets a single record from the storage engine. - - Params: - channel = channel to read from - key = key of record to read - dg = called with the value of the record, if it exists - - Returns: - true if the operation succeeded (the record was fetched or did not - exist); false if an error occurred - - ***************************************************************************/ - - abstract protected bool get ( cstring channel, hash_t key, - void delegate ( Const!(void)[] value ) dg ); -} diff --git a/src/dhtproto/node/neo/request/GetAll.d b/src/dhtproto/node/neo/request/GetAll.d deleted file mode 100644 index 861b4bc7..00000000 --- a/src/dhtproto/node/neo/request/GetAll.d +++ /dev/null @@ -1,532 +0,0 @@ -/******************************************************************************* - - v0 GetAll request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.GetAll; - -import ocean.core.VersionCheck; -import swarm.neo.node.IRequestHandler; -import ocean.util.log.Logger; - -/// ditto -public abstract class GetAllProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import swarm.neo.request.RequestEventDispatcher; - import swarm.neo.util.MessageFiber; - import swarm.util.RecordBatcher; - import dhtproto.common.GetAll; - import dhtproto.node.neo.request.core.Mixins; - import ocean.transition; - import ocean.core.Verify; - - /*************************************************************************** - - Mixin the initialiser and the connection and resources members. - - ***************************************************************************/ - - mixin IRequestHandlerRequestCore!(); - - /// Request event dispatcher. - private RequestEventDispatcher request_event_dispatcher; - - /// Batch of records to send. - private RecordBatcher batcher; - - /// Destination buffer for compressing batches of records. - private void[]* compressed_batch; - - /// Connection to the client. - private RequestOnConn connection; - - /// Writer fiber instance. - private Writer writer; - - /// Controller fiber instance. - private Controller controller; - - /// Set by the Writer when the iteration over the records has finished. Used - /// by the Controller to ignore incoming messages from that point. This is - /// to avoid a race condition between the Finished message and a control - /// message sent by the client. - private bool has_ended; - - /// If true, only record keys will be sent, no values. - private bool keys_only; - - /// Value filtering wrapper struct. - private struct ValueFilter - { - import ocean.text.Search; - - /// Sub-array matcher. (Type ubyte as doesn't compile with void.) - private SearchFruct!(Const!(ubyte)) matcher; - - /// Filtering active? - private bool active; - - /*********************************************************************** - - Initialises the filter from the specified filter array. - - Params: - filter = sub-array to filter records by. If empty, no filtering - is employed - - ***********************************************************************/ - - public void init ( in void[] filter ) - { - if ( filter.length > 0 ) - { - this.matcher = search(cast(Const!(ubyte)[])filter); - this.active = true; - } - } - - /*********************************************************************** - - Checks whether the provided record value passes the filter or not. - - Params: - value = value to filter - - Returns: - true if the value passes (i.e. matches) the filter or if - filtering is not active - - ***********************************************************************/ - - public bool match ( in void[] value ) - { - if ( this.active ) - return this.matcher.forward(cast(Const!(ubyte)[])value) - < value.length; - else - return true; - } - } - - /// Value filter. - private ValueFilter filter; - - /// If true, the request should be started in the suspended state. - bool start_suspended; - - /// Return value of startIteration() or continueIteration(). - private bool initialised_ok; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - bool continuing; - hash_t continue_from; - cstring channel; - Const!(void)[] value_filter; - this.connection.event_dispatcher.message_parser.parseBody(init_payload, - this.start_suspended, channel, continuing, continue_from, - this.keys_only, value_filter); - - if ( continuing ) - this.initialised_ok = this.continueIteration(channel, continue_from); - else - this.initialised_ok = this.startIteration(channel); - - // Set up filtering. - this.filter.init(value_filter); - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - // Send status code - this.connection.event_dispatcher.send( - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(this.initialised_ok - ? MessageType.Started : MessageType.Error); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - this.connection.event_dispatcher.flush(); - - if ( !this.initialised_ok ) - return; - - // Acquire required resources. - this.compressed_batch = this.resources.getVoidBuffer(); - this.batcher = this.resources.getRecordBatcher(); - - // Start the two fibers which form the request handling logic. - scope writer_ = new Writer; - scope controller_ = new Controller; - - this.request_event_dispatcher.initialise(&this.resources.getVoidBuffer); - - // Note: we store refs to the scope instances in class fields as a - // convenience to be able to access them from each other (e.g. the - // writer needs to access the controller and vice-versa). It's normally - // not safe to store refs to scope instances outside of the scope, so we - // need to be careful to only use them while they are in scope. - this.writer = writer_; - this.controller = controller_; - scope ( exit ) - { - this.writer = null; - this.controller = null; - } - - if ( this.start_suspended ) - this.writer.suspender.requestSuspension(); - - this.controller.fiber.start(); - this.writer.fiber.start(); - this.request_event_dispatcher.eventLoop( - this.connection.event_dispatcher); - } - - /*************************************************************************** - - Called to begin the iteration over the channel being fetched. - - Params: - channel = name of channel to iterate over - - Returns: - true if the iteration has been initialised, false to abort the - request - - ***************************************************************************/ - - abstract protected bool startIteration ( cstring channel ); - - /*************************************************************************** - - Called to continue the iteration over the channel being fetched, - continuing from the specified hash (the last record received by the - client). - - Params: - channel = name of channel to iterate over - continue_from = hash of last record received by the client. The - iteration will continue from the next hash in the channel - - Returns: - true if the iteration has been initialised, false to abort the - request - - ***************************************************************************/ - - abstract protected bool continueIteration ( cstring channel, - hash_t continue_from ); - - /*************************************************************************** - - Gets the next record in the iteration, if one exists. - - Params: - dg = called with the key and value of the next record, if available - - Returns: - true if a record was passed to `dg` or false if the iteration is - finished - - ***************************************************************************/ - - abstract protected bool getNext ( - void delegate ( hash_t key, Const!(void)[] value ) dg ); - - /*************************************************************************** - - Fiber which handles: - 1. Iterating over the channel. - 2. Forming batches of records and forwarding them to the client. - 3. Informing the client when the iteration ends. - - ***************************************************************************/ - - private class Writer - { - import swarm.neo.util.DelayedSuspender; - - /// Fiber. - public MessageFiber fiber; - - /// Helper to allow other fibers to request writes to the client to be - /// suspended. - public DelayedSuspender suspender; - - /// Fiber resume code used to resume DelayedSuspender. - private const ResumeAfterSuspension = 1; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - this.suspender = DelayedSuspender( - &this.outer.request_event_dispatcher, - this.outer.connection.event_dispatcher, - this.fiber, ResumeAfterSuspension); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - hash_t key; - uint yield_counter; - - // Iterate over the channel and send each record to the client. - bool more; - do - { - more = this.outer.getNext( - ( hash_t key, Const!(void)[] value ) - { - if ( !this.outer.filter.match(cast(cstring)value) ) - return; - - cstring key_slice = (cast(char*)&key)[0..key.sizeof]; - auto add_result = this.addToBatch(key_slice, - cast(cstring)value); - - // If suspended, wait until resumed. - this.suspender.suspendIfRequested(); - - with ( RecordBatcher.AddResult ) switch ( add_result ) - { - case Added: - // Can add more data to the batch - break; - case BatchFull: - // New record does not fit into this batch, send - // it and add the record to the next batch - this.sendBatch(); - add_result = this.addToBatch(key_slice, - cast(cstring)value); - verify(add_result == Added); - break; - case TooBig: - // Impossible to fit the record even in empty batch - log.warn( - "GetAll: Large record 0x{:x16} ({} bytes) skipped.", - key, value.length); - break; - default: - assert(false, "Invalid AddResult in switch"); - } - } - ); - - this.outer.request_event_dispatcher.periodicYield( - this.fiber, yield_counter, 10); - } - while ( more ); - - // Handle the last pending batch at the end of the iteration (does - // nothing if no records are pending) - this.sendBatch(); - - // The request is now finished. Inform client of this and ignore any - // further incoming control messages. - this.outer.has_ended = true; - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.Finished); - } - ); - - // Wait for ACK from client - this.outer.request_event_dispatcher.receive(this.fiber, - Message(MessageType.Ack)); - - // It's no longer valid to handle control messages. - this.outer.request_event_dispatcher.abort( - this.outer.controller.fiber); - } - - /*********************************************************************** - - Tries to add the provided record to the batch. If a keys-only - iteration was requested by the client, only the key is added to the - batch. - - Params: - key = record key - value = record value; added if `this.outer.keys_only` is false - - Returns: - result of adding the record to the batch - - ***********************************************************************/ - - private RecordBatcher.AddResult addToBatch ( cstring key, cstring value ) - { - if ( this.outer.keys_only ) - return this.outer.batcher.add(key); - else - return this.outer.batcher.add(key, value); - } - - /*********************************************************************** - - Sends a batch of records to the client. - - ***********************************************************************/ - - private void sendBatch ( ) - { - this.outer.batcher.compress( - *(cast(ubyte[]*)this.outer.compressed_batch)); - - if ( this.outer.compressed_batch.length == 0 ) - return; // Nothing in the batch - - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.RecordBatch); - payload.addArray(*this.outer.compressed_batch); - } - ); - - // flush() does not suspend the fiber, so is safe to call in a - // RequestEventDispatcher-managed request. - static if (!hasFeaturesFrom!("swarm", 4, 7)) - this.outer.connection.event_dispatcher.flush(); - } - } - - /*************************************************************************** - - Fiber which handles: - 1. Reading control messages from the client. - 2. Sending an ACK message back (the client relies on ACKs to - re-enable the user-facing controller). - 3. Controlling the writer fiber, as appropriate, to carry out the - control messages from the client. - - ***************************************************************************/ - - private class Controller - { - /// Fiber. - public MessageFiber fiber; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - bool stopped; - do - { - // Receive message from client. - auto message = - this.outer.request_event_dispatcher.receive( - this.fiber, - Message(MessageType.Suspend), Message(MessageType.Resume), - Message(MessageType.Stop)); - - if ( this.outer.has_ended ) - continue; - - // Send ACK. The protocol guarantees that the client will not - // send any further messages until it has received the ACK. - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.Ack); - } - ); - - with ( MessageType ) switch ( message.type ) - { - case Suspend: - this.outer.writer.suspender.requestSuspension(); - break; - case Resume: - this.outer.writer.suspender.resumeIfSuspended(); - break; - case Stop: - stopped = true; - - // End the writer fiber. The request is finished. - this.outer.request_event_dispatcher.abort( - this.outer.writer.fiber); - break; - default: - assert(false); - } - } - while ( !stopped ); - } - } -} - -/// Static module logger -static private Logger log; - -static this ( ) -{ - log = Log.lookup("dhtproto.node.neo.request.GetAll"); -} diff --git a/src/dhtproto/node/neo/request/GetChannels.d b/src/dhtproto/node/neo/request/GetChannels.d deleted file mode 100644 index 7a1c49b0..00000000 --- a/src/dhtproto/node/neo/request/GetChannels.d +++ /dev/null @@ -1,100 +0,0 @@ -/******************************************************************************* - - v0 GetChannels request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.GetChannels; - -import swarm.neo.node.IRequestHandler; - -/// ditto -public abstract class GetChannelsProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import dhtproto.common.GetChannels; - import dhtproto.node.neo.request.core.Mixins; - import ocean.core.Array : copy; - import ocean.transition; - - /*************************************************************************** - - Mixin the initialiser and the connection and resources members. - - ***************************************************************************/ - - mixin IRequestHandlerRequestCore!(); - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - // Nothing more to parse from the payload. - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - auto channel_buf = this.resources.getVoidBuffer(); - - foreach ( channel; this ) - { - // Copy the channel name in case it changes during sending. - (*channel_buf).copy(channel); - - this.connection.event_dispatcher.send( - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.ChannelName); - payload.addArray(*channel_buf); - } - ); - } - - // Send finished code - this.connection.event_dispatcher.send( - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.Finished); - } - ); - } - - /*************************************************************************** - - opApply iteration over the names of the channels in storage. - - ***************************************************************************/ - - protected abstract int opApply ( int delegate ( ref cstring ) dg ); -} diff --git a/src/dhtproto/node/neo/request/GetHashRange.d b/src/dhtproto/node/neo/request/GetHashRange.d deleted file mode 100644 index 6080fa61..00000000 --- a/src/dhtproto/node/neo/request/GetHashRange.d +++ /dev/null @@ -1,243 +0,0 @@ -/******************************************************************************* - - GetHashRange request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.GetHashRange; - -import ocean.core.VersionCheck; -import swarm.neo.AddrPort; -import swarm.neo.node.IRequestHandler; - -/******************************************************************************* - - Struct containing the details of an update to the hash range (either of - this node -- if the `self` field is true -- or another node). - -*******************************************************************************/ - -public struct HashRangeUpdate -{ - /// If true, the update relates to this node having changed its hash - /// range. If false, the update relates to new information about another - /// node. - bool self; - - /// IP address/port of other node's neo protocol (only used if self is false). - AddrPort addr; - - /// New minimum hash. - hash_t min; - - /// New maximum hash. - hash_t max; -} - - -/******************************************************************************* - - v0 GetHashRange request protocol. - -*******************************************************************************/ - -public abstract scope class GetHashRangeProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import dhtproto.common.GetHashRange; - import dhtproto.node.neo.request.core.Mixins; - - import ocean.transition; - import ocean.core.Verify; - - /// Mixin the initialiser and the connection and resources members. - mixin IRequestHandlerRequestCore!(); - - /*************************************************************************** - - Codes used when resuming the fiber. - - ***************************************************************************/ - - private enum NodeFiberResumeCode : uint - { - HashRangeUpdate = 1 - } - - /*************************************************************************** - - If true, hashRangeUpdate() (called when either the hash range of this - node has changed or information about another node is available) resumes - the fiber. - - ***************************************************************************/ - - private bool resume_fiber_on_update; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - // Nothing more to parse from the payload. - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - hash_t min, max; - this.getCurrentHashRange(min, max); - - this.registerForHashRangeUpdates(); - scope ( exit ) - this.unregisterForHashRangeUpdates(); - - // Send current hash range - auto ed = this.connection.event_dispatcher(); - ed.send( - ( ed.Payload payload ) - { - payload.add(min); - payload.add(max); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - - while ( true ) - { - // Send all pending updates to the client - HashRangeUpdate update; - while ( this.getNextHashRangeUpdate(update) ) - this.sendHashRangeUpdate(update); - - // Wait for updates about the hash range - this.resume_fiber_on_update = true; - auto resume_code = connection.suspendFiber(); - verify(resume_code == NodeFiberResumeCode.HashRangeUpdate, - "Unexpected fiber resume code"); - this.resume_fiber_on_update = false; - } - } - - /*************************************************************************** - - Gets the current hash range of this node. - - Params: - min = out value where the current minimum hash of this node is stored - max = out value where the current maximum hash of this node is stored - - ***************************************************************************/ - - protected abstract void getCurrentHashRange ( out hash_t min, out hash_t max ); - - /*************************************************************************** - - Informs the node that this request is now waiting for hash range - updates. hashRangeUpdate() will be called, when updates are pending. - - ***************************************************************************/ - - protected abstract void registerForHashRangeUpdates ( ); - - /*************************************************************************** - - Informs the node that this request is no longer waiting for hash range - updates. - - ***************************************************************************/ - - protected abstract void unregisterForHashRangeUpdates ( ); - - /*************************************************************************** - - Gets the next pending hash range update (or returns false, if no updates - are pending). The implementing node should store a queue of updates per - GetHashRange request and feed them to the request, in order, when this - method is called. - - Params: - update = out value to receive the next pending update, if one is - available - - Returns: - false if no update is pending - - ***************************************************************************/ - - protected abstract bool getNextHashRangeUpdate ( out HashRangeUpdate update ); - - /*************************************************************************** - - Notifies the request when either the hash range of this node has changed - or information about another node is available. The implementing class - must call this when notified by the node of this occurring. - - ***************************************************************************/ - - final protected void hashRangeUpdate ( ) - { - if ( this.resume_fiber_on_update ) - this.connection.resumeFiber(NodeFiberResumeCode.HashRangeUpdate); - } - - /*************************************************************************** - - Informs the client about a single hash range update. - - Params: - update = hash range update to inform client about - - ***************************************************************************/ - - private void sendHashRangeUpdate ( HashRangeUpdate update ) - { - auto ed = this.connection.event_dispatcher(); - ed.send( - ( ed.Payload payload ) - { - if ( update.self ) - payload.addCopy(MessageType.ChangeHashRange); - else - { - payload.addCopy(MessageType.NewNode); - payload.add(update.addr.naddress); - payload.add(update.addr.nport); - } - - payload.add(update.min); - payload.add(update.max); - } - ); - } -} diff --git a/src/dhtproto/node/neo/request/Mirror.d b/src/dhtproto/node/neo/request/Mirror.d deleted file mode 100644 index b481c385..00000000 --- a/src/dhtproto/node/neo/request/Mirror.d +++ /dev/null @@ -1,1159 +0,0 @@ -/******************************************************************************* - - v0 Mirror request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.Mirror; - -import swarm.neo.node.IRequestHandler; - -import ocean.util.log.Logger; - -version ( UnitTest ) -{ - import ocean.core.Test; -} - -/// ditto -public abstract class MirrorProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import swarm.neo.connection.RequestOnConnBase; - import swarm.neo.request.RequestEventDispatcher; - import swarm.neo.util.Batch; - import swarm.neo.util.MessageFiber; - import dhtproto.common.Mirror; - import dhtproto.node.neo.request.core.Mixins; - import ocean.transition; - import ocean.core.Verify; - - /*************************************************************************** - - Mixin the initialiser and the connection and resources members. - - ***************************************************************************/ - - mixin IRequestHandlerRequestCore!(); - - /// Request event dispatcher. - private RequestEventDispatcher request_event_dispatcher; - - /// Buffer used to store record values to be sent to the client. - private void[]* value_buffer; - - /// Connection to the client. - private RequestOnConn connection; - - /// Enum defining the possible types of updates which can be mirrored. - protected enum UpdateType - { - Change, - Deletion - } - - /// Struct defining a single update on the mirrored channel. - protected struct Update - { - /// The type of the update (see dhtproto.node.neo.request.Mirror). - UpdateType type; - - /// Key of record which was updated. - hash_t key; - } - - /*************************************************************************** - - Struct template implementing a simple, typed, array-based ring queue. - - Params: - E = type of queue element - - ***************************************************************************/ - - private struct Queue ( E ) - { - import ocean.core.array.Mutation : removeShift; - - /// Backing array in which elements are stored. Must be set before using - /// the queue. - private E[] queue; - - /// Maximum allowed size of the queue in bytes. - private const max_size = 256 * 1024; - - /// Maximum allowed elements of type E of the queue. - private size_t max_elems; - - /// Current number of elements in the queue. - private size_t num_elems; - - /// Index (E-based) of the next element to write. - private size_t write_to_elem; - - /// Index (E-based) of the next element to read. - private size_t read_from_elem; - - /// Tests for basic sanity of num_elems and the read/write indices. - invariant ( ) - { - if ( this.num_elems == 0 ) - assert(this.write_to_elem == this.read_from_elem); - else - assert(this.write_to_elem == - (this.read_from_elem + this.num_elems) % this.max_elems); - } - - /*********************************************************************** - - Initialises the queue for use, setting up the provided array for use - as the queue's backing storage. - - Params: - buf = array to use to back queue storage - - ***********************************************************************/ - - public void initialise ( ref void[] buf ) - { - this.max_elems = (max_size / E.sizeof) * E.sizeof; - buf.length = this.max_elems * E.sizeof; - this.queue = cast(E[])buf; - } - - /*********************************************************************** - - Pushes an element to the queue, if there is space. - - Params: - e = element to push - - Returns: - true if the element was pushed, false if the queue is full - - ***********************************************************************/ - - public bool push ( E e ) - { - if ( this.num_elems >= this.max_elems ) - return false; - - this.queue[this.write_to_elem] = e; - this.incWrap(this.write_to_elem); - this.num_elems++; - - return true; - } - - /*********************************************************************** - - Pops an element from the queue. May only be called when the queue - contains elements. - - Returns: - popped element - - ***********************************************************************/ - - public E pop ( ) - { - verify(this.num_elems > 0); - E e; - e = this.queue[this.read_from_elem]; - this.incWrap(this.read_from_elem); - this.num_elems--; - - return e; - } - - /*********************************************************************** - - Returns: - the number of elements in the queue - - ***********************************************************************/ - - public size_t length ( ) - { - return this.num_elems; - } - - /*********************************************************************** - - Returns: - true if the queue is full - - ***********************************************************************/ - - public bool isFull ( ) - { - return this.num_elems == this.max_elems; - } - - /*********************************************************************** - - Helper function to increment an element index, taking account of - wrapping in the ring queue. - - Params: - elem_index = element index to increment and wrap - - ***********************************************************************/ - - private void incWrap ( ref size_t elem_index ) - { - elem_index++; - verify(elem_index <= this.max_elems); - if ( elem_index == this.max_elems ) - elem_index = 0; - } - } - - // Tests Queue pushing, popping, and wrapping. - unittest - { - void[] backing; - Queue!(Update) q; - q.initialise(backing); - - const elems_per_cycle = 7; - uint write_wraps, read_wraps; - Update update; - for ( uint i; i < q.max_elems; i++ ) - { - for ( uint pu; pu < elems_per_cycle; pu++ ) - { - update.key = i * pu; - auto pushed = q.push(update); - test(pushed); - if ( q.write_to_elem == 0 ) - write_wraps++; - } - - for ( uint po; po < elems_per_cycle; po++ ) - { - auto popped = q.pop(); - test!("==")(popped.key, i * po); - if ( q.read_from_elem == 0 ) - read_wraps++; - } - } - - test!("==")(write_wraps, elems_per_cycle); - test!("==")(read_wraps, elems_per_cycle); - } - - /// Queue of updates. - private Queue!(Update) update_queue; - - /// Queue of refreshed records. - private Queue!(hash_t) refresh_queue; - - /// Batch of refreshed records. (Refreshed records are sent in batches as an - /// optimisation, as they always occur in floods. For the client, handling a - /// batch of records is much more efficient than handling a stream of - /// individual records, as it reduces the number of fiber switches.) - private BatchWriter!(hash_t, void[]) refresh_batch; - - /// Batch compression buffer. - private void[]* compress_buffer; - - /// Struct wrapping fields and logic for counting queue overflows and - /// deciding when to notify the client that overflows have occurred. - private struct UpdateQueueOverflows - { - import core.stdc.time : time_t, time; - - /// The number of records which could not be pushed to the update queue - /// because it was full. Cleared each time a notification is sent to the - /// client. - private uint count_since_last_notification; - - /// Timestamp at which the last queue overflow occurred. Storing this - /// value as seconds allows us to limit the rate of client - /// notifications to once per second. - private time_t last_overflow_time; - - /// Timestamp at which the client was last notified of an overflow. - private time_t last_notified_overflow_time; - - /*********************************************************************** - - Called to indicate that an update could not be pushed into the queue - because it was full. - - ***********************************************************************/ - - public void opPostInc ( ) - { - this.last_overflow_time = time(null); - this.count_since_last_notification++; - } - - /*********************************************************************** - - Returns: - true if it's time to send a notification to the client - - ***********************************************************************/ - - public bool notification_pending ( ) - { - return this.last_overflow_time > this.last_notified_overflow_time; - } - - /*********************************************************************** - - Called to indicate that a notification was sent to the client. - - ***********************************************************************/ - - public void notification_sent ( ) - { - this.last_notified_overflow_time = this.last_overflow_time; - this.count_since_last_notification = 0; - } - } - - /// Overflow notification tracker. - private UpdateQueueOverflows update_queue_overflows; - - /// Codes used when resuming the fiber to interrupt waiting for I/O. - private enum NodeFiberResumeCode : ubyte - { - PushedToQueue = 1, - ChannelRemoved = 2, - PeriodicRefresh = 3, - RefreshQueueEmptied = 4, - QueueOverflowNotification = 5, - ResumeAfterSuspension = 6 - } - - /// If true, the request should be started in the suspended state. - private bool start_suspended; - - /// If true, the request, upon starting, will immediately send all records - /// in the channel to the client. - private bool initial_refresh; - - /// If non-zero, the request will repeatedly send all records in the channel - /// to the client after every specified period. If zero, no periodic - /// refreshes will occur. - private uint periodic_refresh_s; - - /// Writer fiber instance. - private Writer writer; - - /// Controller fiber instance. - private Controller controller; - - /// PeriodicRefresh fiber instance. - private PeriodicRefresh periodic_refresh; - - /// Set by the Writer when the iteration over the records has finished. Used - /// by the Controller to ignore incoming messages from that point. This is - /// to avoid a race condition between the Finished message and a control - /// message sent by the client. - private bool has_ended; - - /// Return value of prepareChannel(). - private bool initialised_ok; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - // Parse initial message from client. - cstring channel; - this.connection.event_dispatcher.message_parser.parseBody(init_payload, - this.start_suspended, channel, this.initial_refresh, - this.periodic_refresh_s); - - this.initialised_ok = this.prepareChannel(channel); - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - // Send status code - this.connection.event_dispatcher.send( - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(this.initialised_ok - ? MessageType.Started : MessageType.Error); - } - ); - - if ( !this.initialised_ok ) - return; - - this.value_buffer = this.resources.getVoidBuffer(); - this.update_queue.initialise(*this.resources.getVoidBuffer()); - this.refresh_queue.initialise(*this.resources.getVoidBuffer()); - const max_batch_size = 64 * 1024; // TODO: read from config - this.refresh_batch.initialise(this.resources.getVoidBuffer(), - max_batch_size); - this.compress_buffer = this.resources.getVoidBuffer(); - - // Start the three fibers which form the request handling logic. - scope writer_ = new Writer; - scope controller_ = new Controller; - scope periodic_refresh_ = new PeriodicRefresh; - - this.request_event_dispatcher.initialise(&this.resources.getVoidBuffer); - - // Note: we store refs to the scope instances in class fields as a - // convenience to be able to access them from each other (e.g. the - // writer needs to access the controller and vice-versa). It's normally - // not safe to store refs to scope instances outside of the scope, so we - // need to be careful to only use them while they are in scope. - this.writer = writer_; - this.controller = controller_; - this.periodic_refresh = periodic_refresh_; - scope ( exit ) - { - this.writer = null; - this.controller = null; - this.periodic_refresh = null; - } - - if ( this.start_suspended ) - this.writer.suspender.requestSuspension(); - - // Now the Writer fiber is instantiated, it's safe to register with the - // channel for updates. (When an update occurs, this.updated() will be - // called, which requires this.writer to be non-null.) - this.registerForUpdates(); - scope ( exit ) this.unregisterForUpdates(); - - this.periodic_refresh.fiber.start(); - this.controller.fiber.start(); - this.writer.fiber.start(); - this.request_event_dispatcher.eventLoop( - this.connection.event_dispatcher); - } - - /*************************************************************************** - - Called by the implementing class when notified by the storage engine of - an update. Adds the update to the update queue and wakes up the Writer - fiber, if it's waiting for something to send. If the update queue is - full, increments the counter of missed updates and (at most once per - second) wakes up the Writer fiber to notify the user of the overflow. - - Params: - update = update to push to the queue - - ***************************************************************************/ - - final protected void updated ( Update update ) - { - auto resume_code = NodeFiberResumeCode.PushedToQueue; - - auto pushed = this.update_queue.push(update); - if ( !pushed ) - { - this.update_queue_overflows++; - if ( !this.update_queue_overflows.notification_pending() ) - return; - - auto client_addr = this.connection.event_dispatcher.remote_address(); - log.warn("Mirror request on channel '{}', client {}:{} -- " ~ - "update queue overflowed, {} updates discarded", - this.channelName(), client_addr.address_bytes, client_addr.port, - this.update_queue_overflows.count_since_last_notification); - - resume_code = NodeFiberResumeCode.QueueOverflowNotification; - } - - if ( this.writer.suspended_waiting_for_events ) - this.request_event_dispatcher.signal( - this.connection.event_dispatcher, resume_code); - } - - /*************************************************************************** - - Called by the PeriodicRefresh fiber when a record is iterated. Adds the - record key to the refresh queue and wakes up the Writer fiber, if it's - waiting for something to send. - - Params: - key = key of record to push to the refresh queue - - ***************************************************************************/ - - private void refreshed ( hash_t key ) - { - verify(!this.refresh_queue.isFull()); - auto pushed = this.refresh_queue.push(key); - // The logic in PeriodicRefresh ensures that the refresh queue will - // never overflow. - verify(pushed); - - if ( this.writer.suspended_waiting_for_events ) - this.request_event_dispatcher.signal( - this.connection.event_dispatcher, - NodeFiberResumeCode.PushedToQueue); - } - - /*************************************************************************** - - Called by the implementing class when notified by the storage engine - that the mirrored channel has been removed. - - ***************************************************************************/ - - final protected void channelRemoved ( ) - { - if ( this.writer.suspended_waiting_for_events ) - this.request_event_dispatcher.signal( - this.connection.event_dispatcher, - NodeFiberResumeCode.ChannelRemoved); - } - - /*************************************************************************** - - Performs any logic needed to start mirroring the channel of the given - name. - - Params: - channel_name = channel to mirror - - Returns: - true if the channel may be used, false to abort the request - - ***************************************************************************/ - - abstract protected bool prepareChannel ( cstring channel_name ); - - /*************************************************************************** - - Returns: - the name of the channel being mirrored (for logging) - - ***************************************************************************/ - - abstract protected cstring channelName ( ); - - /*************************************************************************** - - Registers this request to receive updates on the channel. - - ***************************************************************************/ - - abstract protected void registerForUpdates ( ); - - /*************************************************************************** - - Unregisters this request from receiving updates on the channel. - - ***************************************************************************/ - - abstract protected void unregisterForUpdates ( ); - - /*************************************************************************** - - Gets the value of the record with the specified key, if it exists. - - Params: - key = key of record to get from storage - buf = buffer to write the value into - - Returns: - record value or null, if the record does not exist - - ***************************************************************************/ - - abstract protected void[] getRecordValue ( hash_t key, ref void[] buf ); - - /*************************************************************************** - - Called to begin a complete iteration over the channel being mirrored. - - ***************************************************************************/ - - abstract protected void startIteration ( ); - - /*************************************************************************** - - Gets the key of the next record in the iteration. - - Params: - hash_key = output value to receive the next key - - Returns: - true if hash_key was set or false if the iteration is finished - - ***************************************************************************/ - - abstract protected bool iterateNext ( out hash_t hash_key ); - - /*************************************************************************** - - Fiber which handles: - 1. Popping updates from the queue and forwarding them to the client. - 2. If the channel is removed, informing the client of this. - - ***************************************************************************/ - - private class Writer - { - import swarm.neo.util.DelayedSuspender; - - /// Fiber. - public MessageFiber fiber; - - /// Helper to allow other fibers to request writes to the client to be - /// suspended. - public DelayedSuspender suspender; - - /// Flag set to true when suspended waiting for updates. - private bool suspended_waiting_for_events; - - /// Flag set when the channel being mirrored has been removed. - private bool channel_removed; - - /// Loop counter used for yielding after every 10 records is sent. - private uint yield_counter; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - this.suspender = DelayedSuspender( - &this.outer.request_event_dispatcher, - this.outer.connection.event_dispatcher, - this.fiber, NodeFiberResumeCode.ResumeAfterSuspension); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - do - { - // Pop and send all updates currently in the queues. - this.sendFromQueues(); - - // Send a message notifying the user that the updates queue - // overflowed, if one is pending. - this.sendPendingOverflowNotification(); - - // Wait for record updates or channel removed. - if ( !this.channel_removed ) - this.waitForEvents(); - } - while ( !this.channel_removed ); - - // The request is now finished. Inform client of this and ignore any - // further incoming control messages. - this.outer.has_ended = true; - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.ChannelRemoved); - } - ); - - // Wait for ACK from client - this.outer.request_event_dispatcher.receive(this.fiber, - Message(MessageType.Ack)); - - // It's no longer valid to handle control messages. - this.outer.request_event_dispatcher.abort( - this.outer.controller.fiber); - - // Cancel the periodic refresher. - this.outer.request_event_dispatcher.abort( - this.outer.periodic_refresh.fiber); - } - - /*********************************************************************** - - Sends all queued updates / refreshes. - - ***********************************************************************/ - - private void sendFromQueues ( ) - { - while ( !this.channel_removed && - (this.outer.update_queue.length || this.outer.refresh_queue.length) ) - { - // If suspended, wait until resumed. - this.suspender.suspendIfRequested(); - - // Pop and send an update, if available. - if ( this.outer.update_queue.length ) - this.sendQueuedUpdate(); - - // Pop and send a refresh, if available. - if ( this.outer.refresh_queue.length ) - this.sendQueuedRefresh(); - - this.outer.request_event_dispatcher.periodicYield( - this.fiber, this.yield_counter, 10); - } - } - - /*********************************************************************** - - Sends the next element in the update queue. - - ***********************************************************************/ - - private void sendQueuedUpdate ( ) - { - verify(this.outer.update_queue.length > 0); - auto update = this.outer.update_queue.pop(); - with ( UpdateType ) switch ( update.type ) - { - case Change: - if ( this.outer.getRecordValue(update.key, - *this.outer.value_buffer) !is null ) - { - this.sendRecordChanged(update.key, - *this.outer.value_buffer); - } - // else: the record no longer exists; just ignore - break; - - case Deletion: - this.sendRecordDeleted(update.key); - break; - - default: - assert(false); - } - } - - /*********************************************************************** - - Adds the next element in the refresh queue to the output batch and - sends the batch either when it is full or the refresh queue is - empty. - - ***********************************************************************/ - - private void sendQueuedRefresh ( ) - { - verify(this.outer.refresh_queue.length > 0); - void sendBatch ( ) - { - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.RecordRefreshBatch); - this.outer.refresh_batch.getCompressed( - this.outer.resources.lzo, *this.outer.compress_buffer - ); - payload.addArray(*this.outer.compress_buffer); - } - ); - } - - auto key = this.outer.refresh_queue.pop(); - - if ( this.outer.getRecordValue(key, - *this.outer.value_buffer) !is null ) - { - this.outer.refresh_batch.add(key, *this.outer.value_buffer, - &sendBatch); - } - // else: the record no longer exists; just ignore - - if ( this.outer.refresh_queue.length == 0 ) - { - // Send any pending batch. - if ( this.outer.refresh_batch.get().length ) - { - sendBatch(); - this.outer.refresh_batch.clear(); - } - - // Let the refresh fiber know when the refresh queue is emptied. - this.outer.periodic_refresh.queueFlushed(); - } - } - - /*********************************************************************** - - Sends a message to the client, informing it that a record value has - changed. - - Params: - key = key of record which was updated - value = new value of record - - ***********************************************************************/ - - private void sendRecordChanged ( hash_t key, Const!(void)[] value ) - { - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.RecordChanged); - payload.add(key); - payload.addArray(value); - } - ); - } - - /*********************************************************************** - - Sends a message to the client, informing it that a record has been - removed. - - Params: - key = key of record which was removed - - ***********************************************************************/ - - private void sendRecordDeleted ( hash_t key ) - { - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.RecordDeleted); - payload.add(key); - } - ); - } - - /*********************************************************************** - - If update queue overflows have occurred and the update queue - overflow tracker says it's time to send a notification to the - client, do so. - - ***********************************************************************/ - - private void sendPendingOverflowNotification ( ) - { - if ( !this.outer.update_queue_overflows.notification_pending() ) - return; - - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.UpdateOverflow); - } - ); - - this.outer.update_queue_overflows.notification_sent(); - } - - /*********************************************************************** - - Suspends the fiber until a new update is pushed to the queue. - - ***********************************************************************/ - - private void waitForEvents ( ) - { - this.suspended_waiting_for_events = true; - scope ( exit ) - this.suspended_waiting_for_events = false; - - auto event = this.outer.request_event_dispatcher.nextEvent(this.fiber, - Signal(NodeFiberResumeCode.PushedToQueue), - Signal(NodeFiberResumeCode.QueueOverflowNotification), - Signal(NodeFiberResumeCode.ChannelRemoved)); - verify(event.active == event.active.signal, - "Unexpected event: waiting only for fiber resume code"); - - with ( NodeFiberResumeCode ) switch ( event.signal.code ) - { - case PushedToQueue: - case QueueOverflowNotification: - break; - case ChannelRemoved: - this.channel_removed = true; - break; - default: - assert(false); - } - } - } - - /*************************************************************************** - - Fiber which handles: - 1. Reading control messages from the client. - 2. Sending an ACK message back (the client relies on ACKs to - re-enable the user-facing controller). - 3. Controlling the other fibers, as appropriate, to carry out the - control messages from the client. - - ***************************************************************************/ - - private class Controller - { - /// Fiber. - public MessageFiber fiber; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - bool stopped; - do - { - // Receive message from client. - auto message = - this.outer.request_event_dispatcher.receive( - this.fiber, - Message(MessageType.Suspend), Message(MessageType.Resume), - Message(MessageType.Stop)); - - if ( this.outer.has_ended ) - continue; - - // Send ACK. The protocol guarantees that the client will not - // send any further messages until it has received the ACK. - this.outer.request_event_dispatcher.send(this.fiber, - ( RequestOnConnBase.EventDispatcher.Payload payload ) - { - payload.addCopy(MessageType.Ack); - } - ); - - with ( MessageType ) switch ( message.type ) - { - case Suspend: - this.outer.writer.suspender.requestSuspension(); - this.outer.periodic_refresh.suspender.requestSuspension(); - break; - case Resume: - this.outer.writer.suspender.resumeIfSuspended(); - this.outer.periodic_refresh.suspender.resumeIfSuspended(); - break; - case Stop: - stopped = true; - - // End both other fibers. The request is finished. - this.outer.request_event_dispatcher.abort( - this.outer.writer.fiber); - this.outer.request_event_dispatcher.abort( - this.outer.periodic_refresh.fiber); - break; - default: - assert(false); - } - } - while ( !stopped ); - } - } - - /*************************************************************************** - - Fiber which handles (if requested by the client): - 1. The initial refresh of all records in the channel. - 2. The periodic refresh of all records in the channel. - - ***************************************************************************/ - - private class PeriodicRefresh - { - import swarm.neo.util.DelayedSuspender; - - /// Fiber. - public MessageFiber fiber; - - /// Helper to allow other fibers to request writes to the client to be - /// suspended. - public DelayedSuspender suspender; - - /// Enum defining the states of the fiber. - private enum WaitingFor - { - /// Fiber running. - Nothing, - - /// Fiber suspended waiting for the periodic timer to fire. - Timer, - - /// Fiber suspended waiting for everything in the refresh queue to - /// be sent to the client. - QueueEmptied - } - - /// Fiber state. - private WaitingFor waiting_for; - - /*********************************************************************** - - Constructor. Gets a fiber from the shared resources. - - ***********************************************************************/ - - public this ( ) - { - this.fiber = this.outer.resources.getFiber(&this.fiberMethod); - this.suspender = DelayedSuspender( - &this.outer.request_event_dispatcher, - this.outer.connection.event_dispatcher, - this.fiber, NodeFiberResumeCode.ResumeAfterSuspension); - } - - /*********************************************************************** - - Fiber method. - - ***********************************************************************/ - - private void fiberMethod ( ) - { - // Do an initial refresh immediately, if requested. - if ( this.outer.initial_refresh ) - this.refresh(); - - // If no periodic refresh, exit the fiber. - if ( this.outer.periodic_refresh_s == 0 ) - return; - - // Set up periodic timer. - auto timer = this.outer.resources.getTimer( - this.outer.periodic_refresh_s, 0, - { - if ( this.waiting_for == WaitingFor.Timer ) - { - this.outer.request_event_dispatcher.signal( - this.outer.connection.event_dispatcher, - NodeFiberResumeCode.PeriodicRefresh); - } - // else: if the fiber is already running, a refresh - // cycle is in progress. Just let it finish. - } - ); - - timer.start(); - scope ( exit ) - timer.stop(); - - do - { - // Wait for the timer to fire. - this.waiting_for = WaitingFor.Timer; - this.outer.request_event_dispatcher.nextEvent( - this.fiber, - Signal(NodeFiberResumeCode.PeriodicRefresh)); - this.waiting_for = WaitingFor.Nothing; - - this.refresh(); - } - while ( true ); - } - - /*********************************************************************** - - Performs a single refresh cycle, iterating over all records in the - mirrored channel and pushing them to the update queue. - - ***********************************************************************/ - - private void refresh ( ) - { - // Iterate over the storage engine. - this.outer.startIteration(); - - hash_t key; - while ( this.outer.iterateNext(key) ) - { - this.suspender.suspendIfRequested(); - - // If the refresh queue is full, wait until it's been emptied. - if ( this.outer.refresh_queue.isFull() ) - { - this.waiting_for = WaitingFor.QueueEmptied; - this.outer.request_event_dispatcher.nextEvent( - this.fiber, - Signal(NodeFiberResumeCode.RefreshQueueEmptied)); - this.waiting_for = WaitingFor.Nothing; - } - - // Add the iterated key to the refresh queue and wake up the - // Writer fiber, if it's waiting for data to send. - this.outer.refreshed(key); - } - } - - /*********************************************************************** - - Called by the Writer fiber when the queue of refreshed records has - been emptied (i.e. sent to the client). Wakes up this fiber, if it's - waiting for this event. - - ***********************************************************************/ - - private void queueFlushed ( ) - { - if ( this.waiting_for == WaitingFor.QueueEmptied ) - { - this.outer.request_event_dispatcher.signal( - this.outer.connection.event_dispatcher, - NodeFiberResumeCode.RefreshQueueEmptied); - } - } - } -} - -/// Static module logger -static private Logger log; - -static this ( ) -{ - log = Log.lookup("dhtproto.node.neo.request.Mirror"); -} diff --git a/src/dhtproto/node/neo/request/Put.d b/src/dhtproto/node/neo/request/Put.d deleted file mode 100644 index 82366c48..00000000 --- a/src/dhtproto/node/neo/request/Put.d +++ /dev/null @@ -1,132 +0,0 @@ -/******************************************************************************* - - Put request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.Put; - -import ocean.core.VersionCheck; -import swarm.neo.node.IRequestHandler; - -/******************************************************************************* - - v0 Put request protocol. - -*******************************************************************************/ - -public abstract class PutProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import dhtproto.common.Put; - import dhtproto.node.neo.request.core.Mixins; - - import ocean.transition; - - /*************************************************************************** - - Mixin the initialiser and the connection and resources members. - - ***************************************************************************/ - - mixin IRequestHandlerRequestCore!(); - - /// Response status code to send to client. - private MessageType response; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - auto ed = this.connection.event_dispatcher(); - - auto channel = ed.message_parser.getArray!(char)(init_payload); - auto key = *ed.message_parser.getValue!(hash_t)(init_payload); - auto value = ed.message_parser.getArray!(void)(init_payload); - - // Check record key and write to channel, if ok. - if ( this.responsibleForKey(key) ) - { - this.response = this.put(channel, key, value) - ? MessageType.Put : MessageType.Error; - } - else - this.response = MessageType.WrongNode; - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - auto ed = this.connection.event_dispatcher(); - - // Send status code - ed.send( - ( ed.Payload payload ) - { - payload.add(response); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - } - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - abstract protected bool responsibleForKey ( hash_t key ); - - /*************************************************************************** - - Writes a single record to the storage engine. - - Params: - channel = channel to write to - key = key of record to write - value = record value to write - - Returns: - true if the record was written; false if an error occurred - - ***************************************************************************/ - - abstract protected bool put ( cstring channel, hash_t key, in void[] value ); -} diff --git a/src/dhtproto/node/neo/request/Remove.d b/src/dhtproto/node/neo/request/Remove.d deleted file mode 100644 index b5e8bde3..00000000 --- a/src/dhtproto/node/neo/request/Remove.d +++ /dev/null @@ -1,134 +0,0 @@ -/******************************************************************************* - - Remove request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.Remove; - -import ocean.core.VersionCheck; -import swarm.neo.node.IRequestHandler; - -/******************************************************************************* - - v0 Remove request protocol. - -*******************************************************************************/ - -public abstract class RemoveProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import dhtproto.common.Remove; - import dhtproto.node.neo.request.core.Mixins; - - import ocean.transition; - - /// Mixin the initialiser and the connection and resources members. - mixin IRequestHandlerRequestCore!(); - - /// Response status code to send to client. - private MessageType response; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - auto parser = this.connection.event_dispatcher.message_parser(); - - cstring channel; - hash_t key; - parser.parseBody(init_payload, channel, key); - - // Check record key and remove from channel, if ok. - if ( this.responsibleForKey(key) ) - { - bool removed; - if ( this.remove(channel, key, removed) ) - this.response = removed - ? MessageType.Removed : MessageType.NoRecord; - else - this.response = MessageType.Error; - } - else - this.response = MessageType.WrongNode; - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - auto ed = this.connection.event_dispatcher(); - - // Send response message - ed.send( - ( ed.Payload payload ) - { - payload.add(this.response); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - } - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - abstract protected bool responsibleForKey ( hash_t key ); - - /*************************************************************************** - - Removes a single record from the storage engine. - - Params: - channel = channel to remove from - key = key of record to remove - existed = out value, set to true if the record was present and - removed or false if the record was not present - - Returns: - true if the operation succeeded (the record was removed or did not - exist); false if an error occurred - - ***************************************************************************/ - - abstract protected bool remove ( cstring channel, hash_t key, - out bool existed ); -} diff --git a/src/dhtproto/node/neo/request/RemoveChannel.d b/src/dhtproto/node/neo/request/RemoveChannel.d deleted file mode 100644 index 7e83be1b..00000000 --- a/src/dhtproto/node/neo/request/RemoveChannel.d +++ /dev/null @@ -1,119 +0,0 @@ -/******************************************************************************* - - v0 RemoveChannel request protocol. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.RemoveChannel; - -import ocean.core.VersionCheck; -import ocean.util.log.Logger; -import swarm.neo.node.IRequestHandler; - -/// ditto -public abstract scope class RemoveChannelProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import dhtproto.common.RemoveChannel; - import dhtproto.node.neo.request.core.Mixins; - - import ocean.transition; - - /// Mixin the initialiser and the connection and resources members. - mixin IRequestHandlerRequestCore!(); - - /// Response to client. - private MessageType response; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - auto channel = this.connection.event_dispatcher.message_parser. - getArray!(char)(init_payload); - - if ( !this.clientPermitted(this.connection.getClientName()) ) - { - this.response = MessageType.NotPermitted; - return; - } - - this.response = this.removeChannel(channel) - ? MessageType.ChannelRemoved : MessageType.Error; - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - auto ed = this.connection.event_dispatcher(); - - // Send status code - ed.send( - ( ed.Payload payload ) - { - payload.add(this.response); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - } - - /*************************************************************************** - - Checks whether the specified client is permitted to remove channels. - - Params: - client_name = name of client requesting channel removal - - Returns: - true if the client is permitted to remove channels - - ***************************************************************************/ - - abstract protected bool clientPermitted ( cstring client_name ); - - /*************************************************************************** - - Removes the specified channel. - - Params: - channel_name = channel to remove - - Returns: - true if the operation succeeded (the channel was removed or did not - exist); false if an error occurred - - ***************************************************************************/ - - abstract protected bool removeChannel ( cstring channel_name ); -} diff --git a/src/dhtproto/node/neo/request/Update.d b/src/dhtproto/node/neo/request/Update.d deleted file mode 100644 index 4326181c..00000000 --- a/src/dhtproto/node/neo/request/Update.d +++ /dev/null @@ -1,318 +0,0 @@ -/******************************************************************************* - - Update request protocol. - - Copyright: - Copyright (c) 2018 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.Update; - -import ocean.core.VersionCheck; -import swarm.neo.node.IRequestHandler; - -/******************************************************************************* - - v0 Update request protocol. - -*******************************************************************************/ - -public abstract class UpdateProtocol_v0 : IRequestHandler -{ - import swarm.neo.node.RequestOnConn; - import dhtproto.common.Update; - import dhtproto.node.neo.request.core.Mixins; - - import ocean.transition; - import ocean.core.array.Mutation : copy; - import ocean.io.digest.Fnv1; - - /// Mixin the initialiser and the connection and resources members. - mixin IRequestHandlerRequestCore!(); - - /// Slice of acquired buffer into which the initial payload received from - /// the client is copied. - private Const!(void)[] init_payload; - - /*************************************************************************** - - Called by the connection handler immediately after the request code and - version have been parsed from a message received over the connection. - Allows the request handler to process the remainder of the incoming - message, before the connection handler sends the supported code back to - the client. - - Note: the initial payload is a slice of the connection's read buffer. - This means that when the request-on-conn fiber suspends, the contents of - the buffer (hence the slice) may change. It is thus *absolutely - essential* that this method does not suspend the fiber. (This precludes - all I/O operations on the connection.) - - Params: - init_payload = initial message payload read from the connection - - ***************************************************************************/ - - public void preSupportedCodeSent ( Const!(void)[] init_payload ) - { - auto buf = this.resources.getVoidBuffer(); - (*buf).copy(init_payload); - this.init_payload = *buf; - } - - /*************************************************************************** - - Called by the connection handler after the supported code has been sent - back to the client. - - ***************************************************************************/ - - public void postSupportedCodeSent ( ) - { - auto ed = this.connection.event_dispatcher(); - - auto message = - *ed.message_parser.getValue!(MessageType)(this.init_payload); - auto channel = ed.message_parser.getArray!(char)(this.init_payload); - auto key = *ed.message_parser.getValue!(hash_t)(this.init_payload); - - MessageType response; - - if ( this.responsibleForKey(key) ) - { - with ( MessageType ) switch ( message ) - { - // Normal sequence: read, wait for response, update. - case GetRecord: - response = this.getWaitUpdate(channel, key); - break; - - // Either an updated record transferred from another node or a - // new record that did not exist in the DHT before. Just put it. - case UpdateRecord: - response = this.update(channel, key, this.init_payload); - break; - - default: - ed.shutdownWithProtocolError("Unexpected message from client"); - } - } - else - response = MessageType.WrongNode; - - // Send the final message to the client. - ed.send( - ( ed.Payload payload ) - { - payload.add(response); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - } - - /*************************************************************************** - - Normal sequence: get, wait for response, update. - - Params: - channel = channel to update record in - key = key of record to update - - Returns: - message type code to return to the client - - ***************************************************************************/ - - private MessageType getWaitUpdate ( cstring channel, hash_t key ) - { - auto ed = this.connection.event_dispatcher(); - - // Get record value from storage and send it to the client. - bool exists; - auto success = this.get(channel, key, - ( Const!(void)[] value ) - { - exists = true; - - ed.send( - ( ed.Payload payload ) - { - payload.addCopy(MessageType.RecordValue); - payload.addArray(value); - } - ); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - ed.flush(); - } - ); - - // If getting the record failed, end the request. - if ( !success ) - return MessageType.Error; - - // If the record did not exist, end the request. - if ( !exists ) - return MessageType.NoRecord; - - // Wait for the client's response. - MessageType ret; - ed.receive( - ( in void[] payload ) - { - Const!(void)[] payload_slice = payload; - - auto message = *ed.message_parser.getValue!(MessageType)( - payload_slice); - with ( MessageType ) switch ( message ) - { - // Client wants to update the record. - case UpdateRecord: - ret = this.update(channel, key, payload_slice); - break; - - // Client has decided not to update the record. - case LeaveRecord: - ret = MessageType.Ok; - break; - - // Client has sent the updated record to another node. It - // can be removed here. - case RemoveRecord: - ret = this.remove(channel, key) - ? MessageType.Ok : MessageType.Error; - break; - - default: - ed.shutdownWithProtocolError("Unexpected message from client"); - } - } - ); - - return ret; - } - - /*************************************************************************** - - 1. Parses the hash of the original record value and the updated record - value from the message payload. - 2. Gets the record value from storage and hashes it. - 3. Compares the hash of the original value (provided by the client) with - the hash of the value in storage. - 4a. If the hashes differ, the record has been updated by another client. - The request is rejected. - 4b. If the hashes match, the new value is written to storage. - - Params: - channel = channel to update record in - key = key of record to update - payload = message payload received from client; contains the orignal - hash of the record value and the new record value - - Returns: - message type code to return to the client - - ***************************************************************************/ - - private MessageType update ( cstring channel, hash_t key, - Const!(void)[] payload ) - { - // Read original record value hash and updated value from client. - auto ed = this.connection.event_dispatcher(); - hash_t original_hash; - Const!(void)[] new_value; - ed.message_parser.parseBody(payload, original_hash, new_value); - - // Get the currently stored record value and hash it. - bool exists; - hash_t stored_hash; - auto ok = this.get(channel, key, - ( Const!(void)[] value ) - { - exists = true; - stored_hash = Fnv1a(value); - } - ); - if ( !ok ) - return MessageType.Error; - - // Check for update conflicts. - if ( exists && stored_hash != original_hash ) - return MessageType.UpdateConflict; - - // Otherwise, write the updated record value to storage. - return this.put(channel, key, new_value) - ? MessageType.Ok : MessageType.Error; - } - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - abstract protected bool responsibleForKey ( hash_t key ); - - /*************************************************************************** - - Reads a single record from the storage engine. Note that the - implementing class must guarantee that no fiber switch can occur during - this method. - - Params: - channel = channel to read from - key = key of record to read - dg = called with the value of the record, if it exists - - Returns: - true if the operation succeeded (the record was fetched or did not - exist); false if an error occurred - - ***************************************************************************/ - - abstract protected bool get ( cstring channel, hash_t key, - void delegate ( Const!(void)[] value ) dg ); - - /*************************************************************************** - - Writes a single record to the storage engine. - - Params: - channel = channel to write to - key = key of record to write - value = record value to write - - Returns: - true if the record was written; false if an error occurred - - ***************************************************************************/ - - abstract protected bool put ( cstring channel, hash_t key, in void[] value ); - - /*************************************************************************** - - Removes a single record from the storage engine. - - Params: - channel = channel to remove to - key = key of record to remove - - Returns: - true if the record was removed; false if an error occurred - - ***************************************************************************/ - - abstract protected bool remove ( cstring channel, hash_t key ); -} diff --git a/src/dhtproto/node/neo/request/core/IRequestResources.d b/src/dhtproto/node/neo/request/core/IRequestResources.d deleted file mode 100644 index e59e73fc..00000000 --- a/src/dhtproto/node/neo/request/core/IRequestResources.d +++ /dev/null @@ -1,101 +0,0 @@ -/******************************************************************************* - - Request resource acquirer. - - Via an instance of this interface, a request is able to acquire different - types of resource which it requires during its lifetime. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.core.IRequestResources; - -public interface IRequestResources -{ - import swarm.neo.request.RequestEventDispatcher; - import swarm.neo.util.MessageFiber; - import swarm.util.RecordBatcher; - import ocean.io.compress.Lzo; - - /*************************************************************************** - - Returns: - a shared LZO instance - - ***************************************************************************/ - - Lzo lzo ( ); - - - /*************************************************************************** - - Returns: - a pointer to a new chunk of memory (a void[]) to use during the - request's lifetime - - ***************************************************************************/ - - void[]* getVoidBuffer ( ); - - /*************************************************************************** - - Gets a fiber to use during the request's lifetime and assigns the - provided delegate as its entry point. - - Params: - fiber_method = entry point to assign to acquired fiber - - Returns: - a new MessageFiber acquired to use during the request's lifetime - - ***************************************************************************/ - - MessageFiber getFiber ( void delegate ( ) fiber_method ); - - /*************************************************************************** - - Gets a record batcher to use during the request's lifetime. - - Returns: - a new record batcher acquired to use during the request's lifetime - - ***************************************************************************/ - - RecordBatcher getRecordBatcher ( ); - - /*************************************************************************** - - Gets a periodically firing timer. - - Params: - period_s = seconds part of timer period - period_ms = milliseconds part of timer period - timer_dg = delegate to call when timer fires - - Returns: - ITimer interface to a timer to use during the request's lifetime - - ***************************************************************************/ - - ITimer getTimer ( uint period_s, uint period_ms, void delegate ( ) timer_dg ); - - /*************************************************************************** - - Interface to a timer to be used during the request's lifetime. - - ***************************************************************************/ - - interface ITimer - { - /// Starts the timer. - void start ( ); - - /// Stops the timer. - void stop ( ); - } -} diff --git a/src/dhtproto/node/neo/request/core/Mixins.d b/src/dhtproto/node/neo/request/core/Mixins.d deleted file mode 100644 index 369e7eff..00000000 --- a/src/dhtproto/node/neo/request/core/Mixins.d +++ /dev/null @@ -1,82 +0,0 @@ -/******************************************************************************* - - Request protocol mixins. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhtproto.node.neo.request.core.Mixins; - -/******************************************************************************* - - Request core mixin. - -*******************************************************************************/ - -public template RequestCore ( ) -{ - import dhtproto.node.neo.request.core.IRequestResources; - - /*************************************************************************** - - Shared resources getter instance. - - ***************************************************************************/ - - protected IRequestResources resources; - - /*************************************************************************** - - Constructor. - - Params: - shared_resources = DHT request resources getter - - ***************************************************************************/ - - public this ( IRequestResources resources ) - { - this.resources = resources; - } -} - -/******************************************************************************* - - IRequestHandler-based request core mixin. - -*******************************************************************************/ - -public template IRequestHandlerRequestCore ( ) -{ - import ocean.core.Verify; - import swarm.neo.node.RequestOnConn; - import dhtproto.node.neo.request.core.IRequestResources; - - /// Request-on-conn of this request handler. - private RequestOnConn connection; - - /// Acquired resources of this request. - protected IRequestResources resources; - - /*************************************************************************** - - Passes the request-on-conn and request resource acquirer to the handler. - - Params: - connection = request-on-conn in which the request handler is called - resources = request resources acquirer - - ***************************************************************************/ - - public void initialise ( RequestOnConn connection, Object resources_object ) - { - this.connection = connection; - this.resources = cast(IRequestResources)resources_object; - verify(this.resources !is null); - } -} diff --git a/src/dhttest/DhtClient.d b/src/dhttest/DhtClient.d index 3ca542ba..f68a5d59 100644 --- a/src/dhttest/DhtClient.d +++ b/src/dhttest/DhtClient.d @@ -32,7 +32,6 @@ import ocean.util.log.Logger; class DhtClient { import ocean.core.Enforce; - import ocean.core.Verify; import ocean.core.Array : copy; import ocean.task.Task; import ocean.task.Scheduler; @@ -79,7 +78,7 @@ class DhtClient public this ( ) { this.task = Task.getThis(); - verify(this.task !is null); + assert(this.task !is null); } /*********************************************************************** @@ -204,7 +203,7 @@ class DhtClient this.swarm_client.addNode("127.0.0.1".dup, port); auto task = Task.getThis(); - verify(task !is null); + assert(task !is null); bool finished; @@ -629,7 +628,7 @@ class DhtClient public this ( ) { this.task = Task.getThis(); - verify(this.task !is null); + assert(this.task !is null); } /*********************************************************************** diff --git a/src/dhttest/DhtTestCase.d b/src/dhttest/DhtTestCase.d index b2eb416c..e5f79670 100644 --- a/src/dhttest/DhtTestCase.d +++ b/src/dhttest/DhtTestCase.d @@ -21,7 +21,6 @@ module dhttest.DhtTestCase; *******************************************************************************/ import ocean.transition; -import ocean.core.VersionCheck; import turtle.TestCase; @@ -33,6 +32,7 @@ import turtle.TestCase; abstract class DhtTestCase : TestCase { + import ocean.core.Test; // makes `test` available in derivatives import dhttest.DhtClient; /*************************************************************************** @@ -99,223 +99,3 @@ abstract class DhtTestCase : TestCase this.dht.removeChannel(this.test_channel); } } - -/******************************************************************************* - - Neo test case base. Actual tests are located in `dhttest.cases.neo`. - -*******************************************************************************/ - -abstract class NeoDhtTestCase : TestCase -{ - import ocean.core.Enforce; - import ocean.core.Verify; - import ocean.task.Scheduler; - import ocean.task.Task; - import ocean.util.log.Logger; - import dhtproto.client.DhtClient; - import Legacy = dhttest.DhtClient; - import swarm.neo.authentication.HmacDef: Key; - - /*************************************************************************** - - Reference to common test case logger instance - - ***************************************************************************/ - - private Logger log; - - /*************************************************************************** - - Number of records handled in bulk tests. (This value is used by all test - cases which test reading/writing/removing a large number of records from - the DHT. Small sanity check test cases do not use it.) - - ***************************************************************************/ - - public static size_t bulk_test_record_count = 10_000; - - /*************************************************************************** - - DHT client to use in tests. Provides blocking fiber API. - - ***************************************************************************/ - - protected DhtClient dht; - - /*************************************************************************** - - Standard name for a channel with test data which will be cleaned - automatically after the test case ends. - - ***************************************************************************/ - - protected istring test_channel; - - /*************************************************************************** - - Task being suspended during connection. - - ***************************************************************************/ - - private Task connect_task; - - /*************************************************************************** - - Flag indicating that a connection error occurred. - - ***************************************************************************/ - - private bool connect_error; - - /*************************************************************************** - - Constructor - - ***************************************************************************/ - - this ( ) - { - this.log = Log.lookup("dhttest"); - this.test_channel = "test_channel"; - } - - /*************************************************************************** - - Creates new DHT client for a test case and proceeds with handshake so - that client instance will be ready to work by the time `run` method - is called. - - ***************************************************************************/ - - override public void prepare ( ) - { - cstring auth_name = "admin"; - auto auth_key = Key.init; - const max_connections = 2; - this.dht = new DhtClient(theScheduler.epoll, auth_name, - auth_key.content, &this.neoConnectionNotifier, max_connections); - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - this.dht.neo.enableSocketNoDelay(); - - this.connect(10000); - } - - /*************************************************************************** - - Shuts down the client's connections, to avoid the connection notifier - being called (when the node shuts down) after the client has been - deleted. - - Also deletes test channel each time a test case finishes, to avoid using - some state by accident between tests. Note that for the moment, this - uses the legacy protocol, as no neo RemoveChannel request exists. - - ***************************************************************************/ - - override public void cleanup ( ) - { - this.dht.blocking.removeChannel(this.test_channel); - - this.log.info("Shutting down client -- node connection error expected"); - this.dht.neo.shutdown(); - } - - /*************************************************************************** - - Connects to the neo ports of the DHT node. The test Task is suspended - until connection has succeeded. - - Params: - legacy_port = DHT node legacy port. (The neo port is +100, by - convention, currently. See - dhtnode.node.DhtHashRange.newNodeAdded().) - - ***************************************************************************/ - - public void connect ( ushort legacy_port ) - { - this.connect_task = Task.getThis(); - verify(this.connect_task !is null); - - this.dht.neo.addNode("127.0.0.1".dup, cast(ushort)(legacy_port + 100)); - - scope stats = this.dht.neo.new DhtStats; - while ( stats.num_nodes_known_hash_range < stats.num_registered_nodes ) - { - enforce(!this.connect_error, "Test DHT neo connection error"); - this.connect_task.suspend(); - } - - this.connect_task = null; - } - - /*************************************************************************** - - Connects to the legacy port of the DHT node. The test Task is suspended - until connection has succeeded. - - Params: - legacy_port = DHT node legacy port - - ***************************************************************************/ - - public void legacyConnect ( ushort legacy_port ) - { - this.dht.addNode("127.0.0.1".dup, legacy_port); - - auto task = Task.getThis(); - verify(task !is null); - - bool ok, finished; - - void handshake_cb ( DhtClient.RequestContext, bool ok_ ) - { - finished = true; - ok = ok_; - if ( task.suspended() ) - task.resume(); - } - - this.dht.nodeHandshake(&handshake_cb, null); - if ( !finished ) - task.suspend(); - - enforce(ok, "Legacy DHT handshake failed"); - } - - /*************************************************************************** - - Neo connection notifier. - - Params: - info = notification info - - ***************************************************************************/ - - private void neoConnectionNotifier ( DhtClient.Neo.DhtConnNotification info ) - { - with ( info.Active ) switch ( info.active ) - { - case connection_error: - this.connect_error = true; - this.log.warn("Neo connection error on {}:{}: {} @ {}:{}", - info.connection_error.node_addr.address_bytes, - info.connection_error.node_addr.port, - getMsg(info.connection_error.e), - info.connection_error.e.file, info.connection_error.e.line); - break; - - case connected: - case hash_range_queried: - break; - - default: assert(false); - } - - if ( this.connect_task !is null ) - this.connect_task.resume(); - } -} - diff --git a/src/dhttest/TestRunner.d b/src/dhttest/TestRunner.d index be82c498..429e027f 100644 --- a/src/dhttest/TestRunner.d +++ b/src/dhttest/TestRunner.d @@ -33,16 +33,7 @@ import dhttest.cases.OrderedRemove; import dhttest.cases.UnorderedRemove; import dhttest.cases.BatchListen; -import dhttest.cases.neo.Basic; -import dhttest.cases.neo.Put; -import dhttest.cases.neo.OrderedPut; -import dhttest.cases.neo.UnorderedPut; -import dhttest.cases.neo.Mirror; -import dhttest.cases.neo.GetAll; -import dhttest.cases.neo.GetChannels; -import dhttest.cases.neo.Remove; -import dhttest.cases.neo.RemoveChannel; -import dhttest.cases.neo.Update; +import dhttest.cases.neo.Dummy; /******************************************************************************* diff --git a/src/dhttest/cases/Basic.d b/src/dhttest/cases/Basic.d index 3be78d76..a43c9470 100644 --- a/src/dhttest/cases/Basic.d +++ b/src/dhttest/cases/Basic.d @@ -20,7 +20,7 @@ module dhttest.cases.Basic; import ocean.core.Test; import ocean.transition; -import ocean.core.Test; + import dhttest.DhtTestCase; const PRIORITY = 100; diff --git a/src/dhttest/cases/BatchListen.d b/src/dhttest/cases/BatchListen.d index f1b09cf4..b278d44c 100644 --- a/src/dhttest/cases/BatchListen.d +++ b/src/dhttest/cases/BatchListen.d @@ -18,7 +18,6 @@ module dhttest.cases.BatchListen; *******************************************************************************/ -import ocean.core.Test; import dhttest.DhtTestCase; /******************************************************************************* diff --git a/src/dhttest/cases/neo/Basic.d b/src/dhttest/cases/neo/Basic.d deleted file mode 100644 index 5c6b2e96..00000000 --- a/src/dhttest/cases/neo/Basic.d +++ /dev/null @@ -1,118 +0,0 @@ -/******************************************************************************* - - Contains set of very simple test cases for basic DHT commands - - Copyright: - Copyright (c) 2015-2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.Basic; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import ocean.transition; -import ocean.core.Test; - -import dhttest.DhtTestCase : NeoDhtTestCase; - -/******************************************************************************* - - Checks that data stored in DHT can be retrievied back with neo Put/Get. - -*******************************************************************************/ - -class PutGet : NeoDhtTestCase -{ - override public Description description ( ) - { - Description desc; - desc.priority = 100; - desc.name = "Neo Put followed by neo Get"; - return desc; - } - - public override void run ( ) - { - cstring[hash_t] payloads = [ 0xA:"foo", 0xB:"bar", 0xC:"doo" ]; - - foreach (key, val; payloads) - { - auto res = this.dht.blocking.put(this.test_channel, key, val); - test(res.succeeded); - } - - void[] buf; - foreach (key, val; payloads) - { - auto res = this.dht.blocking.get(this.test_channel, key, buf); - test(res.succeeded); - test!("==")(val, cast(mstring)res.value); - } - } -} - -/******************************************************************************* - - Checks that data stored in DHT can be removed with neo Remove. - -*******************************************************************************/ - -class PutRemoveGet : NeoDhtTestCase -{ - override public Description description ( ) - { - Description desc; - desc.priority = 100; - desc.name = "Neo Put followed by neo Remove and neo Get"; - return desc; - } - - public override void run ( ) - { - auto put_res = this.dht.blocking.put(this.test_channel, 0, "test"); - test(put_res.succeeded); - - auto rem_res = this.dht.blocking.remove(this.test_channel, 0); - test(rem_res.succeeded); - test(rem_res.existed); - - void[] buf; - auto get_res = this.dht.blocking.get(this.test_channel, 0, buf); - test(get_res.succeeded); - test!("is")(get_res.value, null); - } -} - -/******************************************************************************* - - Checks that data stored in other test cases is not persistent - and that - it doesn't magically appear from nowhere. - -*******************************************************************************/ - -class GetNonExistent : NeoDhtTestCase -{ - override public Description description ( ) - { - Description desc; - desc.priority = 100; - desc.name = "Neo Get for non-existent entry"; - return desc; - } - - public override void run ( ) - { - void[] buf; - auto res = this.dht.blocking.get(this.test_channel, 0xA, buf); - test(res.succeeded); - test!("is")(res.value, null); - } -} diff --git a/src/dhttest/cases/neo/Dummy.d b/src/dhttest/cases/neo/Dummy.d new file mode 100644 index 00000000..892a8c71 --- /dev/null +++ b/src/dhttest/cases/neo/Dummy.d @@ -0,0 +1,36 @@ +/******************************************************************************* + + Dummy neo test case so that the test runner has at least one test case to + run. + + Otherwise, the test runner exits with a failure. When real neo test are + added, this dummy test case can be removed. + + Copyright: + Copyright (c) 2017 dunnhumby Germany GmbH. All rights reserved. + + License: + Boost Software License Version 1.0. See LICENSE.txt for details. + +*******************************************************************************/ + +module dhttest.cases.neo.Dummy; + +import ocean.transition; + +import dhttest.DhtTestCase; + +/// ditto +class Dummy : DhtTestCase +{ + override public Description description ( ) + { + Description desc; + desc.name = "Dummy test case"; + return desc; + } + + public override void run ( ) + { + } +} diff --git a/src/dhttest/cases/neo/GetAll.d b/src/dhttest/cases/neo/GetAll.d deleted file mode 100644 index abc9332b..00000000 --- a/src/dhttest/cases/neo/GetAll.d +++ /dev/null @@ -1,371 +0,0 @@ -/******************************************************************************* - - Test cases for the neo GetAll request. - - Note that the basic behaviour of the GetAll request is tested in the - NeoVerifier, where it's used to check the results of Put tests. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.GetAll; - -import ocean.transition; -import ocean.core.Test; -import ocean.core.Verify; -import ocean.math.random.Random; -import dhttest.DhtTestCase : NeoDhtTestCase; -import dhtproto.client.DhtClient; - -/******************************************************************************* - - Test case for Task-blocking GetAll. - -*******************************************************************************/ - -public class GetAllBlocking : NeoDhtTestCase -{ - import ocean.task.Task; - import ocean.io.select.client.TimerEvent; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Puts followed by Task-blocking neo GetAll"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - const num_records = 1000; - putRecords(this.dht, this.test_channel, num_records); - - void[] buf; - void[][hash_t] received; - bool duplicate; - foreach ( k, v; this.dht.blocking.getAll(this.test_channel, buf) ) - { - if ( k in received ) - duplicate = true; - received[k] = v.dup; - } - - test!("==")(received.length, num_records); - test(!duplicate); - for ( hash_t key = 0; key < num_records; key++ ) - test!("in")(key, received); - } -} - -/******************************************************************************* - - Test case which starts a GetAll then suspends and resumes it. - -*******************************************************************************/ - -public class GetAllSuspend : NeoDhtTestCase -{ - import ocean.task.Task; - import ocean.task.Scheduler; - import ocean.io.select.client.TimerEvent; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Puts followed by neo GetAll and suspend/resume"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - const num_records = 1000; - putRecords(this.dht, this.test_channel, num_records); - - auto getall = GetAll(this.dht); - - // Timer which resumes the request. - scope resume_timer = new TimerEvent( - { - getall.resume(); - return false; - } - ); - - uint suspend_count; - - getall.start(this.test_channel, - ( DhtClient.Neo.GetAll.Notification info, - Const!(DhtClient.Neo.GetAll.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case started: - // As soon as the GetAll is up, suspend - getall.suspend(); - break; - - case suspended: - suspend_count++; - - // When suspended, register the timer to resume in a bit. - resume_timer.set(0, 500, 0, 0); - theScheduler.epoll.register(resume_timer); - break; - - case resumed: - break; - - case finished: - task.resume(); - break; - - default: - break; - } - } - ); - - scope ( exit ) theScheduler.epoll.unregister(resume_timer); - - task.suspend(); - test!("==")(getall.received_keys.length, num_records); - test(!getall.duplicate); - test!("==")(suspend_count, 1); - } -} - -/******************************************************************************* - - Test case which starts a GetAll then kills all connections and checks that - the GetAll request successfully reconnects and continues. - -*******************************************************************************/ - -public class GetAllConnError : NeoDhtTestCase -{ - import turtle.env.ControlSocket : sendCommand; - import ocean.task.Task; - import ocean.io.select.client.TimerEvent; - import swarm.neo.client.requests.NotificationFormatter; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Puts followed by neo GetAll with connection drop"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - const num_records = 1000; - putRecords(this.dht, this.test_channel, num_records); - - uint disconnection_count; - auto getall = GetAll(this.dht); - getall.start(this.test_channel, - ( DhtClient.Neo.GetAll.Notification info, - Const!(DhtClient.Neo.GetAll.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case received: - if ( getall.received_keys.length == num_records / 2 ) - this.dht.neo.reconnect(); - break; - - case node_disconnected: - disconnection_count++; - break; - - case finished: - task.resume(); - break; - - default: - break; - } - } - ); - - task.suspend(); - test!("==")(getall.received_keys.length, num_records); - test(!getall.duplicate); - test!("==")(disconnection_count, 1); - } -} - -/******************************************************************************* - - Helper function to write some records to the specified channel. Random 8K - record values are written. - - Params: - dht = DHT client instance to perform Puts with - channel = channel to write to - num_records = the number of records to write. Records are written with - keys from 0..num_records - -*******************************************************************************/ - -private void putRecords ( DhtClient dht, cstring channel, size_t num_records ) -{ - auto rand = new Random; - - ubyte[] val; - val.length = 8 * 1024; - for ( hash_t key = 0; key < num_records; key++ ) - { - foreach ( ref b; val ) - b = rand.uniform!(ubyte)(); - - auto res = dht.blocking.put(channel, key, val); - test(res.succeeded); - } -} - -/******************************************************************************* - - Helper for performing a GetAll request and checking the results. Reduces the - amount of boiler-plate in each test case. - -*******************************************************************************/ - -private struct GetAll -{ - import swarm.neo.protocol.Message : RequestId; - - /// DHT client to start GetAll request with. - private DhtClient dht; - - /// GetAll request id. - public RequestId id; - - /// Set of keys received. - public bool[hash_t] received_keys; - - /// Flag set when a key is received twice by the request (an error). - public bool duplicate; - - /// User-provided GetAll notifier, passed to start(). - private DhtClient.Neo.GetAll.Notifier user_notifier; - - /*************************************************************************** - - Starts the GetAll request on the specified channel with the specified - settings. - - Params: - channel = channel to getall - user_notifier = GetAll notifier (must be non-null) - - ***************************************************************************/ - - public void start ( cstring channel, - DhtClient.Neo.GetAll.Notifier user_notifier ) - out - { - assert(this.user_notifier !is null); - } - body - { - verify(this.user_notifier is null); - this.user_notifier = user_notifier; - this.id = this.dht.neo.getAll(channel, &this.counterNotifier); - } - - /*************************************************************************** - - Suspends the GetAll request, using the controller to send a message to - the node. When the request is suspended, the GetAll notifier will be - called. - - ***************************************************************************/ - - public void suspend ( ) - { - verify(this.id != this.id.init); - this.dht.neo.control(this.id, - ( DhtClient.Neo.GetAll.IController getall ) - { - getall.suspend(); - } - ); - } - - /*************************************************************************** - - Resumes the GetAll request, using the controller to send a message to - the node. When the request is resumed, the GetAll notifier will be - called. - - ***************************************************************************/ - - public void resume ( ) - { - verify(this.id != this.id.init); - this.dht.neo.control(this.id, - ( DhtClient.Neo.GetAll.IController getall ) - { - getall.resume(); - } - ); - } - - /*************************************************************************** - - Stops the GetAll request, using the controller to send a message to the - node. When the request has finished, the GetAll notifier will be called. - - ***************************************************************************/ - - public void stop ( ) - { - verify(this.id != this.id.init); - this.dht.neo.control(this.id, - ( DhtClient.Neo.GetAll.IController getall ) - { - getall.stop(); - } - ); - } - - /*************************************************************************** - - Internal GetAll notifier. Updates the counters and calls the user's - notifier. - - Params: - info = GetAll notification - args = request arguments - - ***************************************************************************/ - - private void counterNotifier ( DhtClient.Neo.GetAll.Notification info, - Const!(DhtClient.Neo.GetAll.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case received: - if ( info.received.key in this.received_keys ) - this.duplicate = true; - this.received_keys[info.received.key] = true; - break; - - default: - break; - } - - this.user_notifier(info, args); - } -} diff --git a/src/dhttest/cases/neo/GetChannels.d b/src/dhttest/cases/neo/GetChannels.d deleted file mode 100644 index 586ead79..00000000 --- a/src/dhttest/cases/neo/GetChannels.d +++ /dev/null @@ -1,121 +0,0 @@ -/******************************************************************************* - - Test cases for the neo GetChannels request. - - Note that the basic behaviour of the GetChannels request is tested in the - NeoVerifier, where it's used to check the results of Put tests. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.GetChannels; - -import ocean.transition; -import ocean.core.Test; -import dhttest.DhtTestCase : NeoDhtTestCase; -import dhtproto.client.DhtClient; - -/******************************************************************************* - - Basic test case for GetChannels. - -*******************************************************************************/ - -public class GetChannels : NeoDhtTestCase -{ - import ocean.task.Task; - import ocean.core.array.Search : contains; - - private const channels = ["channel1", "channel2", "channel3", "channel4"]; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Puts followed by neo GetChannels"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - foreach ( channel; this.channels ) - { - auto res = this.dht.blocking.put(channel, 0, "whatever"); - test(res.succeeded); - } - - mstring buf; - mstring[] received_channels; - foreach ( channel; dht.blocking.getChannels(buf) ) - received_channels ~= channel.dup; - - test!("==")(this.channels.length, received_channels.length); - foreach ( channel; this.channels ) - test(received_channels.contains(channel)); - } - - override public void cleanup ( ) - { - foreach ( channel; this.channels ) - this.dht.blocking.removeChannel(channel); - - super.cleanup(); - } -} - -/******************************************************************************* - - Test case for GetChannels + RemoveChannel. - -*******************************************************************************/ - -public class GetChannelsRemove : NeoDhtTestCase -{ - import ocean.task.Task; - import ocean.core.array.Search : contains; - - private const channels = ["channel1", "channel2", "channel3", "channel4"]; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Puts followed by neo RemoveChannel then neo GetChannels"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - foreach ( channel; this.channels ) - { - auto res = this.dht.blocking.put(channel, 0, "whatever"); - test(res.succeeded); - } - - this.dht.blocking.removeChannel(this.channels[0]); - - mstring buf; - mstring[] received_channels; - foreach ( channel; dht.blocking.getChannels(buf) ) - received_channels ~= channel.dup; - - test!("==")(this.channels.length - 1, received_channels.length); - foreach ( channel; this.channels[1..$] ) - test(received_channels.contains(channel)); - } - - override public void cleanup ( ) - { - foreach ( channel; this.channels ) - this.dht.blocking.removeChannel(channel); - - super.cleanup(); - } -} diff --git a/src/dhttest/cases/neo/Mirror.d b/src/dhttest/cases/neo/Mirror.d deleted file mode 100644 index aa3be834..00000000 --- a/src/dhttest/cases/neo/Mirror.d +++ /dev/null @@ -1,737 +0,0 @@ -/******************************************************************************* - - Test cases for the neo Mirror request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.Mirror; - -import ocean.transition; -import ocean.core.Test; -import ocean.core.Verify; -import dhttest.DhtTestCase : NeoDhtTestCase; -import dhtproto.client.DhtClient; - -/******************************************************************************* - - Test case which writes some records to the test channel, starts an updating - mirror, then removes some records. - -*******************************************************************************/ - -public class MirrorRemove : NeoDhtTestCase -{ - import ocean.task.Task; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Put followed by neo Mirror + Removes"; - return desc; - } - - public override void run ( ) - { - this.legacyConnect(10000); - - auto task = Task.getThis(); - - putRecords(this.dht, this.test_channel, 100); - - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = false; - mirror_settings.periodic_refresh_s = 0; - - void delNotifier ( DhtClient.RequestNotification info ) { } - - const end_after_deletions = 5; - auto mirror = Mirror(this.dht); - mirror.start(mirror_settings, this.test_channel, - ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case started: - // As soon as the mirror is up, delete some records - for ( hash_t del_key = 0; del_key < end_after_deletions; - del_key++ ) - { - this.dht.assign( - this.dht.remove(this.test_channel, del_key, - &delNotifier)); - } - break; - - case deleted: - // Stop mirroring after the records have been deleted - if ( mirror.deleted_count == end_after_deletions ) - mirror.stop(); - break; - - case stopped: - task.resume(); - break; - - default: - break; - } - } - ); - - task.suspend(); - test!("==")(mirror.updated_count, 0); - test!("==")(mirror.refreshed_count, 0); - test!("==")(mirror.deleted_count, end_after_deletions); - } -} - -/******************************************************************************* - - Test case which starts an updating mirror then writes some records. - -*******************************************************************************/ - -public class MirrorUpdate : NeoDhtTestCase -{ - import ocean.task.Task; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Mirror followed by neo Puts"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = false; - mirror_settings.periodic_refresh_s = 0; - - void putNotifier ( DhtClient.Neo.Put.Notification info, - Const!(DhtClient.Neo.Put.Args) args ) { } - - const num_written = 100; - auto mirror = Mirror(this.dht); - mirror.start(mirror_settings, this.test_channel, - ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case started: - // As soon as the mirror is up, put some records - putRecords(this.dht, this.test_channel, num_written, - &putNotifier); - break; - - case updated: - // Stop mirroring after the updates have been received - if ( mirror.updated_count == num_written ) - mirror.stop(); - break; - - case stopped: - task.resume(); - break; - - default: - break; - } - } - ); - - task.suspend(); - test!("==")(mirror.updated_count, num_written); - test!("==")(mirror.refreshed_count, 0); - test!("==")(mirror.deleted_count, 0); - } -} - -/******************************************************************************* - - Test case which writes some records, starts a Mirror, waits for the first - refresh cycle to complete (returning all records to the client), then kills - all connections and checks that the Mirror request successfully reconnects - and performs a second refresh cycle. - -*******************************************************************************/ - -public class MirrorConnError : NeoDhtTestCase -{ - import ocean.task.Task; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Puts followed by neo Mirror with connection drop"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - const num_written = 100; - putRecords(this.dht, this.test_channel, num_written); - - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = true; - mirror_settings.periodic_refresh_s = 0; - - uint disconnection_count; - auto mirror = Mirror(this.dht); - mirror.start(mirror_settings, this.test_channel, - ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case started: - break; - - case refreshed: - // After the first refresh is complete, tell the client - // to drop and re-establish all connections. - if ( mirror.refreshed_count == num_written ) - this.dht.neo.reconnect(); - - // After the second refresh is complete, stop the - // request. - if ( mirror.refreshed_count == num_written * 2 ) - mirror.stop(); - break; - - case node_disconnected: - disconnection_count++; - break; - - case stopped: - task.resume(); - break; - - default: - break; - } - } - ); - - task.suspend(); - test!("==")(mirror.updated_count, 0); - test!("==")(mirror.refreshed_count, num_written * 2); - test!("==")(mirror.deleted_count, 0); - test!("==")(disconnection_count, 1); - } -} - -/******************************************************************************* - - Test case which starts an updating mirror then writes some records while - periodically suspending and resuming the request. - -*******************************************************************************/ - -public class MirrorSuspend : NeoDhtTestCase -{ - import ocean.task.Task; - import ocean.task.Scheduler; - import ocean.io.select.client.TimerEvent; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Mirror followed by neo Puts and suspend/resume"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = false; - mirror_settings.periodic_refresh_s = 0; - - const end_after_updates = 1_000; - size_t num_written; - auto mirror = Mirror(this.dht); - - // Timer which resumes the request. - scope resume_timer = new TimerEvent( - { - mirror.resume(); - return false; - } - ); - - // Timer which puts every 1ms, until `end_after_updates` records have - // been put. - hash_t key; - ubyte[] value; - value.length = 16 * 1024; - scope put_timer = new TimerEvent( - { - this.dht.neo.put(this.test_channel, key++, value, null); - return ++num_written < end_after_updates; - } - ); - put_timer.set(0, 1, 0, 1); - - bool[hash_t] received_keys; - uint suspend_count; - bool duplicate; - - mirror.start(mirror_settings, this.test_channel, - ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case started: - // As soon as the mirror is up, start the Put timer and - // suspend. - theScheduler.epoll.register(put_timer); - mirror.suspend(); - break; - - case updated: - if ( info.updated.key in received_keys ) - duplicate = true; - - received_keys[info.updated.key] = true; - // Stop mirroring after the updates have been received - if ( mirror.updated_count == end_after_updates ) - mirror.stop(); - break; - - case suspended: - suspend_count++; - - // When suspended, register the timer to resume in a bit. - resume_timer.set(0, 500, 0, 0); - theScheduler.epoll.register(resume_timer); - break; - - case stopped: - task.resume(); - break; - - default: - break; - } - } - ); - - scope ( exit ) theScheduler.epoll.unregister(resume_timer); - scope ( exit ) theScheduler.epoll.unregister(put_timer); - - task.suspend(); - test(!duplicate); - foreach ( k; received_keys ) - test!("<")(k, num_written); - test!("==")(suspend_count, 1); - test!("==")(mirror.updated_count, num_written); - test!("==")(mirror.refreshed_count, 0); - test!("==")(mirror.deleted_count, 0); - } -} - -/******************************************************************************* - - Test case which writes some records and starts a periodically refreshing - mirror. - -*******************************************************************************/ - -public class MirrorRefresh : NeoDhtTestCase -{ - import ocean.task.Task; - import ocean.io.select.client.TimerEvent; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Put followed by refreshing Mirror"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = false; - mirror_settings.periodic_refresh_s = 1; - - const num_written = 1_000; - putRecords(this.dht, this.test_channel, num_written); - - const end_after_refreshes = num_written * 3; - auto mirror = Mirror(this.dht); - mirror.start(mirror_settings, this.test_channel, - ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case refreshed: - // Stop mirroring after the updates have been received - if ( mirror.refreshed_count == end_after_refreshes ) - mirror.stop(); - break; - - case stopped: - task.resume(); - break; - - default: - break; - } - } - ); - - task.suspend(); - test!("==")(mirror.updated_count, 0); - test!("==")(mirror.refreshed_count, end_after_refreshes); - test!("==")(mirror.deleted_count, 0); - } -} - -/******************************************************************************* - - Test case which writes some records, starts a periodically refreshing - mirror while periodically suspending and resuming the request. - -*******************************************************************************/ - -public class MirrorRefreshSuspend : NeoDhtTestCase -{ - import ocean.task.Task; - import ocean.task.Scheduler; - import ocean.io.select.client.TimerEvent; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Put followed by refreshing Mirror and suspends/resumes"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = false; - mirror_settings.periodic_refresh_s = 1; - - const num_written = 500; - putRecords(this.dht, this.test_channel, num_written); - - auto mirror = Mirror(this.dht); - - // Timer which fires once and resumes the Mirror - scope resume_timer = new TimerEvent( - { - mirror.resume(); - return false; - } - ); - - bool mirror_suspended; - const end_after_refreshes = num_written * 3; - bool received_while_suspended; - mirror.start(mirror_settings, this.test_channel, - ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case started: - // As soon as the request starts, suspend it - mirror.suspend(); - break; - - case refreshed: - if ( mirror_suspended ) - received_while_suspended = true; - - // Stop mirroring after the updates have been received - if ( mirror.refreshed_count == end_after_refreshes ) - mirror.stop(); - break; - - case suspended: - mirror_suspended = true; - - // Set a timer to resume after 2.5s - resume_timer.set(2, 500, 0, 0); - theScheduler.epoll.register(resume_timer); - break; - - case resumed: - mirror_suspended = false; - break; - - case stopped: - task.resume(); - break; - - default: - break; - } - } - ); - - scope ( exit ) theScheduler.epoll.unregister(resume_timer); - - task.suspend(); - test(!received_while_suspended); - test!("==")(mirror.updated_count, 0); - test!("==")(mirror.refreshed_count, end_after_refreshes); - test!("==")(mirror.deleted_count, 0); - } -} - -/******************************************************************************* - - Test case which starts an updating mirror then removes the channel. - -*******************************************************************************/ - -public class MirrorRemoveChannel : NeoDhtTestCase -{ - import ocean.task.Task; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Mirror followed by RemoveChannel"; - return desc; - } - - public override void run ( ) - { - auto task = Task.getThis(); - - DhtClient.Neo.Mirror.Settings mirror_settings; - mirror_settings.initial_refresh = false; - mirror_settings.periodic_refresh_s = 0; - - void putNotifier ( DhtClient.Neo.Put.Notification info, - Const!(DhtClient.Neo.Put.Args) args ) { } - - void delNotifier ( DhtClient.Neo.RemoveChannel.Notification info, - DhtClient.Neo.RemoveChannel.Args args ) { } - - auto mirror = Mirror(this.dht); - mirror.start(mirror_settings, this.test_channel, - ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case started: - // As soon as the mirror is up, remove the channel - this.dht.neo.removeChannel(this.test_channel, - &delNotifier); - break; - - case channel_removed: - task.resume(); - break; - - default: - break; - } - } - ); - - task.suspend(); - test!("==")(mirror.updated_count, 0); - test!("==")(mirror.refreshed_count, 0); - test!("==")(mirror.deleted_count, 0); - } -} - -/******************************************************************************* - - Helper function to write some records to the specified channel. - - Params: - dht = DHT client instance to perform Puts with - channel = channel to write to - num_records = the number of records to write. Records are written with - keys from 0..num_records - notifier = neo Put notifier. If null, blocking Puts are performed - -*******************************************************************************/ - -private void putRecords ( DhtClient dht, cstring channel, size_t num_records, - DhtClient.Neo.Put.Notifier notifier = null ) -{ - ubyte[] val; - val.length = 128; - for ( hash_t key = 0; key < num_records; key++ ) - { - if ( notifier is null ) - { - auto res = dht.blocking.put(channel, key, val); - test(res.succeeded); - } - else - { - dht.neo.put(channel, key, val, notifier); - } - } -} - -/******************************************************************************* - - Helper for performing a Mirror request and checking the results. Reduces the - amount of boiler-plate in each test case. - -*******************************************************************************/ - -private struct Mirror -{ - import swarm.neo.protocol.Message : RequestId; - - /// DHT client to start Mirror request with. - private DhtClient dht; - - /// Mirror request id. - public RequestId id; - - /// Counters of Mirror actions. - public size_t updated_count, refreshed_count, deleted_count; - - /// User-provided Mirror notifier, passed to start(). - private DhtClient.Neo.Mirror.Notifier user_notifier; - - /*************************************************************************** - - Starts the Mirror request on the specified channel with the specified - settings. - - Params: - mirror_settings = settings for Mirror request - channel = channel to mirror - user_notifier = Mirror notifier (must be non-null) - - ***************************************************************************/ - - public void start ( DhtClient.Neo.Mirror.Settings mirror_settings, - cstring channel, DhtClient.Neo.Mirror.Notifier user_notifier ) - out - { - assert(this.user_notifier !is null); - } - body - { - verify(this.user_notifier is null); - this.user_notifier = user_notifier; - this.id = this.dht.neo.mirror(channel, &this.counterNotifier, - mirror_settings); - } - - /*************************************************************************** - - Suspends the Mirror request, using the controller to send a message to - the node. When the request is suspended, the Mirror notifier will be - called. - - ***************************************************************************/ - - public void suspend ( ) - { - verify(this.id != this.id.init); - this.dht.neo.control(this.id, - ( DhtClient.Neo.Mirror.IController mirror ) - { - mirror.suspend(); - } - ); - } - - /*************************************************************************** - - Resumes the Mirror request, using the controller to send a message to - the node. When the request is resumed, the Mirror notifier will be - called. - - ***************************************************************************/ - - public void resume ( ) - { - verify(this.id != this.id.init); - this.dht.neo.control(this.id, - ( DhtClient.Neo.Mirror.IController mirror ) - { - mirror.resume(); - } - ); - } - - /*************************************************************************** - - Stops the Mirror request, using the controller to send a message to the - node. When the request has finished, the Mirror notifier will be called. - - ***************************************************************************/ - - public void stop ( ) - { - verify(this.id != this.id.init); - this.dht.neo.control(this.id, - ( DhtClient.Neo.Mirror.IController mirror ) - { - mirror.stop(); - } - ); - } - - /*************************************************************************** - - Internal Mirror notifier. Updates the counters and calls the user's - notifier. - - Params: - info = Mirror notification - args = request arguments - - ***************************************************************************/ - - private void counterNotifier ( DhtClient.Neo.Mirror.Notification info, - Const!(DhtClient.Neo.Mirror.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case updated: - this.updated_count++; - break; - - case refreshed: - this.refreshed_count++; - break; - - case deleted: - this.deleted_count++; - break; - - default: - break; - } - - this.user_notifier(info, args); - } -} diff --git a/src/dhttest/cases/neo/OrderedPut.d b/src/dhttest/cases/neo/OrderedPut.d deleted file mode 100644 index 6d517b46..00000000 --- a/src/dhttest/cases/neo/OrderedPut.d +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - - Test for sending a set of records with sequential keys via Put - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.OrderedPut; - -import dhttest.DhtTestCase : NeoDhtTestCase; - -/******************************************************************************* - - Checks that a set of records with sequential keys written to the DHT via Put - are correctly added to the database. - -*******************************************************************************/ - -class OrderedPutTest : NeoDhtTestCase -{ - import dhttest.util.LocalStore; - import dhttest.util.Record; - - public override Description description ( ) - { - Description desc; - desc.name = "Neo ordered Put test"; - return desc; - } - - public override void run ( ) - { - LocalStore local; - NeoVerifier verifier; - - for ( uint i = 0; i < bulk_test_record_count; i++ ) - { - auto rec = Record.sequential(i); - this.dht.blocking.put(this.test_channel, rec.key, rec.val); - local.put(rec.key, rec.val); - } - - verifier.verifyAgainstDht(local, this.dht, this.test_channel); - } -} diff --git a/src/dhttest/cases/neo/Put.d b/src/dhttest/cases/neo/Put.d deleted file mode 100644 index c2c4123c..00000000 --- a/src/dhttest/cases/neo/Put.d +++ /dev/null @@ -1,93 +0,0 @@ -/******************************************************************************* - - Tests for the neo Put request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.Put; - -import ocean.transition; -import ocean.core.Test; -import dhttest.DhtTestCase : NeoDhtTestCase; - -/******************************************************************************* - - Test for record value size limit. - -*******************************************************************************/ - -class ValueTooBig : NeoDhtTestCase -{ - import dhtproto.client.request.Put : MaxRecordSize; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Put of a too big record value"; - return desc; - } - - public override void run ( ) - { - // Put a record of exactly MaxRecordSize bytes. - hash_t key; - mstring val; - val.length = MaxRecordSize; - auto res = this.dht.blocking.put(this.test_channel, key, val); - test(res.succeeded); - - // Put a too big record. - val.length = MaxRecordSize + 1; - res = this.dht.blocking.put(this.test_channel, key, val); - test(!res.succeeded); - } -} - -/******************************************************************************* - - Checks that large records can be fetched back from the DHT and that records - over the size limit are not stored. - -*******************************************************************************/ - -class LargeValueFetchTest : NeoDhtTestCase -{ - import dhtproto.client.request.Put : MaxRecordSize; - import dhttest.util.LocalStore; - - public override Description description ( ) - { - Description desc; - desc.name = "Neo Put and fetch of large records"; - return desc; - } - - public override void run ( ) - { - LocalStore local; - NeoVerifier verifier; - - // Put a record of exactly MaxRecordSize bytes. - hash_t key; - mstring val; - val.length = MaxRecordSize; - auto res = this.dht.blocking.put(this.test_channel, key, val); - test(res.succeeded); - local.put(key, val); - - // Put a too big record. - val.length = MaxRecordSize + 1; - res = this.dht.blocking.put(this.test_channel, key, val); - test(!res.succeeded); - - // Check that only the first record made it to the DHT and that it is - // returned by all get requests. - verifier.verifyAgainstDht(local, this.dht, this.test_channel); - } -} diff --git a/src/dhttest/cases/neo/Remove.d b/src/dhttest/cases/neo/Remove.d deleted file mode 100644 index 820814aa..00000000 --- a/src/dhttest/cases/neo/Remove.d +++ /dev/null @@ -1,60 +0,0 @@ -/******************************************************************************* - - Tests for the neo Remove request. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.Remove; - -import ocean.transition; -import dhttest.DhtTestCase : NeoDhtTestCase; - -/******************************************************************************* - - Checks that a set of records written to the DHT via Put are correctly - removed via Remove. - -*******************************************************************************/ - -class PutRemoveTest : NeoDhtTestCase -{ - import dhttest.util.LocalStore; - import dhttest.util.Record; - - public override Description description ( ) - { - Description desc; - desc.name = "Neo Put / Remove test"; - return desc; - } - - public override void run ( ) - { - LocalStore local; - NeoVerifier verifier; - - // Put some records. - for ( uint i = 0; i < bulk_test_record_count; i++ ) - { - auto rec = Record.sequential(i); - this.dht.blocking.put(this.test_channel, rec.key, rec.val); - local.put(rec.key, rec.val); - } - - // Remove half of them. - for ( uint i = 0; i < bulk_test_record_count; i += 2 ) - { - auto rec = Record.sequential(i); - this.dht.blocking.remove(this.test_channel, rec.key); - local.remove(rec.key); - } - - verifier.verifyAgainstDht(local, this.dht, this.test_channel); - } -} diff --git a/src/dhttest/cases/neo/RemoveChannel.d b/src/dhttest/cases/neo/RemoveChannel.d deleted file mode 100644 index 918aa6ff..00000000 --- a/src/dhttest/cases/neo/RemoveChannel.d +++ /dev/null @@ -1,61 +0,0 @@ -/******************************************************************************* - - Test cases for the neo RemoveChannel request. - - Note that a test case for the behaviour of removing a mirrored channel is - in dhttest.cases.neo.Mirror. (It was more convenient to write there due to - the existing framework in that module for handling Mirror requests.) - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.RemoveChannel; - -import ocean.transition; -import ocean.core.Test; -import dhttest.DhtTestCase : NeoDhtTestCase; -import dhtproto.client.DhtClient; - -/******************************************************************************* - - Simple test of removing a channel after writing a record to it. - -*******************************************************************************/ - -class RemoveChannel : NeoDhtTestCase -{ - override public Description description ( ) - { - Description desc; - desc.name = "Neo Put then RemoveChannel"; - return desc; - } - - public override void run ( ) - { - // Put a record. - hash_t key; - auto put_res = this.dht.blocking.put(this.test_channel, key, "hi"); - test(put_res.succeeded); - - // Check that it's in the channel. - void[] buf; - auto get_res = this.dht.blocking.get(this.test_channel, key, buf); - test(get_res.succeeded); - test(get_res.value == "hi"); - - // Remove the channel. - auto rem_res = this.dht.blocking.removeChannel(this.test_channel); - test(rem_res.succeeded); - - // Check that the put record is no longer in the channel. - get_res = this.dht.blocking.get(this.test_channel, key, buf); - test(get_res.succeeded); - test(get_res.value == ""); - } -} diff --git a/src/dhttest/cases/neo/UnorderedPut.d b/src/dhttest/cases/neo/UnorderedPut.d deleted file mode 100644 index 3d34ce91..00000000 --- a/src/dhttest/cases/neo/UnorderedPut.d +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - - Test for sending a set of records with non-sequential keys via Put - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.UnorderedPut; - -import dhttest.DhtTestCase : NeoDhtTestCase; - -/******************************************************************************* - - Checks that a set of records with non-sequential keys written to the DHT via - Put are correctly added to the database. - -*******************************************************************************/ - -class UnorderedPutTest : NeoDhtTestCase -{ - import dhttest.util.LocalStore; - import dhttest.util.Record; - - public override Description description ( ) - { - Description desc; - desc.name = "Neo unordered Put test"; - return desc; - } - - public override void run ( ) - { - LocalStore local; - NeoVerifier verifier; - - for ( uint i = 0; i < bulk_test_record_count; i++ ) - { - auto rec = Record.spread(i); - this.dht.blocking.put(this.test_channel, rec.key, rec.val); - local.put(rec.key, rec.val); - } - - verifier.verifyAgainstDht(local, this.dht, this.test_channel); - } -} diff --git a/src/dhttest/cases/neo/Update.d b/src/dhttest/cases/neo/Update.d deleted file mode 100644 index 5195c232..00000000 --- a/src/dhttest/cases/neo/Update.d +++ /dev/null @@ -1,228 +0,0 @@ -/******************************************************************************* - - Tests for the neo Update request. - - Copyright: - Copyright (c) 2018 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module dhttest.cases.neo.Update; - -import ocean.transition; -import ocean.core.Test; -import dhttest.DhtTestCase : NeoDhtTestCase; -import dhtproto.client.DhtClient; - -/******************************************************************************* - - Tests that Update does nothing on a non-existent record. - -*******************************************************************************/ - -class NoRecord : NeoDhtTestCase -{ - import ocean.core.SmartUnion; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Update on an empty channel"; - return desc; - } - - public override void run ( ) - { - bool record_not_found; - - void notifier ( DhtClient.Neo.Update.Notification info, - Const!(DhtClient.Neo.Update.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case received: - case succeeded: - case conflict: - break; - - case no_record: - record_not_found = true; - break; - - case error: - case no_node: - case node_disconnected: - case node_error: - case wrong_node: - case unsupported: - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - hash_t key; - this.dht.blocking.update(this.test_channel, key, ¬ifier); - test(record_not_found); - } -} - -/******************************************************************************* - - Simple test to Put a record, Update it, then Get the updated value. - -*******************************************************************************/ - -class Update : NeoDhtTestCase -{ - import ocean.core.SmartUnion; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Update"; - return desc; - } - - public override void run ( ) - { - bool update_succeeded; - - void notifier ( DhtClient.Neo.Update.Notification info, - Const!(DhtClient.Neo.Update.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case received: - (*info.received.updated_value) = "hello world".dup; - break; - - case succeeded: - update_succeeded = true; - break; - - case conflict: - case no_record: - break; - - case error: - case no_node: - case node_disconnected: - case node_error: - case wrong_node: - case unsupported: - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - hash_t key; - auto put_res = this.dht.blocking.put(this.test_channel, key, "hello"); - test(put_res.succeeded); - - this.dht.blocking.update(this.test_channel, key, ¬ifier); - test(update_succeeded); - - void[] buf; - auto get_res = this.dht.blocking.get(this.test_channel, key, buf); - test(get_res.succeeded); - test!("==")(get_res.value, "hello world"); - } -} - -/******************************************************************************* - - Simple test to Put a record, Update it, then Get the updated value, using - the serialization methods of the notifier. - -*******************************************************************************/ - -class SerializerUpdate : NeoDhtTestCase -{ - import ocean.core.SmartUnion; - import ocean.util.serialize.contiguous.Contiguous; - import ocean.util.serialize.contiguous.Deserializer; - import ocean.util.serialize.contiguous.Serializer; - - override public Description description ( ) - { - Description desc; - desc.name = "Neo Update with serialization"; - return desc; - } - - public override void run ( ) - { - struct Record - { - mstring name; - hash_t id; - ulong[7] daily_totals; - } - - bool update_succeeded; - - void notifier ( DhtClient.Neo.Update.Notification info, - Const!(DhtClient.Neo.Update.Args) args ) - { - with ( info.Active ) final switch ( info.active ) - { - case received: - // Deserialize the record. - Contiguous!(Record) record; - auto deserialized = info.received.deserialize(record); - - // Update the record. - deserialized.daily_totals[0]++; - - // Serialize the updated record. - info.received.serialize(*deserialized); - break; - - case succeeded: - update_succeeded = true; - break; - - case conflict: - case no_record: - break; - - case error: - case no_node: - case node_disconnected: - case node_error: - case wrong_node: - case unsupported: - break; - - mixin(typeof(info).handleInvalidCases); - } - } - - Record r; - r.name = "hello".dup; - r.id = 23; - r.daily_totals[0] = 1; - void[] buf; - Serializer.serialize(r, buf); - - hash_t key; - auto put_res = this.dht.blocking.put(this.test_channel, key, buf); - test(put_res.succeeded); - - this.dht.blocking.update(this.test_channel, key, ¬ifier); - test(update_succeeded); - - auto get_res = this.dht.blocking.get(this.test_channel, key, buf); - test(get_res.succeeded); - Contiguous!(Record) c; - auto r2 = Deserializer.deserialize(buf, c).ptr; - test!("==")(r2.name, r.name); - test!("==")(r2.id, r.id); - test!("==")(r2.daily_totals[0], 2); - } -} diff --git a/src/dhttest/util/LocalStore.d b/src/dhttest/util/LocalStore.d index 7b8b116f..90da499a 100644 --- a/src/dhttest/util/LocalStore.d +++ b/src/dhttest/util/LocalStore.d @@ -283,309 +283,4 @@ public struct LegacyVerifier } } -/******************************************************************************* - - Verifier which checks the contents of the DHT, using neo requests, against - the contents of a LocalStore instance. - -*******************************************************************************/ - -public struct NeoVerifier -{ - import dhtproto.client.DhtClient; - import turtle.runner.Logging; - import ocean.core.array.Search : contains; - import ocean.core.Test; - import ocean.task.Task; - - /// LocalStore to check against. Set in verifyAgainstDht. - private LocalStore* local; - - /*************************************************************************** - - Performs a series of tests to confirm that the content of the DHT - exactly matches the content of the map. - - Params: - local = LocalStore instance to check against - dht = DHT client to use to perform tests - channel = name of channel to compare against in DHT - - Throws: - TestException upon verification failure - - ***************************************************************************/ - - public void verifyAgainstDht ( ref LocalStore local, DhtClient dht, - cstring channel ) - { - this.local = &local; - - this.verifyGetChannels(dht, channel); - this.verifyGet(dht, channel); - this.verifyExists(dht, channel); - this.verifyGetAll(dht, channel); - this.verifyGetAllKeysOnly(dht, channel); - this.verifyGetAllFilter(dht, channel); - } - - /*************************************************************************** - - Compares all records in the DHT channel against the records in the local - store, using DHT Get requests. - - Params: - dht = DHT client to use to perform tests - channel = name of channel to compare against in DHT - - Throws: - TestException upon verification failure - - ***************************************************************************/ - - private void verifyGet ( DhtClient dht, cstring channel ) - { - log.trace("\tVerifying channel with Get"); - foreach ( k, v; this.local.data ) - { - void[] buf; - auto res = dht.blocking.get(channel, k, buf); - test(res.succeeded); - test!("==")(res.value, v); - } - } - - /*************************************************************************** - - Checks for the presence in the DHT of all records in the local store, - using DHT Exists requests. - - Params: - dht = DHT client to use to perform tests - channel = name of channel to compare against in DHT - - Throws: - TestException upon verification failure - - ***************************************************************************/ - - private void verifyExists ( DhtClient dht, cstring channel ) - { - log.trace("\tVerifying channel with Exists"); - foreach ( k, v; this.local.data ) - { - auto res = dht.blocking.exists(channel, k); - test(res.succeeded); - test(res.exists); - } - } - - /*************************************************************************** - - Compares all records in the DHT channel against the records in the local - store, using a DHT GetAll request. - - Params: - dht = DHT client to use to perform tests - channel = name of channel to compare against in DHT - Throws: - TestException upon verification failure - - ***************************************************************************/ - - private void verifyGetAll ( DhtClient dht, cstring channel ) - { - log.trace("\tVerifying channel with GetAll"); - auto task = Task.getThis(); - - void[][hash_t] records; - bool duplicate; - - void notifier ( DhtClient.Neo.GetAll.Notification info, - Const!(DhtClient.Neo.GetAll.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case received: - if ( info.received.key in records ) - duplicate = true; - else - records[info.received.key] = info.received.value.dup; - break; - - case started: - break; - - case finished: - task.resume(); - break; - - default: - assert(false); - } - } - - dht.neo.getAll(channel, ¬ifier); - task.suspend(); - - test(!duplicate); - test!("==")(records.length, this.local.data.length); - foreach ( k, v; this.local.data ) - { - test!("in")(k, records); - test!("==")(v, records[k]); - } - } - - /*************************************************************************** - - Gets the names of all channels in the DHT and checks that the specified - channel is in the list. - - Params: - dht = DHT client to use to perform tests - channel = name of channel to compare against in DHT - - Throws: - TestException upon verification failure - - ***************************************************************************/ - - private void verifyGetChannels ( DhtClient dht, cstring channel ) - { - log.trace("\tVerifying channel with GetChannels"); - auto task = Task.getThis(); - - mstring buf; - mstring[] channels; - foreach ( channel; dht.blocking.getChannels(buf) ) - channels ~= channel.dup; - - test(channels.contains(channel)); - } - - /*************************************************************************** - - Compares all record keys in the DHT channel against the keys in the - local store, using a DHT GetAll request in keys-only mode. - - Params: - dht = DHT client to use to perform tests - channel = name of channel to compare against in DHT - - Throws: - TestException upon verification failure - - ***************************************************************************/ - - private void verifyGetAllKeysOnly ( DhtClient dht, cstring channel ) - { - log.trace("\tVerifying channel with GetAll, keys-only"); - auto task = Task.getThis(); - - bool[hash_t] keys; - bool duplicate; - - void notifier ( DhtClient.Neo.GetAll.Notification info, - Const!(DhtClient.Neo.GetAll.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case received_key: - if ( info.received_key.key in keys ) - duplicate = true; - else - keys[info.received_key.key] = true; - break; - - case started: - break; - - case finished: - task.resume(); - break; - - default: - assert(false); - } - } - - DhtClient.Neo.GetAll.Settings settings; - settings.keys_only = true; - dht.neo.getAll(channel, ¬ifier, settings); - task.suspend(); - - test(!duplicate); - test!("==")(keys.length, this.local.data.length); - foreach ( k, v; this.local.data ) - test!("in")(k, keys); - } - - /*************************************************************************** - - Compares all records in the DHT channel against the records in the local - store, using a DHT GetAll request with a filter. - - Params: - dht = DHT client to use to perform tests - channel = name of channel to compare against in DHT - - Throws: - TestException upon verification failure - - ***************************************************************************/ - - private void verifyGetAllFilter ( DhtClient dht, cstring channel ) - { - log.trace("\tVerifying channel with GetAll, filtering"); - auto task = Task.getThis(); - - const filter = "0"; - - hash_t[] local_filtered; - foreach ( k, v; this.local.data ) - if ( v.contains(filter) ) - local_filtered ~= k; - - void[][hash_t] records; - bool duplicate; - - void notifier ( DhtClient.Neo.GetAll.Notification info, - Const!(DhtClient.Neo.GetAll.Args) args ) - { - with ( info.Active ) switch ( info.active ) - { - case received: - if ( info.received.key in records ) - duplicate = true; - else - records[info.received.key] = info.received.value.dup; - break; - - case started: - break; - - case finished: - task.resume(); - break; - - default: - assert(false); - } - } - - DhtClient.Neo.GetAll.Settings settings; - settings.value_filter = cast(void[])filter; - dht.neo.getAll(channel, ¬ifier, settings); - task.suspend(); - - test(!duplicate); - test!("==")(records.length, local_filtered.length); - foreach ( k, v; records ) - { - test!("in")(k, this.local.data); - test!("==")(v, this.local.data[k]); - } - } -} diff --git a/src/fakedht/ConnectionHandler.d b/src/fakedht/ConnectionHandler.d index 7d34075d..e44773a1 100644 --- a/src/fakedht/ConnectionHandler.d +++ b/src/fakedht/ConnectionHandler.d @@ -72,8 +72,6 @@ public class DhtConnectionHandler : import fakedht.request.Remove; import fakedht.request.RemoveChannel; - import ocean.core.Enforce; - import ocean.core.Verify; import ocean.io.Stdout : Stderr; import core.stdc.stdlib : abort; @@ -201,8 +199,7 @@ public class DhtConnectionHandler : RedistributeNode[]* getRedistributeNodeBuffer ( ) { - enforce(false); - return null; + assert (false); } } diff --git a/src/fakedht/DhtNode.d b/src/fakedht/DhtNode.d index 5a164dd8..436e8bc8 100644 --- a/src/fakedht/DhtNode.d +++ b/src/fakedht/DhtNode.d @@ -25,7 +25,7 @@ import ocean.util.log.Logger; import fakedht.ConnectionHandler; -import swarm.node.model.NeoNode; +import swarm.node.model.Node; /******************************************************************************* @@ -52,16 +52,12 @@ public class DhtNode { import core.stdc.stdlib : abort; - import ocean.core.VersionCheck; import ocean.io.select.client.model.ISelectClient : IAdvancedSelectClient; import ocean.net.server.connection.IConnectionHandlerInfo; import ocean.io.select.protocol.generic.ErrnoIOException; import dhtproto.client.legacy.DhtConst; import swarm.node.connection.ConnectionHandler; - import swarm.neo.authentication.HmacDef: Key; - import fakedht.neo.RequestHandlers; - import fakedht.neo.SharedResources; import swarm.neo.AddrPort; import fakedht.Storage; @@ -93,20 +89,7 @@ public class DhtNode params.epoll = epoll; params.node_info = this; - Options neo_options; - neo_options.requests = requests; - neo_options.epoll = epoll; - - static if (!hasFeaturesFrom!("swarm", 4, 7)) - neo_options.no_delay = true; // favour network turn-around over packet efficiency - - neo_options.credentials_map["admin"] = Key.init; - - ushort neo_port = node_item.Port; - if ( neo_port != 0) - neo_port += 100; // See dhtnode.node.DhtHashRange.newNodeAdded() - - super(node_item, neo_port, params, neo_options, backlog); + super (node_item, params, backlog); this.error_callback = &this.onError; } @@ -135,25 +118,6 @@ public class DhtNode this.log_errors = false; } - /*************************************************************************** - - Scope allocates a request resource acquirer instance and passes it to - the provided delegate for use in a request. - - Params: - handle_request_dg = delegate that receives a resources acquirer and - initiates handling of a request - - ***************************************************************************/ - - override protected void getResourceAcquirer ( - void delegate ( Object request_resources ) handle_request_dg ) - { - // In the fake node, we don't actually store a shared resources - // instance; a new one is simply passed to each request. - handle_request_dg(new SharedResources); - } - /*************************************************************************** Make any error fatal diff --git a/src/fakedht/Storage.d b/src/fakedht/Storage.d index db9b2511..c8f92a71 100644 --- a/src/fakedht/Storage.d +++ b/src/fakedht/Storage.d @@ -200,9 +200,6 @@ struct DHT class Channel { - import ocean.core.Verify; - import ocean.text.convert.Formatter; - /*************************************************************************** Defines a channel listener type which expects one argument for @@ -259,7 +256,7 @@ class Channel public void listenerFlushed ( ) { - verify(this.sending_listeners > 0); + assert(this.sending_listeners); this.sending_listeners--; if ( this.sending_listeners == 0 && (this.caller !is null) ) @@ -364,19 +361,6 @@ class Channel return (value is null) ? null : *value; } - /*************************************************************************** - - Ditto - - ***************************************************************************/ - - public ValueType get ( hash_t key ) - { - mstring key_str; - sformat(key_str, "{:x16}", key); - return this.get(key_str); - } - /*************************************************************************** Params: @@ -397,19 +381,6 @@ class Channel return *value; } - /*************************************************************************** - - Ditto - - ***************************************************************************/ - - public ValueType getVerify ( hash_t key ) - { - mstring key_str; - sformat(key_str, "{:x16}", key); - return this.getVerify(key_str); - } - /*************************************************************************** Adds a new record or modifies an old one @@ -431,19 +402,6 @@ class Channel this.listeners.waitUntilFlushed(); } - /*************************************************************************** - - Ditto - - ***************************************************************************/ - - public void put ( hash_t key, ValueType value ) - { - mstring key_str; - sformat(key_str, "{:x16}", key); - this.put(key_str, value); - } - /*************************************************************************** Counts total size taken by all records @@ -466,39 +424,11 @@ class Channel Params: key = key of the record to remove - Returns: - true if the record existed or false if it did not - ***************************************************************************/ - public bool remove ( cstring key ) - out ( existed ) - { - assert((key in this.data) is null); - } - body + public void remove ( cstring key ) { - auto existed = (key in this.data) !is null; this.data.remove(idup(key)); - - this.listeners.trigger(Listeners.Listener.Code.Deletion, key); - if (Task.getThis() !is null) - this.listeners.waitUntilFlushed(); - - return existed; - } - - /*************************************************************************** - - Ditto - - ***************************************************************************/ - - public bool remove ( hash_t key ) - { - mstring key_str; - sformat(key_str, "{:x16}", key); - return this.remove(key_str); } /*************************************************************************** diff --git a/src/fakedht/main.d b/src/fakedht/main.d index 735e92d8..9befff92 100644 --- a/src/fakedht/main.d +++ b/src/fakedht/main.d @@ -26,7 +26,6 @@ import dhtproto.client.legacy.DhtConst; import ocean.io.select.EpollSelectDispatcher; import ocean.io.select.client.TimerEvent; import ocean.core.MessageFiber; -import ocean.task.Scheduler; import ocean.util.log.Logger; import ocean.util.log.AppendConsole; @@ -53,16 +52,13 @@ version ( UnitTest ) { } else void main ( ) { - SchedulerConfiguration config; - initScheduler(config); - - auto node = new DhtNode(DhtConst.NodeItem("127.0.0.1".dup, 10000), - theScheduler.epoll); + auto epoll = new EpollSelectDispatcher; + auto node = new DhtNode(DhtConst.NodeItem("127.0.0.1".dup, 10000), epoll); auto log = Log.lookup("fakedht.main"); log.info("Registering fake node"); - node.register(theScheduler.epoll); + node.register(epoll); log.info("Starting infinite event loop, kill the process if not needed anymore"); - theScheduler.epoll.eventLoop(); + epoll.eventLoop(); } diff --git a/src/fakedht/neo/RequestHandlers.d b/src/fakedht/neo/RequestHandlers.d deleted file mode 100644 index febb8b20..00000000 --- a/src/fakedht/neo/RequestHandlers.d +++ /dev/null @@ -1,58 +0,0 @@ -/******************************************************************************* - - Table of request handlers by command. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.RequestHandlers; - -import swarm.neo.node.ConnectionHandler; -import swarm.neo.request.Command; - -import dhtproto.common.RequestCodes; - -import fakedht.neo.request.GetHashRange; -import fakedht.neo.request.Put; -import fakedht.neo.request.Get; -import fakedht.neo.request.Mirror; -import fakedht.neo.request.GetAll; -import fakedht.neo.request.Remove; -import fakedht.neo.request.GetChannels; -import fakedht.neo.request.Exists; -import fakedht.neo.request.RemoveChannel; -import fakedht.neo.request.Update; - -/******************************************************************************* - - This table of request handlers by command is used by the connection handler. - When creating a new request, the function corresponding to the request - command is called in a fiber. - -*******************************************************************************/ - -public ConnectionHandler.RequestMap requests; - -static this ( ) -{ - requests.add(Command(RequestCode.GetHashRange, 0), "GetHashRange", - GetHashRangeImpl_v0.classinfo); - requests.add(Command(RequestCode.Put, 0), "Put", PutImpl_v0.classinfo); - requests.add(Command(RequestCode.Get, 0), "Get", GetImpl_v0.classinfo); - requests.add(Command(RequestCode.Mirror, 0), "Mirror", MirrorImpl_v0.classinfo); - requests.add(Command(RequestCode.GetAll, 0), "GetAll", GetAllImpl_v0.classinfo); - requests.add(Command(RequestCode.Remove, 0), "Remove", RemoveImpl_v0.classinfo); - requests.add(Command(RequestCode.GetChannels, 0), - "GetChannels", GetChannelsImpl_v0.classinfo); - requests.add(Command(RequestCode.Exists, 0), - "Exists", ExistsImpl_v0.classinfo); - requests.add(Command(RequestCode.RemoveChannel, 0), "RemoveChannel", - RemoveChannelImpl_v0.classinfo); - requests.add(Command(RequestCode.Update, 0), - "Update", UpdateImpl_v0.classinfo); -} diff --git a/src/fakedht/neo/SharedResources.d b/src/fakedht/neo/SharedResources.d deleted file mode 100644 index 7899a82a..00000000 --- a/src/fakedht/neo/SharedResources.d +++ /dev/null @@ -1,200 +0,0 @@ -/******************************************************************************* - - Fake DHT node neo request shared resources getter. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.SharedResources; - -import ocean.transition; - -import dhtproto.node.neo.request.core.IRequestResources; - -/******************************************************************************* - - Provides resources required by the protocol. As this implementation is fpr - testing purposes only, it simply allocates as much stuff as necessary to - keep the code simple. - -*******************************************************************************/ - -class SharedResources : IRequestResources -{ - import ocean.core.Verify; - import ocean.io.compress.Lzo; - import swarm.neo.util.MessageFiber; - import swarm.util.RecordBatcher; - - /*************************************************************************** - - Struct wrapper used to workaround D's inability to allocate slices on - the heap via `new`. - - ***************************************************************************/ - - private static struct Buffer - { - void[] data; - } - - /*************************************************************************** - - Returns: - a shared LZO instance - - ***************************************************************************/ - - override public Lzo lzo ( ) - { - return new Lzo; - } - - /*************************************************************************** - - Returns: - a new buffer to store record values in - - ***************************************************************************/ - - override public void[]* getVoidBuffer ( ) - { - return &((new Buffer).data); - } - - /*************************************************************************** - - Gets a fiber to use during the request's lifetime and assigns the - provided delegate as its entry point. - - Params: - fiber_method = entry point to assign to acquired fiber - - Returns: - a new MessageFiber acquired to use during the request's lifetime - - ***************************************************************************/ - - public MessageFiber getFiber ( void delegate ( ) fiber_method ) - { - return new MessageFiber(fiber_method, 64 * 1024); - } - - /*************************************************************************** - - Gets a record batcher to use during the request's lifetime. - - Returns: - a new record batcher acquired to use during the request's lifetime - - ***************************************************************************/ - - public RecordBatcher getRecordBatcher ( ) - { - return new RecordBatcher(new Lzo); - } - - /*************************************************************************** - - Gets a periodically firing timer. - - Params: - period_s = seconds part of timer period - period_ms = milliseconds part of timer period - timer_dg = delegate to call when timer fires - - Returns: - ITimer interface to a timer to use during the request's lifetime - - ***************************************************************************/ - - public ITimer getTimer ( uint period_s, uint period_ms, - void delegate ( ) timer_dg ) - { - verify(period_ms < 1_000); - verify(period_s > 0 || period_ms > 0); - verify(timer_dg !is null); - return new Timer(period_s, period_ms, timer_dg); - } - - /*************************************************************************** - - Timer to be used during the request's lifetime. - - ***************************************************************************/ - - private class Timer : ITimer - { - import ocean.io.select.client.TimerEvent; - import ocean.task.Scheduler; - - /// Flag set to true when the timer is running. - private bool running; - - /// Timer event registered with epoll. - private TimerEvent timer; - - // User's timer delegate. - private void delegate ( ) timer_dg; - - /*********************************************************************** - - Constructor. - - Params: - period_s = seconds part of timer period - period_ms = milliseconds part of timer period - timer_dg = delegate to call when timer fires - - ***********************************************************************/ - - private this ( uint period_s, uint period_ms, void delegate ( ) timer_dg ) - { - this.timer_dg = timer_dg; - this.timer = new TimerEvent(&this.timerDg); - this.timer.set(period_s, period_ms, period_s, period_ms); - } - - /*********************************************************************** - - Starts the timer, registering it with epoll. - - ***********************************************************************/ - - public void start ( ) - { - this.running = true; - theScheduler.epoll.register(this.timer); - } - - /*********************************************************************** - - Stops the timer, unregistering it from epoll. - - ***********************************************************************/ - - public void stop ( ) - { - this.running = false; - theScheduler.epoll.unregister(this.timer); - } - - /*********************************************************************** - - Internal delegate called when timer fires. Calls the user's delegate - and handles unregistering when stopped. - - ***********************************************************************/ - - private bool timerDg ( ) - { - this.timer_dg(); - return this.running; - } - } -} diff --git a/src/fakedht/neo/request/Exists.d b/src/fakedht/neo/request/Exists.d deleted file mode 100644 index 021b58b9..00000000 --- a/src/fakedht/neo/request/Exists.d +++ /dev/null @@ -1,72 +0,0 @@ -/******************************************************************************* - - Fake DHT node Exists request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.Exists; - -import dhtproto.node.neo.request.Exists; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 Exists request protocol. - -*******************************************************************************/ - -public class ExistsImpl_v0 : ExistsProtocol_v0 -{ - import fakedht.Storage; - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - override protected bool responsibleForKey ( hash_t key ) - { - // In the fake DHT, we always have a single node responsible for all - // keys. - return true; - } - - /*************************************************************************** - - Checks whether a single record exists in the storage engine. - - Params: - channel = channel to check in - key = key of record to check - found = out value, set to true if the record exists - - Returns: - true if the operation succeeded; false if an error occurred - - ***************************************************************************/ - - override protected bool exists ( cstring channel, hash_t key, out bool exists ) - { - auto value_in_channel = global_storage.getCreate(channel).get(key); - exists = value_in_channel !is null; - return true; - } -} diff --git a/src/fakedht/neo/request/Get.d b/src/fakedht/neo/request/Get.d deleted file mode 100644 index b27a0412..00000000 --- a/src/fakedht/neo/request/Get.d +++ /dev/null @@ -1,82 +0,0 @@ -/******************************************************************************* - - Fake DHT node Get request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.Get; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import dhtproto.node.neo.request.Get; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 Get request protocol. - -*******************************************************************************/ - -public class GetImpl_v0 : GetProtocol_v0 -{ - import fakedht.Storage; - import ocean.core.Array : copy; - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - override protected bool responsibleForKey ( hash_t key ) - { - // In the fake DHT, we always have a single node responsible for all - // keys. - return true; - } - - /*************************************************************************** - - Gets a single record from the storage engine. - - Params: - channel = channel to read from - key = key of record to read - dg = called with the value of the record, if it exists - - Returns: - true if the operation succeeded (the record was fetched or did not - exist); false if an error occurred - - ***************************************************************************/ - - override protected bool get ( cstring channel, hash_t key, - void delegate ( Const!(void)[] value ) dg ) - { - auto value_in_channel = global_storage.getCreate(channel).get(key); - if ( value_in_channel !is null ) - dg(value_in_channel); - return true; - } -} diff --git a/src/fakedht/neo/request/GetAll.d b/src/fakedht/neo/request/GetAll.d deleted file mode 100644 index 9ff7ad5e..00000000 --- a/src/fakedht/neo/request/GetAll.d +++ /dev/null @@ -1,140 +0,0 @@ -/******************************************************************************* - - Fake DHT node GetAll request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.GetAll; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import dhtproto.node.neo.request.GetAll; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 GetAll request protocol. - -*******************************************************************************/ - -public class GetAllImpl_v0 : GetAllProtocol_v0 -{ - import fakedht.Storage; - import ocean.core.Verify; - import ocean.text.convert.Hash : toHashT; - - /// Reference to channel being iterated over. - private Channel channel; - - /// List of keys to visit during an iteration. - private istring[] iterate_keys; - - /*************************************************************************** - - Called to begin the iteration over the channel being fetched. - - Params: - channel_name = name of channel to iterate over - - Returns: - true if the iteration has been initialised, false to abort the - request - - ***************************************************************************/ - - override protected bool startIteration ( cstring channel_name ) - { - this.channel = global_storage.getCreate(channel_name); - this.iterate_keys = this.channel.getKeys(); - return true; - } - - /*************************************************************************** - - Called to continue the iteration over the channel being fetched, - continuing from the specified hash (the last record received by the - client). - - Params: - channel_name = name of channel to iterate over - continue_from = hash of last record received by the client. The - iteration will continue from the next hash in the channel - - Returns: - true if the iteration has been initialised, false to abort the - request - - ***************************************************************************/ - - override protected bool continueIteration ( cstring channel_name, - hash_t continue_from ) - { - this.channel = global_storage.getCreate(channel_name); - this.iterate_keys = this.channel.getKeys(); - - size_t index = this.iterate_keys.length; - foreach_reverse ( i, str_key; this.iterate_keys ) - { - hash_t key; - auto ok = toHashT(str_key, key); - verify(ok); - if ( key == continue_from ) - { - index = i; - break; - } - } - - // Cut off any records after the last key iterated over. - if ( index < this.iterate_keys.length ) - this.iterate_keys.length = index; - - return true; - } - - /*************************************************************************** - - Gets the next record in the iteration, if one exists. - - Params: - dg = called with the key and value of the next record, if available - - Returns: - true if a record was passed to `dg` or false if the iteration is - finished - - ***************************************************************************/ - - override protected bool getNext ( - void delegate ( hash_t key, Const!(void)[] value ) dg ) - { - if ( this.iterate_keys.length == 0 ) - return false; - - auto str_key = this.iterate_keys[$-1]; - this.iterate_keys.length = this.iterate_keys.length - 1; - - hash_t key; - auto ok = toHashT(str_key, key); - verify(ok); - auto value = this.channel.get(key); - - dg(key, value); - return true; - } -} diff --git a/src/fakedht/neo/request/GetChannels.d b/src/fakedht/neo/request/GetChannels.d deleted file mode 100644 index 553a7744..00000000 --- a/src/fakedht/neo/request/GetChannels.d +++ /dev/null @@ -1,52 +0,0 @@ -/******************************************************************************* - - Fake DHT node GetChannels request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.GetChannels; - -import dhtproto.node.neo.request.GetChannels; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 GetChannels request protocol. - -*******************************************************************************/ - -public class GetChannelsImpl_v0 : GetChannelsProtocol_v0 -{ - import fakedht.Storage; - import ocean.text.convert.Hash : toHashT; - - /*************************************************************************** - - opApply iteration over the names of the channels in storage. - - ***************************************************************************/ - - override protected int opApply ( int delegate ( ref cstring ) dg ) - { - int ret; - foreach ( channel; global_storage.getChannelList() ) - { - cstring const_channel = channel; - ret = dg(const_channel); - if ( ret ) - break; - } - return ret; - } -} diff --git a/src/fakedht/neo/request/GetHashRange.d b/src/fakedht/neo/request/GetHashRange.d deleted file mode 100644 index c5b2d3ac..00000000 --- a/src/fakedht/neo/request/GetHashRange.d +++ /dev/null @@ -1,102 +0,0 @@ -/******************************************************************************* - - Fake DHT node GetHashRange request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.GetHashRange; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import dhtproto.node.neo.request.GetHashRange; - -import fakedht.neo.SharedResources; - -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 GetHashRange request protocol. - -*******************************************************************************/ - -public class GetHashRangeImpl_v0 : GetHashRangeProtocol_v0 -{ - /*************************************************************************** - - Gets the current hash range of this node. - - Params: - min = out value where the current minimum hash of this node is stored - max = out value where the current maximum hash of this node is stored - - ***************************************************************************/ - - override protected void getCurrentHashRange ( out hash_t min, out hash_t max ) - { - min = hash_t.min; - max = hash_t.max; - } - - /*************************************************************************** - - Informs the node that this request is now waiting for hash range - updates. hashRangeUpdate() will be called, when updates are pending. - - ***************************************************************************/ - - override protected void registerForHashRangeUpdates ( ) - { - // The fake DHT node currently does not handle forwarding of hash range - // updates to the client. - } - - /*************************************************************************** - - Informs the node that this request is no longer waiting for hash range - updates. - - ***************************************************************************/ - - override protected void unregisterForHashRangeUpdates ( ) - { - // The fake DHT node currently does not handle forwarding of hash range - // updates to the client. - } - - /*************************************************************************** - - Gets the next pending hash range update (or returns false, if no updates - are pending). The implementing node should store a queue of updates per - GetHashRange request and feed them to the request, in order, when this - method is called. - - Params: - update = out value to receive the next pending update, if one is - available - - Returns: - false if no update is pending - - ***************************************************************************/ - - override protected bool getNextHashRangeUpdate ( out HashRangeUpdate update ) - { - // The fake DHT node currently does not handle forwarding of hash range - // updates to the client. - return false; - } -} diff --git a/src/fakedht/neo/request/Mirror.d b/src/fakedht/neo/request/Mirror.d deleted file mode 100644 index 77db9780..00000000 --- a/src/fakedht/neo/request/Mirror.d +++ /dev/null @@ -1,213 +0,0 @@ -/******************************************************************************* - - Fake DHT node Mirror request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.Mirror; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import dhtproto.node.neo.request.Mirror; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 Mirror request protocol. - -*******************************************************************************/ - -import fakedht.Storage; // DhtListener - -public class MirrorImpl_v0 : MirrorProtocol_v0, DhtListener -{ - import fakedht.Storage; - import ocean.core.Verify; - import ocean.core.array.Mutation : copy; - import ocean.text.convert.Hash : toHashT; - - /// Reference to channel being mirrored. - private Channel channel; - - /// Name of channel being mirrored. - private cstring channel_name; - - /// List of keys to visit during an iteration. - private istring[] iterate_keys; - - /*************************************************************************** - - Performs any logic needed to subscribe to and start mirroring the - channel of the given name. - - Params: - channel_name = channel to mirror - - Returns: - true if the channel may be used, false to abort the request - - ***************************************************************************/ - - override protected bool prepareChannel ( cstring channel_name ) - { - this.channel = global_storage.getCreate(channel_name); - this.channel_name = channel_name.dup; - return true; - } - - /*************************************************************************** - - Returns: - the name of the channel being mirrored (for logging) - - ***************************************************************************/ - - override protected cstring channelName ( ) - { - return this.channel_name; - } - - /*************************************************************************** - - Registers this request to receive updates on the channel. - - ***************************************************************************/ - - override protected void registerForUpdates ( ) - { - verify(this.channel !is null); - this.channel.register(this); - } - - /*************************************************************************** - - Unregisters this request from receiving updates on the channel. - - ***************************************************************************/ - - override protected void unregisterForUpdates ( ) - { - if (this.channel !is null) - this.channel.unregister(this); - } - - /*************************************************************************** - - Gets the value of the record with the specified key, if it exists. - - Params: - key = key of record to get from storage - buf = buffer to write the value into - - Returns: - record value or null, if the record does not exist - - ***************************************************************************/ - - override protected void[] getRecordValue ( hash_t key, ref void[] buf ) - { - auto storage_value = this.channel.get(key); - if ( storage_value is null ) - return null; - else - { - buf.copy(storage_value); - return buf; - } - } - - /*************************************************************************** - - Called to begin iterating over the channel being mirrored. - - ***************************************************************************/ - - override protected void startIteration ( ) - { - verify(this.iterate_keys.length == 0, "Iteration already in progress"); - this.iterate_keys = this.channel.getKeys(); - } - - /*************************************************************************** - - Adds the next record in the iteration to the update queue, if one - exists. - - Params: - hash_key = output value to receive the next key to add to the queue - - Returns: - true if hash_key was set or false if the iteration is finished - - ***************************************************************************/ - - override protected bool iterateNext ( out hash_t hash_key ) - { - if ( this.iterate_keys.length == 0 ) - return false; - - auto key = this.iterate_keys[$-1]; - this.iterate_keys.length = this.iterate_keys.length - 1; - - auto ok = toHashT(key, hash_key); - verify(ok); - - return true; - } - - /*************************************************************************** - - DhtListener interface method. Called by Storage when records are - modified or the channel is deleted. - - Params: - code = trigger event code - key = new dht key - - ***************************************************************************/ - - public void trigger ( Code code, cstring key ) - { - with ( Code ) switch ( code ) - { - case DataReady: - hash_t hash_key; - auto ok = toHashT(key, hash_key); - verify(ok); - - this.updated(Update(UpdateType.Change, hash_key)); - break; - - case Deletion: - hash_t hash_key; - auto ok = toHashT(key, hash_key); - verify(ok); - verify(ok); - - this.updated(Update(UpdateType.Deletion, hash_key)); - break; - - case Finish: - this.channelRemoved(); - break; - - default: - break; - } - } -} diff --git a/src/fakedht/neo/request/Put.d b/src/fakedht/neo/request/Put.d deleted file mode 100644 index 7cd05c93..00000000 --- a/src/fakedht/neo/request/Put.d +++ /dev/null @@ -1,79 +0,0 @@ -/******************************************************************************* - - Fake DHT node Put request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.Put; - -/******************************************************************************* - - Imports - -*******************************************************************************/ - -import dhtproto.node.neo.request.Put; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 Put request protocol. - -*******************************************************************************/ - -public class PutImpl_v0 : PutProtocol_v0 -{ - import fakedht.Storage; - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - override protected bool responsibleForKey ( hash_t key ) - { - // In the fake DHT, we always have a single node responsible for all - // keys. - return true; - } - - /*************************************************************************** - - Writes a single record to the storage engine. - - Params: - channel = channel to write to - key = key of record to write - value = record value to write - - Returns: - true if the record was written; false if an error occurred - - ***************************************************************************/ - - override protected bool put ( cstring channel, hash_t key, in void[] value ) - { - // We need to dup value, as it is a slice to the neo connection's read - // buffer. - global_storage.getCreate(channel).put(key, value.dup); - return true; - } -} diff --git a/src/fakedht/neo/request/Remove.d b/src/fakedht/neo/request/Remove.d deleted file mode 100644 index 4f253188..00000000 --- a/src/fakedht/neo/request/Remove.d +++ /dev/null @@ -1,74 +0,0 @@ -/******************************************************************************* - - Fake DHT node Remove request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.Remove; - -import dhtproto.node.neo.request.Remove; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 Remove request protocol. - -*******************************************************************************/ - -public class RemoveImpl_v0 : RemoveProtocol_v0 -{ - import fakedht.Storage; - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - override protected bool responsibleForKey ( hash_t key ) - { - // In the fake DHT, we always have a single node responsible for all - // keys. - return true; - } - - /*************************************************************************** - - Removes a single record from the storage engine. - - Params: - channel = channel to remove from - key = key of record to remove - existed = out value, set to true if the record was present and - removed or false if the record was not present - - Returns: - true if the operation succeeded (the record was removed or did not - exist); false if an error occurred - - ***************************************************************************/ - - override protected bool remove ( cstring channel, hash_t key, - out bool existed ) - { - existed = global_storage.getCreate(channel).remove(key); - return true; - } -} diff --git a/src/fakedht/neo/request/RemoveChannel.d b/src/fakedht/neo/request/RemoveChannel.d deleted file mode 100644 index ebdcc80b..00000000 --- a/src/fakedht/neo/request/RemoveChannel.d +++ /dev/null @@ -1,69 +0,0 @@ -/******************************************************************************* - - Fake DHT node RemoveChannel request implementation. - - Copyright: - Copyright (c) 2017 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.RemoveChannel; - -import dhtproto.node.neo.request.RemoveChannel; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 RemoveChannel request protocol. - -*******************************************************************************/ - -public class RemoveChannelImpl_v0 : RemoveChannelProtocol_v0 -{ - import fakedht.Storage; - - /*************************************************************************** - - Checks whether the specified client is permitted to remove channels. - - Params: - client_name = name of client requesting channel removal - - Returns: - true if the client is permitted to remove channels - - ***************************************************************************/ - - override protected bool clientPermitted ( cstring client_name ) - { - // In tests, always allow channel removal. - return true; - } - - /*************************************************************************** - - Removes the specified channel. - - Params: - channel_name = channel to remove - - Returns: - true if the operation succeeded (the channel was removed or did not - exist); false if an error occurred - - ***************************************************************************/ - - override protected bool removeChannel ( cstring channel_name ) - { - global_storage.remove(channel_name); - return true; - } -} diff --git a/src/fakedht/neo/request/Update.d b/src/fakedht/neo/request/Update.d deleted file mode 100644 index 6183d8f9..00000000 --- a/src/fakedht/neo/request/Update.d +++ /dev/null @@ -1,116 +0,0 @@ -/******************************************************************************* - - Fake DHT node Update request implementation. - - Copyright: - Copyright (c) 2018 sociomantic labs GmbH. All rights reserved. - - License: - Boost Software License Version 1.0. See LICENSE.txt for details. - -*******************************************************************************/ - -module fakedht.neo.request.Update; - -import dhtproto.node.neo.request.Update; - -import fakedht.neo.SharedResources; -import swarm.neo.node.RequestOnConn; -import swarm.neo.request.Command; - -import ocean.transition; - -/******************************************************************************* - - Fake node implementation of the v0 Update request protocol. - -*******************************************************************************/ - -public class UpdateImpl_v0 : UpdateProtocol_v0 -{ - import fakedht.Storage; - - /*************************************************************************** - - Checks whether the node is responsible for the specified key. - - Params: - key = key of record to write - - Returns: - true if the node is responsible for the key - - ***************************************************************************/ - - override protected bool responsibleForKey ( hash_t key ) - { - // In the fake DHT, we always have a single node responsible for all - // keys. - return true; - } - - /*************************************************************************** - - Reads a single record from the storage engine. - - Params: - channel = channel to read from - key = key of record to read - dg = called with the value of the record, if it exists - - Returns: - true if the operation succeeded (the record was fetched or did not - exist); false if an error occurred - - ***************************************************************************/ - - override protected bool get ( cstring channel, hash_t key, - void delegate ( Const!(void)[] value ) dg ) - { - auto value_in_channel = global_storage.getCreate(channel).get(key); - if ( value_in_channel !is null ) - dg(value_in_channel); - return true; - } - - /*************************************************************************** - - Writes a single record to the storage engine. - - Params: - channel = channel to write to - key = key of record to write - value = record value to write - - Returns: - true if the record was written; false if an error occurred - - ***************************************************************************/ - - override protected bool put ( cstring channel, hash_t key, in void[] value ) - { - // We need to dup value, as it is a slice to the neo connection's read - // buffer. - global_storage.getCreate(channel).put(key, value.dup); - return true; - } - - /*************************************************************************** - - Removes a single record from the storage engine. - - Params: - channel = channel to remove to - key = key of record to remove - - Returns: - true if the record was removed; false if an error occurred - - ***************************************************************************/ - - override protected bool remove ( cstring channel, hash_t key ) - { - global_storage.getCreate(channel).remove(key); - return true; - } -} diff --git a/src/fakedht/request/GetNumConnections.d b/src/fakedht/request/GetNumConnections.d index f5da110a..e7d4ebdc 100644 --- a/src/fakedht/request/GetNumConnections.d +++ b/src/fakedht/request/GetNumConnections.d @@ -28,7 +28,6 @@ import Protocol = dhtproto.node.request.GetNumConnections; public scope class GetNumConnections : Protocol.GetNumConnections { - import ocean.core.Enforce; import fakedht.mixins.RequestConstruction; /*************************************************************************** @@ -51,8 +50,7 @@ public scope class GetNumConnections : Protocol.GetNumConnections override protected NumConnectionsData getConnectionsData ( ) { - enforce(false, + assert (false, "GetNumConnections is not supported by the fake DHT node"); - return NumConnectionsData.init; } } diff --git a/src/fakedht/request/Listen.d b/src/fakedht/request/Listen.d index 8ff02fdb..031a0929 100644 --- a/src/fakedht/request/Listen.d +++ b/src/fakedht/request/Listen.d @@ -46,7 +46,6 @@ static this ( ) public scope class Listen : Protocol.Listen, DhtListener { - import ocean.core.Verify; import fakedht.mixins.RequestConstruction; /*************************************************************************** @@ -144,7 +143,7 @@ public scope class Listen : Protocol.Listen, DhtListener override protected bool getNextRecord( cstring channel_name, mstring key, out Const!(void)[] value ) { - verify(key.length == HashDigits); + assert (key.length == HashDigits); if (this.remaining_keys.length == 0) return false; diff --git a/src/fakedht/request/Redistribute.d b/src/fakedht/request/Redistribute.d index dd4582d2..3b60bb88 100644 --- a/src/fakedht/request/Redistribute.d +++ b/src/fakedht/request/Redistribute.d @@ -28,7 +28,6 @@ import Protocol = dhtproto.node.request.Redistribute; public scope class Redistribute : Protocol.Redistribute { - import ocean.core.Enforce; import dhtproto.node.request.params.RedistributeNode; import fakedht.mixins.RequestConstruction; @@ -45,13 +44,13 @@ public scope class Redistribute : Protocol.Redistribute override protected void adjustHashRange ( hash_t min, hash_t max ) { - enforce(false, "Not supported by fake DHT node"); + assert (false, "Not supported by fake DHT node"); } /**************************************************************************/ override protected void redistributeData ( RedistributeNode[] dataset ) { - enforce(false, "Not supported by fake DHT node"); + assert (false, "Not supported by fake DHT node"); } } diff --git a/src/turtle/env/Dht.d b/src/turtle/env/Dht.d index 039028e9..c062fc9d 100644 --- a/src/turtle/env/Dht.d +++ b/src/turtle/env/Dht.d @@ -22,7 +22,6 @@ import turtle.env.model.Node; import fakedht.DhtNode; import fakedht.Storage; -import ocean.core.Verify; import ocean.core.Test; import ocean.task.Scheduler; import ocean.task.util.Timer; @@ -51,8 +50,12 @@ public alias fakedht.Storage.MissingRecordException MissingRecordException; *******************************************************************************/ public Dht dht() +in +{ + assert (_dht !is null, "Must call `Dht.initialize` first"); +} +body { - verify(_dht !is null, "Must call `Dht.initialize` first"); return _dht; } @@ -484,7 +487,7 @@ public class Dht : Node!(DhtNode, "dht") override public AddrPort node_addrport ( ) { - verify(this.node !is null); + assert(this.node); AddrPort addrport; addrport.setAddress(this.node.node_item.Address);