Merge pull request #131 from muflihun/develop

2.3.0

.gitattributes

@@ -0,0 +1 @@
+*.css linguist-detectable=false

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Change Log
 
+## [2.3.0] - 24-03-2018
+### Extensions API
+- Added `reset` in `DispatchErrorExtension` which is called when previously failed is no longer failing
+
+### Updates
+- Added `stats dyn` and `stats queue`
+- Backup keys only visible with `--with-key` parameter
+- Max bulk items can be set to 5-500
+
+### Fixes
+- Dynamic buffer recovery check fix
+- Crash when rotating an invalid file
+
+### Internal Updates
+- Abstract server and I/O service
+- Use shared pointer to keep reference for the session in the handler
+
 ## [2.2.1] - 19-03-2018
 ### Extensions API
 - Added `LogExtension::Level` enum for readability of `LogExtension::Data::level`

CMakeLists.txt

@@ -11,8 +11,8 @@ option (use_boost "Use boost or standalone networking lib" OFF)
 option (old_toolchain "Is old toolchain" OFF)
 
 set (RESIDUE_MAJOR "2")
-set (RESIDUE_MINOR "2")
-set (RESIDUE_PATCH "1")
+set (RESIDUE_MINOR "3")
+set (RESIDUE_PATCH "0")
 set (RESIDUE_VERSION "${RESIDUE_MAJOR}.${RESIDUE_MINOR}.${RESIDUE_PATCH}")
 set (RESIDUE_NAME "Residue")
 

README.md

@@ -40,6 +40,8 @@ Some of the notable features are listed below
  * Encryption: All the incoming and outgoing data is automatically encrypted. Residue also has several layers and techniques to secure the requests and to prevent unauthorised application from logging without compromising the speed.
  * Compression: Residue server and official client libraries provide ability to compress the network packets which makes a big difference in amount of data transferred over the wire.
  * Speed is not compromised in any way and there is a lot of testing being done every day to ensure that logging server meets the speed requirements.
+ * Dynamic buffers allow logs to be queued up in the memory while there is some I/O error (e.g, disk full) and cleared as soon as disk errors are fixed
+ * Residue extensions allow certain code to be executed in case of specific events (e.g, log dispatch failure) - see [Extensions API](/docs/CONFIGURATION.md#extensions)
  * Reliability: We are actively working on making the server much more reliable from restarting from the crash to handling bad traffic.
  * Residue comes with various [tools](https://github.com/topics/residue-tools) for devops and [client libraries](https://github.com/topics/residue-client) for easy integration.
  * There are many more features available. Please feel free to download your copy of residue binary and give it a try.

docs/CLI_COMMANDS.md

@@ -71,9 +71,31 @@ Only check the schedule for this logger rotation
 ### `stats`
 Displays server stats and number of active sessions
 
-##### `list`
+#### `list`
 Lists for active sessions (received, sent and how long session has been active for and associated clients if registered)
 
+#### `dyn`
+List dynamic buffer status
+
+#### `queue`
+List processing queue status
+
+You can also run sampling on the queue by providing `sampling`. Thread will be stalled for the amount of seconds value you provide (valid value from 3 to 10)
+
+For example
+
+```
+stats queue sampling 6 --client-id unmanaged
+```
+
+It will list the speed of each queue, e.g,
+
+```
+Queue For: unmanaged     Active:  8376 Backlog:  7442 Speed:  5 items/s (incl. bulk)
+```
+
+Sampling is done only on a non-empty _active_ queue
+
 ##### `--client-id <client-id>`
 Filters stats for specified client. Some of the clients may not be listed as they're only registered when server receives anything from them.
 

samples/extensions/failure-notify/README.md

@@ -9,7 +9,9 @@ To install this extension simply add following lines in `extensions` config
         "module": "path/to/failure-notify.so",
         "description": "Notifies the recipients when log failures have hit the threshold",
         "config": {
+            "script": "send.sh",
             "threshold": 20000,
+            "repeat": false,
             "recipients": ["[email protected]"]
         }
     }

samples/extensions/failure-notify/failure-notify.cc

@@ -14,11 +14,16 @@ Extension::Result FailureNotify::execute(const DispatchErrorExtension::Data* con
         return {1, true};
     }
 
+    if (!(conf().get<bool>("repeat", true)) && m_sent) {
+        return {0, true};
+    }
+
     m_failureCount++;
 
     if (m_failureCount >= conf().get<unsigned long>("threshold", 200UL)) {
         notifyRecipients(data);
         m_failureCount = 0UL;
+        m_sent = true;
     }
 
     return {0, true};
@@ -33,11 +38,14 @@ void FailureNotify::notifyRecipients(const DispatchErrorExtension::Data* const d
             JsonDoc recipient(recipientNode);
             std::string email = recipient.as<std::string>("");
             writeLog("Notifying " + email);
-
-            ss << "echo 'Residue dispatch error " 
-               << std::strerror(data->errorNumber) << ". File: " 
-               << data->filename << "' | mail -s 'residue err' " << email;
+            ss << conf().get<std::string>("script", "send.sh") << " '" <<  std::strerror(data->errorNumber) << "' '" << data->filename << "' " << email;
             system(ss.str().c_str());
         }
     }
 }
+
+void FailureNotify::reset()
+{
+    m_failureCount = 0;
+    m_sent = false;
+}

samples/extensions/failure-notify/failure-notify.h

@@ -8,16 +8,18 @@ using namespace residue;
 class FailureNotify : public DispatchErrorExtension {
 public:
 
-	FailureNotify() : DispatchErrorExtension("failure-notify"), m_failureCount(0UL) 
+	FailureNotify() : DispatchErrorExtension("failure-notify"), m_failureCount(0UL), m_sent(false)
 	{
 
 	}
 
 	virtual ~FailureNotify() = default;
 
 	virtual Extension::Result execute(const DispatchErrorExtension::Data* const) override;
+	virtual void reset() override;
 private:
 	unsigned long m_failureCount;
+        bool m_sent;
 
 	void notifyRecipients(const DispatchErrorExtension::Data* const data);
 };

samples/extensions/failure-notify/send.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Residue dispatch error $1. File: $2" | mail -s 'residue err' $3

samples/keys/epri.pem

@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-256-CBC,6B8E2DD75947B469A061FC706E612069
+
+hLhpVBm/5npaIhoNDE4m6gLNpwUKd0PCPTq7s9BhPS6+2+vTS2XqrVQn/rH1qehZ
+Cqv8LQEiRVNQpZ6BXfr2GDWr/4QyeBVcuFAviAk7euT/l4zkUtqK91gqqvLevDvg
+0BQ799w921fu5dGw1MnTnz+PxqLveIzmE3fHa4MlUGXRk7FHn8uDI6CwI3w2fZG/
+LF1R86Dwk8TpZdXk0VUGmLrKrJms18ZD1q1jIUTzZVscWNQQaom1ODr2JVmqKyzP
+u06wucrBVyiyouZ4PMKam0R+zOn71kwQV4Y7mW0Ze4J9PNSMu7JZq0gl4uRH5Zvg
+387+HH3unb306QjiS2ZDpfLwg3qiGIAqxIrkDr3a5d0bVR3u6cTM0J+1V2aTvNp6
+oBDSkv4JgnPhvSxsI3EIOQs+7v6c3gY5BdykUEc6/1gTiIuixUEz+lqQ5M/8gdU6
+tHWu0JCVFO2iVAYbitWK6PM05ol47FZsQQUNFMOeJGGTG+o5bo7SuOllWVDBCzbt
+ZEdGMX42aCM8ZgSDLbHMyWH7MrRDnWZJq5ifE/hD5UP6n+MVNa7zG5OG70Fw1pQA
+DTlYPX7mMTO0VvkaGPmrJQ+rbqUhauVQl9ktlpd5d1qnVlWIoA5KiQgO2l9Wm9nm
+3ECmM7lNydWw/8/yW5S3DM3LwzGHaVBjBGcK4y5Qb87CPWlV9CeMVI4AfT3VERtl
+aoJiVzAj+ASz0/S/bid+9ezwIEnFat0O1wA+j+NIzd/s8zW+8/mJdo5qmIqC4+nX
+bcYulM8MgpoDjSYOBiext5aNlHfffCrqcLCdXsxgDHZjv1tHw61QO9VPIbcrLanS
+GEwgTGHY8QSmVcCr0nzvkSXyT2Ey5Rq0k+BluaX8nN3350qHw7nxiskysiREj/KB
+rkgrvA/ZEUcPSCOYTlcFPiwtTSVlYL7jXtTJlJ811DgBONrFFuFIkrwyr57jogEN
+RwCqQfa8pp4bjYKUsYpaRqHAF/tY29mMGqStGGF6/ekkJKLMlC0UCBVq5mxYnJOo
+9bqNCz3h5kXM6+k3O6ntp5+64btftNd78NEGiJSrm/LSE7lJNNknmGf0Ew2UqOD8
+6EFlkjoyNU76a3bwAVc+ltWx8gBe2Txlu1jTFa1UuZhIyoLPeKG2sh/bew32YfNm
+G9AfTezELYdPpta6aMko1u8tVk8ImbwARe83xgMVuF4OKHkA4KxXcq7hZmMZdinV
+wDEkLG9KUD7odFaF1wPnpOTeFlCX3MiXpTWLFDquoXFzfiy9XF24J3oEJ6hnZkuv
+r0lpjWjVCqEscHVLCqgvWbEqDhlINHyPXWbnCp3m/0ZmT8MkvOABCM5SDy/x6+qo
+fYpWv8gKFyb0aU9Kd3tEK2kNQnCJJrRJPbdy0I9A9Kg8BEaTDsbanpXOirPqgm+s
+ZI37WUpqyIbr2TRX8SRg3jbBhIcyfjRS5Qn3KA6UIo9SwN+XWChsdVU6xJ2Tlhy+
+5BPcqX4YWK+sXAvA6R8qgQ+KajZEi1Zutfpx73WSF0duSROdgNRpQvvP8QYNpRPU
+8xlHgRmDXeQ4r5Dzdfjh9RfONek+LYL1R4XMe/O6x8NkG6a40PViJrt3L3VUQPMh
+-----END RSA PRIVATE KEY-----

samples/keys/epub.pem

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIDANBgkqhkiG9w0BAQEFAAOCAQ0AMIIBCAKCAQEAkVl+xbOQqA8iSXWRNlYu
+8Mu1wo9YrGnZUBV52gvR+8QHrYZztDMsgOSjoecvVAPsCbd9h7v2ZJk7sllN/RVs
+QQXEjK0zAiASumvL3SWx9mqyFIESWdMNoWvGAQx7qmRv1+AMsgXVK9LrF7QNxXEm
+BGZdavRP1MT0E/EvmD5HZvpS329CaBXjTYTYSCH8N4kmh/WAzlVSfwI8Hs/Fh7cJ
+/7slaf2aHOi0ulhwXiM+Zt85/pbyAv77Zuysu/U3LJT0kNTTXZWEVWuBGX5Hazf9
+dUF9UQNatZjSiHvD7nzXOjvxAoP1cN3MfGhxF0nHBrpy6pJg93jcpsay5HmbdzTP
+SwIBEQ==
+-----END PUBLIC KEY-----

samples/residue.conf.json

@@ -36,7 +36,7 @@
         },
         {
             "client_id": "muflihun00102031",
-            "public_key": "$RESIDUE_HOME/samples/keys/encrypted-pubkey.pem",
+            "public_key": "$RESIDUE_HOME/samples/keys/epub.pem",
             "loggers": ["residue"]
         }
     ],
@@ -48,7 +48,8 @@
         },
         {
             "logger_id": "default",
-            "configuration_file": "$RESIDUE_HOME/samples/configurations/default-Logger.conf"
+            "configuration_file": "$RESIDUE_HOME/samples/configurations/default-Logger.conf",
+            "rotation_freq": "hourly"
         },
         {
             "logger_id": "sample-app",

src/cli/clients.cc

@@ -23,16 +23,15 @@
 
 #include "core/configuration.h"
 #include "core/registry.h"
-#include "logging/residue-log-dispatcher.h"
 #include "tasks/client-integrity-task.h"
 #include "utils/utils.h"
 
 using namespace residue;
 
 Clients::Clients(Registry* registry) :
     Command("clients",
-            "List, add or remove clients from the server configuration",
-            "clients [list] [clean]",
+            "List clients available on the server (dead or alive)",
+            "clients [list] [clean] [--with-key] [--with-token]",
             registry)
 {
 }
@@ -43,7 +42,7 @@ void Clients::execute(std::vector<std::string>&& params, std::ostringstream& res
         result << "Clients: " << registry()->clients().size() << std::endl;
     }
     if (hasParam(params, "list")) {
-        list(result, hasParam(params, "--with-key"));
+        list(result, &params);
     } else if (hasParam(params, "clean")) {
         if (registry()->clientIntegrityTask()->isExecuting()) {
             result << "\nAlready running, please try again later" << std::endl;;
@@ -54,27 +53,10 @@ void Clients::execute(std::vector<std::string>&& params, std::ostringstream& res
             registry()->clientIntegrityTask()->kickOff();
             result << "\nFinished client integrity task" << std::endl;
         }
-    } else if (hasParam(params, "checkdyn")) {
-        // check dynamic buffer
-        ResidueLogDispatcher* dispatcher = el::Helpers::logDispatchCallback<ResidueLogDispatcher>("ResidueLogDispatcher");
-        if (dispatcher != nullptr) {
-            if (dispatcher->m_dynamicBuffer.empty()) {
-                result << "Dynamic buffer is empty";
-            } else {
-                result << "Dynamic buffer information:\n";
-                for (auto& pair : dispatcher->m_dynamicBuffer) {
-                    result << "Logger: " << pair.second.logger->id() << "\t";
-                    result << "Filename: " << pair.first << "\t";
-                    result << "Items: " << pair.second.lines.size() << "\n";
-                }
-            }
-        } else {
-            result << "Could not extract dispatcher";
-        }
     }
 }
 
-void Clients::list(std::ostringstream& result, bool withKey) const
+void Clients::list(std::ostringstream& result, const std::vector<std::string>* paramsPtr) const
 {
     int i = 1;
     for (auto& c : registry()->clients()) {
@@ -83,11 +65,15 @@ void Clients::list(std::ostringstream& result, bool withKey) const
                << ", Age: " << (Utils::now() - c.second.dateCreated()) << "s"
                << ", Status: "
                << (!c.second.isAlive() ? "DEAD" : "ALIVE " + std::to_string(c.second.age() - (Utils::now() - c.second.dateCreated())) + "s");
-        if (withKey) {
-            result << ", Key: " << c.second.key();
+        if (hasParam(*paramsPtr, "--with-token")) {
+            result << ", Token: " << c.second.token();
         }
-        if (!c.second.backupKey().empty()) {
-            result << ", Backup Key: " << c.second.backupKey();
+        if (hasParam(*paramsPtr, "--with-key")) {
+            result << ", Key: " << c.second.key();
+
+            if (!c.second.backupKey().empty()) {
+                result << ", Backup Key: " << c.second.backupKey();
+            }
         }
         result << std::endl;
     }

src/cli/clients.h

@@ -39,7 +39,7 @@ class Clients final : public Command
     virtual void execute(std::vector<std::string>&&, std::ostringstream&, bool) const override;
 
 private:
-    void list(std::ostringstream&, bool withKey) const;
+    void list(std::ostringstream&, const std::vector<std::string>*) const;
 };
 }