Merge pull request CESNET#33 from CESNET/hutak-fds-input

FDS input: initial version of FDS File reader

src/plugins/input/CMakeLists.txt

@@ -1,5 +1,6 @@
 # List of input plugins to build and install
 add_subdirectory(dummy)
-add_subdirectory(ipfix)
 add_subdirectory(tcp)
-add_subdirectory(udp)
+add_subdirectory(udp)
+add_subdirectory(ipfix)
+add_subdirectory(fds)

src/plugins/input/fds/Builder.cpp

@@ -0,0 +1,263 @@
+/**
+ * \file src/plugins/input/fds/Builder.cpp
+ * \author Lukas Hutak <[email protected]>
+ * \brief IPFIX Message builder (implementation)
+ * \date May 2020
+ */
+
+#include <arpa/inet.h>
+#include <cstdlib>
+
+#include "Builder.hpp"
+#include "Exception.hpp"
+
+
+Builder::Builder(uint16_t size)
+{
+    struct fds_ipfix_msg_hdr *hdr_ptr;
+
+    if (size < FDS_IPFIX_MSG_HDR_LEN) {
+        throw FDS_exception("[internal] Invalid size of a message to generate!");
+    }
+
+    m_msg.reset((uint8_t *) malloc(size));
+    if (!m_msg) {
+        throw FDS_exception("Memory allocation error " + std::string(__PRETTY_FUNCTION__));
+    }
+
+    // Fill the message header (size will be filled on release)
+    hdr_ptr = reinterpret_cast<fds_ipfix_msg_hdr *>(m_msg.get());
+    hdr_ptr->version = htons(FDS_IPFIX_VERSION);
+    hdr_ptr->odid = 0;
+    hdr_ptr->seq_num = 0;
+    hdr_ptr->export_time = 0;
+
+    // Positions
+    m_msg_alloc = size;
+    m_msg_valid = FDS_IPFIX_MSG_HDR_LEN;
+    m_set_offset = 0;
+    m_set_id = 0; // invalid
+}
+
+void
+Builder::resize(uint16_t size)
+{
+    uint8_t *new_ptr = (uint8_t *) realloc(m_msg.get(), size);
+    if (!new_ptr) {
+        throw FDS_exception("Memory allocation error " + std::string(__PRETTY_FUNCTION__));
+    }
+
+    m_msg.release(); // To avoid calling free()
+    m_msg.reset(new_ptr);
+    m_msg_alloc = size;
+
+    if (m_msg_valid > m_msg_alloc) {
+        // The message has been trimmed!
+        m_msg_valid = m_msg_alloc;
+    }
+
+    if (m_set_offset + FDS_IPFIX_SET_HDR_LEN > m_msg_alloc) {
+        // The current offset is out of range
+        m_set_offset = 0;
+        m_set_id = 0;
+    }
+}
+
+bool
+Builder::empty()
+{
+    return (!m_msg || m_msg_valid == FDS_IPFIX_MSG_HDR_LEN);
+}
+
+uint8_t *
+Builder::release()
+{
+    // Close the current set (if any)
+    fset_close();
+
+    // Update IPFIX Message header (size)
+    struct fds_ipfix_msg_hdr *hdr_ptr;
+    hdr_ptr = reinterpret_cast<fds_ipfix_msg_hdr *>(m_msg.get());
+    hdr_ptr->length = htons(m_msg_valid);
+
+    m_msg_alloc = 0;
+    m_msg_valid = 0;
+    return m_msg.release();
+}
+
+/**
+ * @brief Create a new Set
+ *
+ * The previous Set is always closed even if the ID is the same.
+ * @param[in] sid Set ID
+ * @throw FDS_exception if the Message is full and the Set cannot be created
+ */
+void
+Builder::fset_new(uint16_t sid)
+{
+    // Close the previous set (if any)
+    fset_close();
+
+    // Initialize a new IPFIX Set
+    if (FDS_IPFIX_SET_HDR_LEN > m_msg_alloc - m_msg_valid) {
+        throw FDS_exception("[internal] Insufficient space for Set in an IPFIX Message");
+    }
+
+    m_set_offset = m_msg_valid;
+    auto *set_ptr = reinterpret_cast<struct fds_ipfix_set_hdr *>(&m_msg.get()[m_set_offset]);
+    set_ptr->flowset_id = htons(sid);
+    m_msg_valid += FDS_IPFIX_SET_HDR_LEN;
+    m_set_size = FDS_IPFIX_SET_HDR_LEN;
+    m_set_id = sid;
+}
+
+/**
+ * @brief Close the current Set (if any)
+ */
+void
+Builder::fset_close()
+{
+    if (m_set_offset == 0) {
+        return;
+    }
+
+    auto *set_ptr = reinterpret_cast<struct fds_ipfix_set_hdr *>(&m_msg.get()[m_set_offset]);
+    set_ptr->length = htons(m_set_size);
+    m_set_offset = 0;
+    m_set_id = 0;
+}
+
+void
+Builder::set_etime(uint32_t time)
+{
+    if (!m_msg) {
+        throw FDS_exception("[internal] IPFIX Message is not allocated!");
+    }
+
+    // Update IPFIX Message header (export time)
+    struct fds_ipfix_msg_hdr *hdr_ptr;
+    hdr_ptr = reinterpret_cast<fds_ipfix_msg_hdr *>(m_msg.get());
+    hdr_ptr->export_time = htonl(time);
+}
+
+void
+Builder::set_odid(uint32_t odid)
+{
+    if (!m_msg) {
+        throw FDS_exception("[internal] IPFIX Message is not allocated!");
+    }
+
+    // Update IPFIX Message header (export time)
+    struct fds_ipfix_msg_hdr *hdr_ptr;
+    hdr_ptr = reinterpret_cast<fds_ipfix_msg_hdr *>(m_msg.get());
+    hdr_ptr->odid = htonl(odid);
+}
+
+void
+Builder::set_seqnum(uint32_t seq_num)
+{
+    if (!m_msg) {
+        throw FDS_exception("[internal] IPFIX Message is not allocated!");
+    }
+
+    // Update IPFIX Message header (export time)
+    struct fds_ipfix_msg_hdr *hdr_ptr;
+    hdr_ptr = reinterpret_cast<fds_ipfix_msg_hdr *>(m_msg.get());
+    hdr_ptr->seq_num = htonl(seq_num);
+}
+
+bool
+Builder::add_template(const struct fds_template *tmplt)
+{
+    uint16_t tmplt_len = tmplt->raw.length;
+    uint16_t size_req = tmplt_len;
+    uint16_t set_id;
+
+    switch (tmplt->type) {
+    case FDS_TYPE_TEMPLATE:
+        set_id = FDS_IPFIX_SET_TMPLT;
+        break;
+    case FDS_TYPE_TEMPLATE_OPTS:
+        set_id = FDS_IPFIX_SET_OPTS_TMPLT;
+        break;
+    default:
+        throw FDS_exception("[internal] Unexpected Template type cannot be used!");
+    }
+
+    if (m_set_offset == 0 || set_id != m_set_id) {
+        // New (Options) Template Set must be created
+        fset_close();
+        size_req += FDS_IPFIX_SET_HDR_LEN;
+    }
+
+    if (size_req > m_msg_alloc - m_msg_valid) {
+        // Unable to add
+        return false;
+    }
+
+    if (m_set_offset == 0) {
+        fset_new(set_id);
+    }
+
+    memcpy(&m_msg.get()[m_msg_valid], tmplt->raw.data, tmplt_len);
+    m_msg_valid += tmplt_len;
+    m_set_size += tmplt_len;
+    return true;
+}
+
+
+bool
+Builder::add_record(const struct fds_drec *rec)
+{
+    uint16_t size_req = rec->size;
+    if (m_set_offset == 0 || rec->tmplt->id != m_set_id) {
+        // New Data Set must be created
+        fset_close();
+        size_req += FDS_IPFIX_SET_HDR_LEN;
+    }
+
+    if (size_req > m_msg_alloc - m_msg_valid) {
+        // Unable to add
+        return false;
+    }
+
+    if (m_set_offset == 0) {
+        fset_new(rec->tmplt->id);
+    }
+
+    memcpy(&m_msg.get()[m_msg_valid], rec->data, rec->size);
+    m_msg_valid += rec->size;
+    m_set_size += rec->size;
+    return true;
+}
+
+bool
+Builder::add_withdrawals()
+{
+    struct fds_ipfix_trec *rec_ptr = nullptr;
+    uint16_t size_req = 2U * FDS_IPFIX_WDRL_ALLSET_LEN;
+
+    if (size_req > m_msg_alloc - m_msg_valid) {
+        return false;
+    }
+
+    // All Templates Withdrawal
+    fset_new(FDS_IPFIX_SET_TMPLT);
+    rec_ptr = reinterpret_cast<struct fds_ipfix_trec *>(&m_msg.get()[m_msg_valid]);
+    rec_ptr->template_id = htons(FDS_IPFIX_SET_TMPLT);
+    rec_ptr->count = htons(0);
+    m_msg_valid += 4U; // Only 4 bytes as specified in RFC 7011, 8.1
+    m_set_size += 4U;
+    fset_close();
+
+    // All Options Template Withdrawal
+    fset_new(FDS_IPFIX_SET_OPTS_TMPLT);
+    rec_ptr = reinterpret_cast<struct fds_ipfix_trec *>(&m_msg.get()[m_msg_valid]);
+    rec_ptr->template_id = htons(FDS_IPFIX_SET_OPTS_TMPLT);
+    rec_ptr->count = htons(0);
+    m_msg_valid += 4U; // Only 4 bytes as specified in RFC 7011, 8.1
+    m_set_size += 4U;
+    fset_close();
+
+    return true;
+}

src/plugins/input/fds/Builder.hpp

@@ -0,0 +1,119 @@
+/**
+ * \file src/plugins/input/fds/Builder.hpp
+ * \author Lukas Hutak <[email protected]>
+ * \brief IPFIX Message builder
+ * \date May 2020
+ */
+
+#ifndef FDS_BUILDER_HPP
+#define FDS_BUILDER_HPP
+
+#include <memory>
+#include <libfds.h>
+
+/// IPFIX Message builder
+class Builder {
+private:
+    /// Memory of IPFIX Message to generate (can be nullptr)
+    std::unique_ptr<uint8_t, decltype(&free)> m_msg = {nullptr, &free};
+    /// Allocated size (bytes)
+    uint16_t m_msg_alloc;
+    /// Filled size (bytes)
+    uint16_t m_msg_valid;
+
+    /// Currently edited Flow Set (zero == invalid)
+    uint16_t m_set_offset;
+    /// Set ID of the current Flow Set
+    uint16_t m_set_id;
+    /// Size of the current IPFIX Set
+    uint16_t m_set_size;
+
+    void
+    fset_new(uint16_t sid);
+    void
+    fset_close();
+
+public:
+    /**
+     * @brief Create an IPFIX Message generator
+     *
+     * By default, ODID, Sequence Number, and Export Time are set to zeros.
+     * @param[in] size    Maximal size of the message (allocation size)
+     */
+    Builder(uint16_t size);
+    ~Builder() = default;
+    Builder(Builder &&other) = default;
+
+    /**
+     * @brief Change maximal size of the message
+     *
+     * If the size is less than the size of the currently built message, the
+     * message is trimmed!
+     * @param[in] size Maximal size (i.e. allocation size)
+     */
+    void
+    resize(uint16_t size);
+    /**
+     * @brief Test if the builder contains an IPFIX Message without content
+     *
+     * @note The builder is also considered as empty after release().
+     * @return True/false
+     */
+    bool
+    empty();
+    /**
+     * @brief Release the generated IPFIX Message
+     * @warning After releasing, the class functions MUST NOT be used anymore!
+     * @return Pointer to the message (real size is part of the Message)
+     */
+    uint8_t *
+    release();
+
+    /**
+     * @brief Set Export Time of the IPFIX Message
+     * @param[in] time Export Time
+     */
+    void
+    set_etime(uint32_t time);
+    /**
+     * @brief Set Observation Domain ID (ODID) of the IPFIX Message
+     * @param[in] odid ODID
+     */
+    void
+    set_odid(uint32_t odid);
+    /**
+     * @brief Set Sequence Number of the IPFIX Message
+     * @param[in] seq_num Sequence Number
+     */
+    void
+    set_seqnum(uint32_t seq_num);
+
+    /**
+     * @brief Add an (Options) Template Record
+     * @param[in] tmplt IPFIX (Options) Template
+     * @return True, if the Template has been successfully added
+     * @return False, if the Message is already full
+     */
+    bool
+    add_template(const struct fds_template *tmplt);
+    /**
+     * @brief Add a Data Record
+     * @param[in] rec IPFIX Data Record
+     * @return True, if the Record has been successfully added
+     * @return False, if the Message is already full
+     */
+    bool
+    add_record(const struct fds_drec *rec);
+    /**
+     * @brief Add an All (Options) Template Withdrawals (only TCP, SCTP, and File sessions)
+     * @note
+     *   After calling the function, all previous (Options) Templates are considered to
+     *   be invalid.
+     * @return True, if the Withdrawals has been successfully added
+     * @return False, if the Message is already full
+     */
+    bool
+    add_withdrawals();
+};
+
+#endif // FDS_BUILDER_HPP