Merge remote-tracking branch 'origin/devel' into netflow

CMakeLists.txt

@@ -14,16 +14,16 @@ endif()
 
 # Versions and other informations
 set(IPFIXCOL_VERSION_MAJOR 2)
-set(IPFIXCOL_VERSION_MINOR 0)
+set(IPFIXCOL_VERSION_MINOR 1)
 set(IPFIXCOL_VERSION_PATCH 0)
 set(IPFIXCOL_VERSION
     ${IPFIXCOL_VERSION_MAJOR}.${IPFIXCOL_VERSION_MINOR}.${IPFIXCOL_VERSION_PATCH})
 
 # Include modules
 include(CMakeModules/git_info.cmake)
+include(CMakeModules/install_dirs.cmake)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
-include(GNUInstallDirs)
 
 # Include custom FindXXX modules
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")

CMakeModules/install_dirs.cmake

@@ -0,0 +1,44 @@
+# The purpose of this file is to automatically determine install directories
+#
+# If no directories are defined, use GNU install directories by default.
+# However, in case of RPM build, install directories are typically passed
+# to CMake as definitions that overwrites the default paths.
+#
+
+include(GNUInstallDirs)
+
+# Binary directories
+set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR})
+
+# Library directories
+if (DEFINED LIB_INSTALL_DIR)
+    set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR})
+endif()
+
+# Include directories
+if (DEFINED INCLUDE_INSTALL_DIR)
+    set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR})
+endif()
+
+# System configuration
+if (DEFINED SYSCONF_INSTALL_DIR)
+    set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR})
+endif()
+
+# Share files (architecture independend data)
+if (DEFINED SHARE_INSTALL_PREFIX)
+    set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX})
+else()
+    set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR})
+endif()
+
+set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/")
+set(INSTALL_DIR_MAN  "${INSTALL_DIR_SHARE}/man/")
+set(INSTALL_DIR_DOC  "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/")
+

README.rst

@@ -35,8 +35,12 @@ network interface and a port. Multiple instances of these plugins can run concur
 
 **Output plugins** - store or forward your flows.
 
+- `FDS <src/plugins/output/fds>`_ - store all flows in FDS file format (efficient long-term storage)
 - `JSON <src/plugins/output/json>`_ - convert flow records to JSON and send/store them
-- `dummy <src/plugins/output/dummy>`_ - simple module example
+- `Viewer <src/plugins/output/viewer>`_ - convert IPFIX into plain text and print
+  it on standard output
+- `Time Check <src/plugins/output/timecheck>`_ - flow timestamp check
+- `Dummy <src/plugins/output/dummy>`_ - simple module example
 - `lnfstore <extra_plugins/output/lnfstore>`_ (*) - store all flows in nfdump compatible
   format for long-term preservation
 - `UniRec <extra_plugins/output/unirec>`_ (*)  - send flow records in UniRec format

extra_plugins/output/unirec/CMakeLists.txt

@@ -1,20 +1,20 @@
 cmake_minimum_required(VERSION 2.8.8)
-project(unirec)
+project(ipfixcol2-unirec-output)
 
 # Description of the project
 set(UNIREC_DESCRIPTION
     "Output plugin for IPFIXcol2 that sends flow records in UniRec format into NEMEA modules."
 )
 
-set(UNIREC_VERSION_MAJOR 1)
+set(UNIREC_VERSION_MAJOR 2)
 set(UNIREC_VERSION_MINOR 0)
 set(UNIREC_VERSION_PATCH 0)
 set(UNIREC_VERSION
     ${UNIREC_VERSION_MAJOR}.${UNIREC_VERSION_MINOR}.${UNIREC_VERSION_PATCH})
 
+include(CMakeModules/install_dirs.cmake)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
-include(GNUInstallDirs)
 # Include custom FindXXX modules
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
 
@@ -39,6 +39,12 @@ set(CMAKE_CXX_FLAGS          "${CMAKE_CXX_FLAGS} -fvisibility=hidden -std=gnu++1
 set(CMAKE_CXX_FLAGS_RELEASE  "-O2 -DNDEBUG")
 set(CMAKE_CXX_FLAGS_DEBUG    "-g -O0 -Wall -Wextra -pedantic")
 
+# Prepare SPEC file
+configure_file(
+    "${PROJECT_SOURCE_DIR}/ipfixcol2-unirec-output.spec.in"
+    "${PROJECT_BINARY_DIR}/ipfixcol2-unirec-output.spec"
+)
+
 # Header files for source code building
 include_directories(
     "${IPFIXCOL2_INCLUDE_DIRS}"  # IPFIXcol2 header files
@@ -67,11 +73,11 @@ target_link_libraries(unirec-output
 
 install(
     TARGETS unirec-output
-    LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/"
+    LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/"
 )
 install(
     FILES config/unirec-elements.txt
-    DESTINATION "${CMAKE_INSTALL_FULL_SYSCONFDIR}/ipfixcol2/"
+    DESTINATION "${INSTALL_DIR_SYSCONF}/ipfixcol2/"
 )
 
 if (ENABLE_DOC_MANPAGE)
@@ -92,6 +98,6 @@ if (ENABLE_DOC_MANPAGE)
 
     install(
         FILES "${DST_FILE}"
-        DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7"
+        DESTINATION "${INSTALL_DIR_MAN}/man7"
     )
 endif()

extra_plugins/output/unirec/CMakeModules/install_dirs.cmake

@@ -0,0 +1,44 @@
+# The purpose of this file is to automatically determine install directories
+#
+# If no directories are defined, use GNU install directories by default.
+# However, in case of RPM build, install directories are typically passed
+# to CMake as definitions that overwrites the default paths.
+#
+
+include(GNUInstallDirs)
+
+# Binary directories
+set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR})
+
+# Library directories
+if (DEFINED LIB_INSTALL_DIR)
+    set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR})
+endif()
+
+# Include directories
+if (DEFINED INCLUDE_INSTALL_DIR)
+    set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR})
+endif()
+
+# System configuration
+if (DEFINED SYSCONF_INSTALL_DIR)
+    set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR})
+else()
+    set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR})
+endif()
+
+# Share files (architecture independend data)
+if (DEFINED SHARE_INSTALL_PREFIX)
+    set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX})
+else()
+    set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR})
+endif()
+
+set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/")
+set(INSTALL_DIR_MAN  "${INSTALL_DIR_SHARE}/man/")
+set(INSTALL_DIR_DOC  "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/")
+

extra_plugins/output/unirec/README.rst

@@ -211,6 +211,8 @@ a line: ``"BYTES uint64 e0id1"``:
   the ``<uniRecFormat>`` element.
 - Second parameter specifies a data type of the UniRec field. List of all supported types is available
   in `UniRec documentation <https://github.com/CESNET/Nemea-Framework/tree/master/unirec>`_.
+  The plugin also supports additional virtual type "string_trimmed" that converts data to UniRec
+  string type, but only characters up to the first occurrence of '\\0' (excluding) are copied.
 - The third parameter is comma separated list of corresponding IPFIX Information Elements (IEs). In
   this case, "e0id1" means IPFIX IE with Private Enterprise ID 0 and IE ID 1 (which is
   "octetDeltaCount"). Instead of numeric identification an IE name in "<scope>:<name>" format

extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in

@@ -0,0 +1,47 @@
+Name:           ipfixcol2-unirec-output
+Version:        @UNIREC_VERSION@
+Release:        1%{?dist}
+Summary:        Plugin for converting IPFIX data to UniRec format.
+
+License:        BSD
+URL:            http://www.liberouter.org/
+Source0:        %{name}-%{version}.tar.gz
+Group:          Liberouter
+Vendor:         CESNET, z.s.p.o.
+Packager:
+
+BuildRoot:      %{_tmppath}/%{name}-%{version}-%{release}
+BuildRequires:  gcc >= 4.8, cmake >= 2.8.8, make
+BuildRequires:  ipfixcol2-devel, libfds-devel, python2-docutils
+BuildRequires:  libtrap-devel, unirec >= 2.3.0
+Requires:       libtrap >= 0.12.0, ipfixcol2 >= 2.0.0, libfds >= 0.1.0
+
+%description
+Plugin for converting IPFIX data to UniRec format.
+
+%prep
+%autosetup
+
+#%post
+#/sbin/ldconfig
+
+#%postun
+#/sbin/ldconfig
+
+# Build source code
+%build
+%cmake .
+make %{?_smp_mflags}
+
+# Perform installation into build directory
+%install
+make install DESTDIR=%{buildroot}
+
+%files
+%{_libdir}/ipfixcol2/*.so*
+%{_mandir}/man7/*.7.gz
+%config(noreplace) %{_sysconfdir}/ipfixcol2/unirec-elements.txt
+
+%changelog
+* Wed Nov  21 2018 Lukas Hutak <[email protected]> 2.0.0-1
+- Initial RPM release

extra_plugins/output/unirec/src/map.c

@@ -52,6 +52,8 @@
 #define DEF_SIZE 32
 /** Size of error buffer                 */
 #define ERR_SIZE 256
+/** Special type of UniRec string field that is trimmed before conversion from IPFIX */
+#define TYPE_STRING_TRIM "string_trimmed"
 
 /** Internal structure of the mapping database */
 struct map_s {
@@ -234,13 +236,14 @@ map_elem_get_internal(const char *elem)
  * \param[in] ur_name     UniRec name
  * \param[in] ur_type     UniRec type
  * \param[in] ur_type_str UniRec type string
+ * \param[in] cflags      Additional conversion flags
  * \param[in] ie_defs     Definition of IPFIX IEs (comma separated list of definitions)
  * \param[in] line_id Line ID (just for error messages)
  * \return #IPX_OK on success
  * \return #IPX_ERR_NOMEM or #IPX_ERR_FORMAT on failure
  */
 static int
-map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str,
+map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, uint32_t cflags,
     const char *ie_defs, size_t line_id)
 {
     char *defs_cpy = strdup(ie_defs);
@@ -264,6 +267,7 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str,
     rec.unirec.name = ur_name;
     rec.unirec.type = ur_type;
     rec.unirec.type_str = ur_type_str;
+    rec.unirec.flags = cflags;
 
     // Process IPFIX fields
     char *subsave_ptr = NULL;
@@ -323,6 +327,7 @@ map_load_line(map_t *map, const char *line, size_t line_id)
     int rc = IPX_OK;
     char *ur_name = NULL;
     char *ur_type_str = NULL;
+    uint32_t conv_flags = 0;
     ur_field_type_t ur_type;
 
     char *line_cpy = strdup(line);
@@ -356,7 +361,15 @@ map_load_line(map_t *map, const char *line, size_t line_id)
         goto end;
     }
 
-    int type = ur_get_field_type_from_str(token);
+    int type;
+    if (strcmp(token, TYPE_STRING_TRIM) == 0) {
+        // Special version of string
+        type = UR_TYPE_STRING;
+        conv_flags |= MAP_FLAGS_STR_TRIM;
+    } else {
+        type = ur_get_field_type_from_str(token);
+    }
+
     if (type == UR_E_INVALID_TYPE) {
         snprintf(map->err_buffer, ERR_SIZE, "Line %zu: Invalid type '%s' of UniRec field '%s'",
             line_id, token, ur_name);
@@ -382,7 +395,7 @@ map_load_line(map_t *map, const char *line, size_t line_id)
 
     ptrdiff_t offset = token - line_cpy;
     const char *ie_defs = line + offset;
-    rc = map_load_line_ie_defs(map, ur_name, ur_type, ur_type_str, ie_defs, line_id);
+    rc = map_load_line_ie_defs(map, ur_name, ur_type, ur_type_str, conv_flags, ie_defs, line_id);
 
 end:
     free(ur_type_str);

extra_plugins/output/unirec/src/map.h

@@ -61,6 +61,11 @@ enum MAP_SRC {
     MAP_SRC_INTERNAL_DBF
 };
 
+enum MAP_FLAGS {
+    /** Perform trim of an IPFIX string (convert only characters before the first '\0')  */
+    MAP_FLAGS_STR_TRIM = (1 << 0)
+};
+
 /** IPFIX-to-UniRec mapping record                    */
 struct map_rec {
     struct {
@@ -85,6 +90,11 @@ struct map_rec {
         ur_field_type_t type;
         /** Data type (string, for log!)              */
         char *type_str;
+        /**
+         * \brief Additional conversion flags
+         * \note See ::MAP_FLAGS
+         */
+        uint32_t flags;
     } unirec;
 };
 

extra_plugins/output/unirec/src/translator.c

@@ -316,6 +316,22 @@ translate_bytes(translator_t *trans, const struct translator_rec *rec,
     return 0;
 }
 
+/**
+ * \brief Convert IPFIX string to trimmed UniRec string
+ *
+ * The function will copy only characters up to the first occurrence of '\0' (excluding).
+ * \copydetails translate_uint()
+ */
+static int
+translator_string_trim(translator_t *trans, const struct translator_rec *rec,
+    const struct fds_drec_field *field)
+{
+    ur_field_id_t  ur_id = rec->unirec.id;
+    size_t copy_len = strnlen((const char *) field->data, field->size);
+    ur_set_var(trans->record.ur_tmplt, trans->record.data, ur_id, field->data, copy_len);
+    return 0;
+}
+
 /**
  * \brief Convert IPFIX boolean to UniRec char/(un)signed integer
  * \copydetails translate_uint()
@@ -699,6 +715,11 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec)
 
     switch (type_ur) {
     case UR_TYPE_STRING:
+        // String array
+        if (type_ipx == FDS_ET_STRING && (rec->unirec.flags & MAP_FLAGS_STR_TRIM) != 0) {
+            return translator_string_trim;
+        }
+        // Fall through
     case UR_TYPE_BYTES:
         // String/byte array
         if (type_ipx == FDS_ET_STRING || type_ipx == FDS_ET_OCTET_ARRAY) {

extra_plugins/output/unirec/src/unirecplugin.c

@@ -78,7 +78,7 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = {
     // Configuration flags (reserved for future use)
     .flags = 0,
     // Plugin version string (like "1.2.3")
-    .version = "2.0.0",
+    .version = "2.1.0",
     // Minimal IPFIXcol version string (like "1.2.3")
     .ipx_min = "2.0.0"
 };

include/CMakeLists.txt

@@ -16,6 +16,7 @@ set(SUB_HEADERS
     ipfixcol2/message_session.h
     ipfixcol2/plugins.h
     ipfixcol2/session.h
+    ipfixcol2/utils.h
     ipfixcol2/verbose.h
     "${PROJECT_BINARY_DIR}/include/ipfixcol2/api.h"
 )

include/ipfixcol2.h

@@ -63,6 +63,7 @@
 
 #include <ipfixcol2/plugins.h>
 #include <ipfixcol2/session.h>
+#include <ipfixcol2/utils.h>
 #include <ipfixcol2/verbose.h>