From 5dadbee8fe3c404312d9df01dad5bef34ee6c884 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Thu, 25 Oct 2018 08:52:24 +0200 Subject: [PATCH 01/22] CMake: added support for parsing install directories defined as parameters RPM build tools pass specifications of installation directories to CMake that must be used to properly install binary and configuration files. Manual processing is necessary because CMake module GNUInstallDirs ingores these parameters and provides different directories on old systems. --- CMakeLists.txt | 2 +- CMakeModules/install_dirs.cmake | 44 +++++++++++++++++++ pkg/CMakeLists.txt | 2 +- pkg/ipfixcol2.pc.in | 3 +- src/build_config.h.in | 4 +- src/plugins/input/dummy/CMakeLists.txt | 4 +- src/plugins/input/tcp/CMakeLists.txt | 6 +-- src/plugins/input/udp/CMakeLists.txt | 6 +-- .../intermediate/anonymization/CMakeLists.txt | 6 +-- src/plugins/output/dummy/CMakeLists.txt | 6 +-- src/plugins/output/json/CMakeLists.txt | 6 +-- 11 files changed, 66 insertions(+), 23 deletions(-) create mode 100644 CMakeModules/install_dirs.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f824c45..d50bf573 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,9 +21,9 @@ set(IPFIXCOL_VERSION # Include modules include(CMakeModules/git_info.cmake) +include(CMakeModules/install_dirs.cmake) include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) -include(GNUInstallDirs) # Include custom FindXXX modules list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") diff --git a/CMakeModules/install_dirs.cmake b/CMakeModules/install_dirs.cmake new file mode 100644 index 00000000..65f845a1 --- /dev/null +++ b/CMakeModules/install_dirs.cmake @@ -0,0 +1,44 @@ +# The purpose of this file is to automatically determine install directories +# +# If no directories are defined, use GNU install directories by default. +# However, in case of RPM build, install directories are typically passed +# to CMake as definitions that overwrites the default paths. +# + +include(GNUInstallDirs) + +# Binary directories +set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR}) + +# Library directories +if (DEFINED LIB_INSTALL_DIR) + set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR}) +else() + set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR}) +endif() + +# Include directories +if (DEFINED INCLUDE_INSTALL_DIR) + set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR}) +else() + set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR}) +endif() + +# System configuration +if (DEFINED SYSCONF_INSTALL_DIR) + set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR}) +else() + set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR}) +endif() + +# Share files (architecture independend data) +if (DEFINED SHARE_INSTALL_PREFIX) + set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX}) +else() + set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR}) +endif() + +set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/") +set(INSTALL_DIR_MAN "${INSTALL_DIR_SHARE}/man/") +set(INSTALL_DIR_DOC "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/") + diff --git a/pkg/CMakeLists.txt b/pkg/CMakeLists.txt index db1abd9a..9f3c4847 100644 --- a/pkg/CMakeLists.txt +++ b/pkg/CMakeLists.txt @@ -67,7 +67,7 @@ if (PKG_CONFIG_FOUND) @ONLY) install( FILES "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2.pc" - DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/pkgconfig/") + DESTINATION "${INSTALL_DIR_LIB}/pkgconfig/") endif() # Package builders diff --git a/pkg/ipfixcol2.pc.in b/pkg/ipfixcol2.pc.in index d0633e78..5a02f70a 100644 --- a/pkg/ipfixcol2.pc.in +++ b/pkg/ipfixcol2.pc.in @@ -1,5 +1,4 @@ -prefix=@CMAKE_INSTALL_PREFIX@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +includedir=@INSTALL_DIR_INCLUDE@ Name: @PROJECT_NAME@ Description: @IPFIXCOL_DESCRIPTION@ diff --git a/src/build_config.h.in b/src/build_config.h.in index ab352ec6..d1175e1b 100644 --- a/src/build_config.h.in +++ b/src/build_config.h.in @@ -68,13 +68,13 @@ /** \brief Path to the directory with configuration */ #define IPX_DEFAULT_CONFIG_DIR \ - "@CMAKE_INSTALL_FULL_SYSCONFDIR@/ipfixcol2/" + "@INSTALL_DIR_SYSCONF@/ipfixcol2/" /** \brief Path to default (if not specified) user configuration file */ #define IPX_DEFAULT_STARTUP_CONFIG \ IPX_DEFAULT_CONFIG_DIR "startup.xml" /** \brief Path to the directory with plugins */ #define IPX_DEFAULT_PLUGINS_DIR \ - "@CMAKE_INSTALL_FULL_LIBDIR@/ipfixcol2/" + "@INSTALL_DIR_LIB@/ipfixcol2/" /** \brief Compiler name and version */ #define IPX_BUILD_COMPILER \ diff --git a/src/plugins/input/dummy/CMakeLists.txt b/src/plugins/input/dummy/CMakeLists.txt index 8806d1d7..2e3f465f 100644 --- a/src/plugins/input/dummy/CMakeLists.txt +++ b/src/plugins/input/dummy/CMakeLists.txt @@ -7,5 +7,5 @@ add_library(dummy-input MODULE install( TARGETS dummy-input - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" -) \ No newline at end of file + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) diff --git a/src/plugins/input/tcp/CMakeLists.txt b/src/plugins/input/tcp/CMakeLists.txt index c817da60..6e743889 100644 --- a/src/plugins/input/tcp/CMakeLists.txt +++ b/src/plugins/input/tcp/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(tcp-input MODULE install( TARGETS tcp-input - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -23,6 +23,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/input/udp/CMakeLists.txt b/src/plugins/input/udp/CMakeLists.txt index ab132396..4ff410ee 100644 --- a/src/plugins/input/udp/CMakeLists.txt +++ b/src/plugins/input/udp/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(udp-input MODULE install( TARGETS udp-input - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -23,6 +23,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/intermediate/anonymization/CMakeLists.txt b/src/plugins/intermediate/anonymization/CMakeLists.txt index 10db29d5..8cf6dc77 100644 --- a/src/plugins/intermediate/anonymization/CMakeLists.txt +++ b/src/plugins/intermediate/anonymization/CMakeLists.txt @@ -11,7 +11,7 @@ add_library(anonymization-intermediate MODULE install( TARGETS anonymization-intermediate - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -27,6 +27,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/output/dummy/CMakeLists.txt b/src/plugins/output/dummy/CMakeLists.txt index 9f60a012..fa18064e 100644 --- a/src/plugins/output/dummy/CMakeLists.txt +++ b/src/plugins/output/dummy/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(dummy-output MODULE install( TARGETS dummy-output - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -23,6 +23,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/output/json/CMakeLists.txt b/src/plugins/output/json/CMakeLists.txt index 74175acc..1dd569c7 100644 --- a/src/plugins/output/json/CMakeLists.txt +++ b/src/plugins/output/json/CMakeLists.txt @@ -19,7 +19,7 @@ add_library(json-output MODULE install( TARGETS json-output - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -35,6 +35,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() From fefd6a184cbce643d8822ef696a78f4c27227a50 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Thu, 25 Oct 2018 10:48:23 +0200 Subject: [PATCH 02/22] Debian: fixed package building files --- pkg/deb/CMakeLists.txt | 4 ++-- pkg/deb/templates/changelog | 4 ++-- pkg/deb/templates/control.in | 4 ++-- pkg/deb/templates/ipfixcol.install | 1 - .../templates/{ipfixcol-dev.install => ipfixcol2-dev.install} | 0 pkg/deb/templates/ipfixcol2.install | 3 +++ 6 files changed, 9 insertions(+), 7 deletions(-) delete mode 100644 pkg/deb/templates/ipfixcol.install rename pkg/deb/templates/{ipfixcol-dev.install => ipfixcol2-dev.install} (100%) create mode 100644 pkg/deb/templates/ipfixcol2.install diff --git a/pkg/deb/CMakeLists.txt b/pkg/deb/CMakeLists.txt index 56361566..129af061 100644 --- a/pkg/deb/CMakeLists.txt +++ b/pkg/deb/CMakeLists.txt @@ -51,8 +51,8 @@ file(COPY "templates/changelog" "templates/rules" "templates/compat" - "templates/ipfixcol.install" - "templates/ipfixcol-dev.install" + "templates/ipfixcol2.install" + "templates/ipfixcol2-dev.install" DESTINATION "${DEB_CFG_DIR}" ) diff --git a/pkg/deb/templates/changelog b/pkg/deb/templates/changelog index 5cae7c6e..d9137276 100644 --- a/pkg/deb/templates/changelog +++ b/pkg/deb/templates/changelog @@ -1,5 +1,5 @@ -ipfixcol (2.0.0-1) unstable; urgency=low +ipfixcol2 (2.0.0-1) unstable; urgency=low * Initial release. - -- Lukas Hutak Mon, 19 Jun 2017 14:24:56 +0200 + -- Lukas Hutak Thu, 25 Oct 2018 10:17:56 +0200 diff --git a/pkg/deb/templates/control.in b/pkg/deb/templates/control.in index dde00e1d..1f21a7ac 100644 --- a/pkg/deb/templates/control.in +++ b/pkg/deb/templates/control.in @@ -4,8 +4,8 @@ Homepage: http://www.liberouter.org/ Section: net Priority: optional Standards-Version: 3.9.8 -Build-Depends: debhelper (>= 10), cmake (>= 2.8.8), doxygen, pkg-config, - libfds-dev +Build-Depends: debhelper (>= 10), cmake (>= 2.8.8), doxygen, make, + libfds-dev, gcc (>= 4.8), g++ (>= 4.8), pkg-config Package: @CPACK_PACKAGE_NAME@ Architecture: any diff --git a/pkg/deb/templates/ipfixcol.install b/pkg/deb/templates/ipfixcol.install deleted file mode 100644 index bc7b6898..00000000 --- a/pkg/deb/templates/ipfixcol.install +++ /dev/null @@ -1 +0,0 @@ -/usr/bin/ diff --git a/pkg/deb/templates/ipfixcol-dev.install b/pkg/deb/templates/ipfixcol2-dev.install similarity index 100% rename from pkg/deb/templates/ipfixcol-dev.install rename to pkg/deb/templates/ipfixcol2-dev.install diff --git a/pkg/deb/templates/ipfixcol2.install b/pkg/deb/templates/ipfixcol2.install new file mode 100644 index 00000000..765c78be --- /dev/null +++ b/pkg/deb/templates/ipfixcol2.install @@ -0,0 +1,3 @@ +/usr/bin/ +/usr/lib/*/ipfixcol2/lib*.so* +/usr/share/man/ From 2eaad5a4a16b610653b04c69335fc9a6a7d39346 Mon Sep 17 00:00:00 2001 From: Jan Kala Date: Tue, 13 Nov 2018 16:51:59 +0100 Subject: [PATCH 03/22] Viewer: Added module from old repository. Minor changes in output, mostly spaces and indentation has been fixed. --- src/plugins/output/CMakeLists.txt | 3 +- src/plugins/output/viewer/CMakeLists.txt | 31 ++ src/plugins/output/viewer/README.rst | 21 ++ src/plugins/output/viewer/Reader.c | 335 ++++++++++++++++++ src/plugins/output/viewer/Reader.h | 126 +++++++ src/plugins/output/viewer/config.c | 154 ++++++++ src/plugins/output/viewer/config.h | 71 ++++ .../viewer/doc/ipfixcol2-viewer-output.7.rst | 21 ++ src/plugins/output/viewer/viewer.c | 133 +++++++ 9 files changed, 894 insertions(+), 1 deletion(-) create mode 100644 src/plugins/output/viewer/CMakeLists.txt create mode 100644 src/plugins/output/viewer/README.rst create mode 100644 src/plugins/output/viewer/Reader.c create mode 100644 src/plugins/output/viewer/Reader.h create mode 100644 src/plugins/output/viewer/config.c create mode 100644 src/plugins/output/viewer/config.h create mode 100644 src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst create mode 100644 src/plugins/output/viewer/viewer.c diff --git a/src/plugins/output/CMakeLists.txt b/src/plugins/output/CMakeLists.txt index 5d3af5fb..0aafa523 100644 --- a/src/plugins/output/CMakeLists.txt +++ b/src/plugins/output/CMakeLists.txt @@ -1,3 +1,4 @@ # List of output plugin to build and install add_subdirectory(dummy) -add_subdirectory(json) \ No newline at end of file +add_subdirectory(json) +add_subdirectory(viewer) \ No newline at end of file diff --git a/src/plugins/output/viewer/CMakeLists.txt b/src/plugins/output/viewer/CMakeLists.txt new file mode 100644 index 00000000..5e0341f4 --- /dev/null +++ b/src/plugins/output/viewer/CMakeLists.txt @@ -0,0 +1,31 @@ +# Create a linkable module + +add_library(viewer-output MODULE + viewer.c + config.c + config.h + Reader.h + Reader.c +) + +install( + TARGETS viewer-output + LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-viewer-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-viewer-output.7") + + add_custom_command(TARGET viewer-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + ) +endif() \ No newline at end of file diff --git a/src/plugins/output/viewer/README.rst b/src/plugins/output/viewer/README.rst new file mode 100644 index 00000000..fbda0ee1 --- /dev/null +++ b/src/plugins/output/viewer/README.rst @@ -0,0 +1,21 @@ +Viewer (output plugin) +===================== + +The plugin provides a way how to observe functionality of collector. +Viewer module prints information about incoming IPFIX packets and data inside of them. + +Example configuration +--------------------- + +.. code-block:: xml + + + Viewer output + viewer + + + + +Parameters +---------- + diff --git a/src/plugins/output/viewer/Reader.c b/src/plugins/output/viewer/Reader.c new file mode 100644 index 00000000..77bc4da9 --- /dev/null +++ b/src/plugins/output/viewer/Reader.c @@ -0,0 +1,335 @@ +/** + * \file src/plugins/output/viewer/Reader.c + * \author Jan Kala + * \brief Viewer - output module for printing information about incoming packets on stdout (source file) + * \date 2018 + */ +/* Copyright (C) 2018 CESNET, z.s.p.o. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided with the +* distribution. +* 3. Neither the name of the Company nor the names of its contributors +* may be used to endorse or promote products derived from this +* software without specific prior written permission. +* +* ALTERNATIVELY, provided that this notice is retained in full, this +* product may be distributed under the terms of the GNU General Public +* License (GPL) version 2 or later, in which case the provisions +* of the GPL apply INSTEAD OF those given above. +* +* This software is provided ``as is'', and any express or implied +* warranties, including, but not limited to, the implied warranties of +* merchantability and fitness for a particular purpose are disclaimed. +* In no event shall the company or contributors be liable for any +* direct, indirect, incidental, special, exemplary, or consequential +* damages (including, but not limited to, procurement of substitute +* goods or services; loss of use, data, or profits; or business +* interruption) however caused and on any theory of liability, whether +* in contract, strict liability, or tort (including negligence or +* otherwise) arising in any way out of the use of this software, even +* if advised of the possibility of such damage. +* +*/ + +#include +#include +#include "Reader.h" +#include + +void read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr) { + + const struct fds_ipfix_msg_hdr *ipfix_msg_hdr; + ipfix_msg_hdr = (const struct fds_ipfix_msg_hdr*)ipx_msg_ipfix_get_packet(msg); + + if (ipfix_msg_hdr->length < FDS_IPFIX_MSG_HDR_LEN){ + return; + } + + // Print packet header + printf("## Message header\n"); + printf("\tversion:\t%"PRIu16"\n",ntohs(ipfix_msg_hdr->version)); + printf("\tlength:\t\t%"PRIu16"\n",ntohs(ipfix_msg_hdr->length)); + printf("\texport time:\t%"PRIu32"\n",ntohl(ipfix_msg_hdr->export_time)); + printf("\tsequence no.:\t%"PRIu32"\n",ntohl(ipfix_msg_hdr->seq_num)); + printf("\tODID:\t\t%"PRIu32"\n", ntohl(ipfix_msg_hdr->odid)); + + // Get number of sets + struct ipx_ipfix_set *sets; + size_t set_cnt; + ipx_msg_ipfix_get_sets(msg, &sets, &set_cnt); + + // Record counter of total records in IPFIX message + uint32_t rec_i = 0; + + // Iteration through all the sets + for (uint32_t i = 0; i < set_cnt; ++i){ + read_set(&sets[i], msg, iemgr, &rec_i); + } +} + +void read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr, uint32_t *rec_i) { + uint8_t *set_end = (uint8_t *)set->ptr + ntohs(set->ptr->length); + uint16_t set_id = ntohs(set->ptr->flowset_id); + + const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); + + printf("** Set header\n"); + printf("\tset ID: %"PRIu16"\n",set_id); + printf("\tlength: %"PRIu16"\n",ntohs(set->ptr->length)); + + // Template set + if (set_id == FDS_IPFIX_SET_TMPLT || set_id ==FDS_IPFIX_SET_OPTS_TMPLT){ + + struct fds_tset_iter tset_iter; + fds_tset_iter_init(&tset_iter, set->ptr); + + // Iteration through all templates in the set + while (fds_tset_iter_next(&tset_iter) == FDS_OK){ + // Read and print single template + read_template_set(&tset_iter, set_id,iemgr); + putchar('\n'); + } + return; + } + // Data set + if (set_id >= FDS_IPFIX_SET_MIN_DSET){ + struct ipx_ipfix_record *ipfix_rec = ipx_msg_ipfix_get_drec(msg,*rec_i); + + //All the records in the set has same template id, so we extract it from the first record and print it + printf("\ttemplate id: %"PRIu16"\n", ipfix_rec->rec.tmplt->id); + + // Iteration through the records which belongs to the current set + while ((ipfix_rec != NULL) && (ipfix_rec->rec.data < set_end) && (*rec_i < rec_cnt)){ + // Print record header + printf("-- Record [n.%"PRIu32" of %"PRIu32"]\n", *rec_i+1, rec_cnt); + + // Get the specific record and read all the fields + read_record(&ipfix_rec->rec, 1, iemgr); //add iemgr + putchar('\n'); + // Get the next record + (*rec_i)++; + ipfix_rec = ipx_msg_ipfix_get_drec(msg, *rec_i); + } + return; + } + + //Unknown set ID + printf("\t\n"); +} + +void read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const fds_iemgr_t *iemgr) { + enum fds_template_type type; + void *ptr; + switch (set_id){ + case FDS_IPFIX_SET_TMPLT: + type = FDS_TYPE_TEMPLATE; + ptr = tset_iter->ptr.trec; + break; + case FDS_IPFIX_SET_OPTS_TMPLT: + type = FDS_TYPE_TEMPLATE_OPTS; + ptr = tset_iter->ptr.opts_trec; + break; + default: + printf("\t\n"); + return; + } + // Filling the template structure with data from raw packet + uint16_t tmplt_size = tset_iter->size; + struct fds_template *tmplt; + if (fds_template_parse(type, ptr, &tmplt_size, &tmplt) != FDS_OK){ + printf("*Template parsing error*\n"); + fds_template_destroy(tmplt); + return; + } + + // Printing out the header + printf("-- Template header\n"); + printf("\ttemplate id: %"PRIu16"\n", tmplt->id); + printf("\tfield count: %"PRIu16"\n", tmplt->fields_cnt_total); + + // Using IEManager to fill the definitions of the fields in the template + if(fds_template_ies_define(tmplt, iemgr , false) != FDS_OK){ + printf("*Error while assigning element definitions in template*\n"); + fds_template_destroy(tmplt); + return; + } + + printf("\tfields:\n"); + // Iteration through the fields and printing them out + for (uint16_t i = 0; i < tmplt->fields_cnt_total ; ++i) { + struct fds_tfield current = tmplt->fields[i]; + printf("\t"); + printf("en:%*"PRIu32" ",WRITER_EN_SPACE, current.en); + printf("id:%*"PRIu16" ",WRITER_ID_SPACE, current.id); + printf("size:"); + // In case of variable length print keyword "var" + current.length == FDS_IPFIX_VAR_IE_LEN ? printf("%*s ", WRITER_SIZE_SPACE, "var") : printf("%*"PRIu16" ", WRITER_SIZE_SPACE,current.length); + + // Unknown field definition + if (current.def == NULL){ + printf("\n"); + } + // Known field definition + else { + printf("%*s ", WRITER_ORG_NAME_SPACE, current.def->scope->name); + printf("%s", current.def->name); + } + putchar('\n'); + } + fds_template_destroy(tmplt); +} + +void print_indent(unsigned int n){ + for (unsigned int i = 0; i < n; i++){ + putchar('\t'); + } +} + +void read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr) { + // Write info from header about the record template + print_indent(indent); + printf("field count: %"PRIu16"\n", rec->tmplt->fields_cnt_total); + print_indent(indent); + printf("data length: %"PRIu16"\n", rec->tmplt->data_length); + print_indent(indent); + printf("size : %"PRIu16"\n", rec->size); + + // Iterate through all the fields in record + struct fds_drec_iter iter; + fds_drec_iter_init(&iter, rec, 0); + + print_indent(indent); + printf("fields:\n"); + while (fds_drec_iter_next(&iter) != FDS_EOC) { + struct fds_drec_field field = iter.field; + read_field(&field,indent, iemgr, rec->snap, false); // add iemgr + } +} + +const char * fds_semantic2str(enum fds_ipfix_list_semantics semantic){ + switch (semantic){ + case (FDS_IPFIX_LIST_ALL_OF): + return "All of"; + case (FDS_IPFIX_LIST_EXACTLY_ONE_OF): + return "Exactly one of"; + case (FDS_IPFIX_LIST_ORDERED): + return "Ordered"; + case (FDS_IPFIX_LIST_NONE_OF): + return "None of"; + case (FDS_IPFIX_LIST_ONE_OR_MORE_OF): + return "One or more of"; + default: + return "Undefined"; + } +} + +void read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, const fds_tsnapshot_t *snap, bool in_basicList) { + // Write info from header about field + print_indent(indent); + if (!in_basicList) { + printf("en:%*" PRIu32 " id:%*" PRIu16" ", WRITER_EN_SPACE, field->info->en, WRITER_ID_SPACE, field->info->id); + } + + enum fds_iemgr_element_type type; + char *org; + char *field_name; + const char *unit; + + // Get the organisation name, field name and unit of the data. + if (field->info->def == NULL){ + type = FDS_ET_OCTET_ARRAY; + org = ""; + field_name = ""; + unit = ""; + } + else { + type = field->info->def->data_type; + org = field->info->def->scope->name; + field_name = field->info->def->name; + if (field->info->def->data_unit != FDS_EU_NONE){ + unit = fds_iemgr_unit2str(field->info->def->data_unit); + } + else{ + unit = ""; + } + } + + switch(type){ + case FDS_ET_BASIC_LIST: { + // Iteration through the basic list + bool did_read = false; + struct fds_blist_iter blist_it; + + printf("%*s %s", WRITER_ORG_NAME_SPACE, org, field_name); + fds_blist_iter_init(&blist_it,field, iemgr); + printf("%4s[semantic: %s]"," ",fds_semantic2str(blist_it.semantic)); // Semantic is known after initialization + + while (fds_blist_iter_next(&blist_it) == FDS_OK){ + if (!did_read){ + putchar('\n'); + did_read = true; + } + read_field(&blist_it.field,indent+1, iemgr, snap, true); + } + if (!did_read){ // if we didn't read a single field + printf("%4s : empty\n"," "); + } + return; + } + case FDS_ET_SUB_TEMPLATE_LIST: + case FDS_ET_SUB_TEMPLATE_MULTILIST: { + // Iteration through the subTemplate and subTemplateMulti lists + printf("%*s %s\n", WRITER_ORG_NAME_SPACE, org, field_name); + struct fds_stlist_iter stlist_iter; + print_indent(indent); + fds_stlist_iter_init(&stlist_iter, field, snap, 0); + printf("- semantic: %d\n",stlist_iter.semantic); + while (fds_stlist_iter_next(&stlist_iter) == FDS_OK){ + read_record(&stlist_iter.rec, indent+1, iemgr); + putchar('\n'); + } + putchar('\b'); + return; + } + default: + printf("%*s %-*s : ", WRITER_ORG_NAME_SPACE,org, WRITER_FIELD_NAME_SPACE, field_name); + } + + // Read and write the data from the field + char buffer[1024]; + int res = fds_field2str_be(field->data, field->size, type, buffer, sizeof(buffer)); + + if(res >= 0){ + // Conversion was successful + if (type == FDS_ET_STRING){ + printf("\"%s\"", buffer); + } + else { + printf("%s", buffer); + } + + if (*unit != 0) + printf(" %s", unit); + putchar('\n'); + + return; + } + else if (res == FDS_ERR_BUFFER){ + // Buffer too small + printf("\n"); + return; + } + else { + // Any other error + printf("*Invalid value*\n"); + return; + } +} + diff --git a/src/plugins/output/viewer/Reader.h b/src/plugins/output/viewer/Reader.h new file mode 100644 index 00000000..87ac2f6e --- /dev/null +++ b/src/plugins/output/viewer/Reader.h @@ -0,0 +1,126 @@ +/** + * \file src/plugins/output/viewer/Reader.h + * \author Jan Kala + * \brief Viewer - output module for printing information about incoming packets on stdout (header file) + * \date 2018 + */ +/* Copyright (C) 2018 CESNET, z.s.p.o. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided with the +* distribution. +* 3. Neither the name of the Company nor the names of its contributors +* may be used to endorse or promote products derived from this +* software without specific prior written permission. +* +* ALTERNATIVELY, provided that this notice is retained in full, this +* product may be distributed under the terms of the GNU General Public +* License (GPL) version 2 or later, in which case the provisions +* of the GPL apply INSTEAD OF those given above. +* +* This software is provided ``as is'', and any express or implied +* warranties, including, but not limited to, the implied warranties of +* merchantability and fitness for a particular purpose are disclaimed. +* In no event shall the company or contributors be liable for any +* direct, indirect, incidental, special, exemplary, or consequential +* damages (including, but not limited to, procurement of substitute +* goods or services; loss of use, data, or profits; or business +* interruption) however caused and on any theory of liability, whether +* in contract, strict liability, or tort (including negligence or +* otherwise) arising in any way out of the use of this software, even +* if advised of the possibility of such damage. +* +*/ +#ifndef IPFIXCOL_READER_H +#define IPFIXCOL_READER_H + +#include + +/** + * \brief spaces in output for Enterprise number field + */ +#define WRITER_EN_SPACE 8 +/** + * \brief Spaces in output for ID field + */ +#define WRITER_ID_SPACE 6 +/** + * \brief Spaces in output for size field + */ +#define WRITER_SIZE_SPACE 7 +/** + * \brief Spaces in output for Field name + */ +#define WRITER_FIELD_NAME_SPACE 35 +/** + * \brief Spaces in output for Organization name + */ +#define WRITER_ORG_NAME_SPACE 12 + +/** + * \brief Reads the data inside of ipfix message + * + * Function reads and prints header of the packet and then iterates through the records. + * Information printed from header of packet are: + * version, length, export time, sequence number and Observation Domain ID + * \param[in] msg ipfix message which will be printed + */ +void +read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr); + +/** + * \brief Reads data inside the single record of IPFIX message + * + * Reads and prints the header of the record and then iterates through the fields. + * Information printed from header of record are: + * Template ID and number of the fields inside the record + * \param[in] rec record which will be printed + */ +void +read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr); + +/** + * \brief Reads the data inside the field of IPFIX message + * + * Reads and prints all the information about the data in the field + * if the detailed definition is known, data are printed in readable format + * as well as the information about the data (organisation name and name of the data). + * Otherwise data are printed in the raw format (hexadecimal) + * In both cases Enterprise number and ID will be printed. + * \param[in] field Field which will be printed + */ +void +read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, const fds_tsnapshot_t *snap, bool in_basicList); + +/** + * \brief Reads set which contains template + * + * Reads template type of sets which has set id == 2 or set id == 3. Uses Information Element manager from + * parameters for parsing the data inside of the template. + * \param tset_iter Template iterator + * \param set_id ID of the whole set + * \param iemgr IE Manager + */ +void +read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const fds_iemgr_t *iemgr); + +/** + * \brief Reads single set + * + * Reads single set and determines what kind of set it is and which read_xxx function to use for reading the fields. + * + * \param sets_iter Sets iterator + * \param msg IPFix message + * \param iemgr IE Manager for parsing the fields + * \param rec_i Number of record + */ +void +read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr, uint32_t *rec_i); + +#endif //IPFIXCOL_READER_H diff --git a/src/plugins/output/viewer/config.c b/src/plugins/output/viewer/config.c new file mode 100644 index 00000000..8d35e89c --- /dev/null +++ b/src/plugins/output/viewer/config.c @@ -0,0 +1,154 @@ +/** + * \file src/plugins/output/viewer/config.c + * \author Jan Kala + * \brief Parser of an XML configuration for viewer module (source file) + * \date 2018 + */ + +/* Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include "config.h" + +/* + * + * ... + * + */ + +/** XML nodes */ +enum params_xml_nodes { + NODE_DELAY = 1 +}; + +/** Definition of the \ node */ +static const struct fds_xml_args args_params[] = { + FDS_OPTS_ROOT("params"), + FDS_OPTS_END +}; + +/** + * \brief Process \ node + * \param[in] ctx Plugin context + * \param[in] root XML context to process + * \param[in] cfg Parsed configuration + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT in case of failure + */ +static int +config_parser_root(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct instance_config *cfg) +{ + (void) ctx; + + const struct fds_xml_cont *content; + while(fds_xml_next(root, &content) != FDS_EOC) { + switch (content->id) { + case NODE_DELAY: + // Delay between messages [microseconds] + assert(content->type == FDS_OPTS_T_UINT); + cfg->sleep_time.tv_nsec = (content->val_uint % 1000000LL) * 1000LL; + cfg->sleep_time.tv_sec = content->val_uint / 1000000LL; + break; + default: + // Internal error + assert(false); + break; + } + } + + return IPX_OK; +} + +/** + * \brief Set default parameters of the configuration + * \param[in] cfg Configuration + */ +static void +config_default_set(struct instance_config *cfg) +{ + (void) cfg; +} + +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params) +{ + struct instance_config *cfg = calloc(1, sizeof(*cfg)); + if (!cfg) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + return NULL; + } + + // Set default parameters + config_default_set(cfg); + + // Create an XML parser + fds_xml_t *parser = fds_xml_create(); + if (!parser) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + free(cfg); + } + + if (fds_xml_set_args(parser, args_params) != IPX_OK) { + IPX_CTX_ERROR(ctx, "Failed to parse the description of an XML document!", '\0'); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + fds_xml_ctx_t *params_ctx = fds_xml_parse_mem(parser, params, true); + if (params_ctx == NULL) { + IPX_CTX_ERROR(ctx, "Failed to parse the configuration: %s", fds_xml_last_err(parser)); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + // Parse parameters + int rc = config_parser_root(ctx, params_ctx, cfg); + fds_xml_destroy(parser); + if (rc != IPX_OK) { + free(cfg); + return NULL; + } + + return cfg; +} + +void +config_destroy(struct instance_config *cfg) +{ + free(cfg); +} diff --git a/src/plugins/output/viewer/config.h b/src/plugins/output/viewer/config.h new file mode 100644 index 00000000..46f864ae --- /dev/null +++ b/src/plugins/output/viewer/config.h @@ -0,0 +1,71 @@ +/** + * \file src/plugins/output/dummy/config.h + * \author Jan Kala + * \brief Parser of an XML configuration for viewer module (header file) + * \date 2018 + */ + +/* Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef CONFIG_H +#define CONFIG_H + +#include +#include "stdint.h" + +/** Configuration of a instance of the dummy plugin */ +struct instance_config { + /** Sleep time */ + struct timespec sleep_time; +}; + +/** + * \brief Parse configuration of the plugin + * \param[in] ctx Instance context + * \param[in] params XML parameters + * \return Pointer to the parse configuration of the instance on success + * \return NULL if arguments are not valid or if a memory allocation error has occurred + */ +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params); + +/** + * \brief Destroy parsed configuration + * \param[in] cfg Parsed configuration + */ +void +config_destroy(struct instance_config *cfg); + +#endif // CONFIG_H diff --git a/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst new file mode 100644 index 00000000..31aeb871 --- /dev/null +++ b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst @@ -0,0 +1,21 @@ +======================== + ipfixcol2-dummy-output +======================== + +--------------------- +Dummy (output plugin) +--------------------- + +:Author: Lukáš Huták (lukas.hutak@cesnet.cz) +:Author: Petr Velan (petr.velan@cesnet.cz) +:Date: 2018-09-20 +:Copyright: Copyright © 2018 CESNET, z.s.p.o. +:Version: 2.0 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 diff --git a/src/plugins/output/viewer/viewer.c b/src/plugins/output/viewer/viewer.c new file mode 100644 index 00000000..2935790a --- /dev/null +++ b/src/plugins/output/viewer/viewer.c @@ -0,0 +1,133 @@ +/** + * \file src/plugins/output/viewer/viewer.c + * \author Jan Kala + * \brief Viewer output plugin for IPFIXcol 2 + * \date 2018 + */ + +/* Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include +#include + +#include "config.h" +#include "../../input/dummy/config.h" +#include "../../../core/message_ipfix.h" +#include "Reader.h" + +/** Plugin description */ +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin type + .type = IPX_PT_OUTPUT, + // Plugin identification name + .name = "viewer", + // Brief description of plugin + .dsc = "Output plugin for ptinting information about incoming IPFIX messages.", + // Configuration flags (reserved for future use) + .flags = 0, + // Plugin version string (like "1.2.3") + .version = "0.1.0", + // Minimal IPFIXcol version string (like "1.2.3") + .ipx_min = "2.0.0" +}; + +/** Instance */ +struct instance_data { + /** Parsed configuration of the instance */ + struct instance_config *config; +}; + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *params) +{ + // Create a private data + struct instance_data *data = calloc(1, sizeof(*data)); + if (!data) { + return IPX_ERR_DENIED; + } + + if ((data->config = config_parse(ctx, params)) == NULL) { + free(data); + return IPX_ERR_DENIED; + } + + ipx_ctx_private_set(ctx, data); + + // Subscribe to receive IPFIX messages and Transport Session events + uint16_t new_mask = IPX_MSG_IPFIX | IPX_MSG_SESSION; + ipx_ctx_subscribe(ctx, &new_mask, NULL); + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *cfg) +{ + (void) ctx; // Suppress warnings + + struct instance_data *data = (struct instance_data *) cfg; + config_destroy(data->config); + free(data); +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) +{ + (void) cfg; + //Check of the type + int type = ipx_msg_get_type(msg); + if (type != IPX_MSG_IPFIX) { + return IPX_OK; + } + + const fds_iemgr_t *iemgr = ipx_ctx_iemgr_get(ctx); + + //Convert the message to the IPFIX message and read it + ipx_msg_ipfix_t *ipfix_msg = ipx_msg_base2ipfix(msg); + read_packet(ipfix_msg, iemgr); + + + // This part of code is not used just because the delay was implemented but not used yet +// Delay between messages printing - can be setup in config file +// struct instance_data *data = (struct instance_data *) cfg; +// const struct timespec *delay = &data->config->sleep_time; +// if (delay->tv_sec != 0 || delay->tv_nsec != 0) { +// nanosleep(delay, NULL); +// } + + return IPX_OK; + +} From d4050813de63a801253bc31089ef766cd9256f94 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 21 Nov 2018 10:32:12 +0100 Subject: [PATCH 04/22] UniRec output: added SPEC file for RPM build, improved installation directories detection --- extra_plugins/output/unirec/CMakeLists.txt | 18 ++++--- .../unirec/CMakeModules/install_dirs.cmake | 44 +++++++++++++++++ .../unirec/ipfixcol2-unirec-output.spec.in | 47 +++++++++++++++++++ 3 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 extra_plugins/output/unirec/CMakeModules/install_dirs.cmake create mode 100644 extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in diff --git a/extra_plugins/output/unirec/CMakeLists.txt b/extra_plugins/output/unirec/CMakeLists.txt index 6ae51b28..5908e7ce 100644 --- a/extra_plugins/output/unirec/CMakeLists.txt +++ b/extra_plugins/output/unirec/CMakeLists.txt @@ -1,20 +1,20 @@ cmake_minimum_required(VERSION 2.8.8) -project(unirec) +project(ipfixcol2-unirec-output) # Description of the project set(UNIREC_DESCRIPTION "Output plugin for IPFIXcol2 that sends flow records in UniRec format into NEMEA modules." ) -set(UNIREC_VERSION_MAJOR 1) +set(UNIREC_VERSION_MAJOR 2) set(UNIREC_VERSION_MINOR 0) set(UNIREC_VERSION_PATCH 0) set(UNIREC_VERSION ${UNIREC_VERSION_MAJOR}.${UNIREC_VERSION_MINOR}.${UNIREC_VERSION_PATCH}) +include(CMakeModules/install_dirs.cmake) include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) -include(GNUInstallDirs) # Include custom FindXXX modules list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") @@ -39,6 +39,12 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -std=gnu++1 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic") +# Prepare SPEC file +configure_file( + "${PROJECT_SOURCE_DIR}/ipfixcol2-unirec-output.spec.in" + "${PROJECT_BINARY_DIR}/ipfixcol2-unirec-output.spec" +) + # Header files for source code building include_directories( "${IPFIXCOL2_INCLUDE_DIRS}" # IPFIXcol2 header files @@ -67,11 +73,11 @@ target_link_libraries(unirec-output install( TARGETS unirec-output - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) install( FILES config/unirec-elements.txt - DESTINATION "${CMAKE_INSTALL_FULL_SYSCONFDIR}/ipfixcol2/" + DESTINATION "${INSTALL_DIR_SYSCONF}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -92,6 +98,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) endif() diff --git a/extra_plugins/output/unirec/CMakeModules/install_dirs.cmake b/extra_plugins/output/unirec/CMakeModules/install_dirs.cmake new file mode 100644 index 00000000..65f845a1 --- /dev/null +++ b/extra_plugins/output/unirec/CMakeModules/install_dirs.cmake @@ -0,0 +1,44 @@ +# The purpose of this file is to automatically determine install directories +# +# If no directories are defined, use GNU install directories by default. +# However, in case of RPM build, install directories are typically passed +# to CMake as definitions that overwrites the default paths. +# + +include(GNUInstallDirs) + +# Binary directories +set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR}) + +# Library directories +if (DEFINED LIB_INSTALL_DIR) + set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR}) +else() + set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR}) +endif() + +# Include directories +if (DEFINED INCLUDE_INSTALL_DIR) + set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR}) +else() + set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR}) +endif() + +# System configuration +if (DEFINED SYSCONF_INSTALL_DIR) + set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR}) +else() + set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR}) +endif() + +# Share files (architecture independend data) +if (DEFINED SHARE_INSTALL_PREFIX) + set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX}) +else() + set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR}) +endif() + +set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/") +set(INSTALL_DIR_MAN "${INSTALL_DIR_SHARE}/man/") +set(INSTALL_DIR_DOC "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/") + diff --git a/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in b/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in new file mode 100644 index 00000000..71c348de --- /dev/null +++ b/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in @@ -0,0 +1,47 @@ +Name: ipfixcol2-unirec-output +Version: @UNIREC_VERSION@ +Release: 1%{?dist} +Summary: Plugin for converting IPFIX data to UniRec format. + +License: BSD +URL: http://www.liberouter.org/ +Source0: %{name}-%{version}.tar.gz +Group: Liberouter +Vendor: CESNET, z.s.p.o. +Packager: + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} +BuildRequires: gcc >= 4.8, cmake >= 2.8.8, make +BuildRequires: ipfixcol2-devel, libfds-devel, python2-docutils +BuildRequires: libtrap-devel, unirec >= 2.3.0 +Requires: libtrap >= 0.12.0, ipfixcol2 >= 2.0.0, libfds >= 0.1.0 + +%description +Plugin for converting IPFIX data to UniRec format. + +%prep +%autosetup + +#%post +#/sbin/ldconfig + +#%postun +#/sbin/ldconfig + +# Build source code +%build +%cmake . +make %{?_smp_mflags} + +# Perform installation into build directory +%install +make install DESTDIR=%{buildroot} + +%files +%{_libdir}/ipfixcol2/*.so* +%{_mandir}/man7/*.7.gz +%config(noreplace) %{_sysconfdir}/ipfixcol2/unirec-elements.txt + +%changelog +* Wed Nov 21 2018 Lukas Hutak 2.0.0-1 +- Initial RPM release From b60d756d2c5a251f96a1032e586794e7dc0dd319 Mon Sep 17 00:00:00 2001 From: Jan Kala Date: Tue, 4 Dec 2018 10:56:14 +0100 Subject: [PATCH 05/22] Viewer: edited output format for printing --- src/plugins/output/viewer/Reader.c | 66 +++++++++++-------- .../viewer/doc/ipfixcol2-viewer-output.7.rst | 11 ++-- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/src/plugins/output/viewer/Reader.c b/src/plugins/output/viewer/Reader.c index 77bc4da9..1595b082 100644 --- a/src/plugins/output/viewer/Reader.c +++ b/src/plugins/output/viewer/Reader.c @@ -101,6 +101,7 @@ void read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t // Data set if (set_id >= FDS_IPFIX_SET_MIN_DSET){ struct ipx_ipfix_record *ipfix_rec = ipx_msg_ipfix_get_drec(msg,*rec_i); + if (ipfix_rec == NULL) return; //All the records in the set has same template id, so we extract it from the first record and print it printf("\ttemplate id: %"PRIu16"\n", ipfix_rec->rec.tmplt->id); @@ -174,7 +175,7 @@ void read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const f // Unknown field definition if (current.def == NULL){ - printf("\n"); + printf(""); } // Known field definition else { @@ -195,18 +196,15 @@ void print_indent(unsigned int n){ void read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr) { // Write info from header about the record template print_indent(indent); - printf("field count: %"PRIu16"\n", rec->tmplt->fields_cnt_total); - print_indent(indent); - printf("data length: %"PRIu16"\n", rec->tmplt->data_length); - print_indent(indent); - printf("size : %"PRIu16"\n", rec->size); + printf("[templateID: %-*"PRIu16" ", 10, rec->tmplt->id); + printf("field count: %-*"PRIu16" ", 10, rec->tmplt->fields_cnt_total); + printf("data length: %-*"PRIu16" ", 10, rec->tmplt->data_length); + printf("size: %"PRIu16"]\n", 10, rec->size); // Iterate through all the fields in record struct fds_drec_iter iter; fds_drec_iter_init(&iter, rec, 0); - print_indent(indent); - printf("fields:\n"); while (fds_drec_iter_next(&iter) != FDS_EOC) { struct fds_drec_field field = iter.field; read_field(&field,indent, iemgr, rec->snap, false); // add iemgr @@ -234,7 +232,7 @@ void read_field(struct fds_drec_field *field, unsigned int indent, const fds_iem // Write info from header about field print_indent(indent); if (!in_basicList) { - printf("en:%*" PRIu32 " id:%*" PRIu16" ", WRITER_EN_SPACE, field->info->en, WRITER_ID_SPACE, field->info->id); + printf("en:%*"PRIu32" id:%*"PRIu16" ", WRITER_EN_SPACE, field->info->en, WRITER_ID_SPACE, field->info->id); } enum fds_iemgr_element_type type; @@ -264,38 +262,49 @@ void read_field(struct fds_drec_field *field, unsigned int indent, const fds_iem switch(type){ case FDS_ET_BASIC_LIST: { // Iteration through the basic list - bool did_read = false; struct fds_blist_iter blist_it; printf("%*s %s", WRITER_ORG_NAME_SPACE, org, field_name); fds_blist_iter_init(&blist_it,field, iemgr); - printf("%4s[semantic: %s]"," ",fds_semantic2str(blist_it.semantic)); // Semantic is known after initialization - - while (fds_blist_iter_next(&blist_it) == FDS_OK){ - if (!did_read){ - putchar('\n'); - did_read = true; - } + // Before printing the frist record, we need to print semantics of list + // only function _next returns the state of the iterator + int ret = fds_blist_iter_next(&blist_it); + if (ret == FDS_OK || ret == FDS_EOC){ + printf(" (semantic: %s[%d])", fds_semantic2str(blist_it.semantic), blist_it.semantic); + } + if (ret == FDS_OK) { + putchar('\n'); read_field(&blist_it.field,indent+1, iemgr, snap, true); + } else if(ret == FDS_EOC){ + printf(" - empty\n"); } - if (!did_read){ // if we didn't read a single field - printf("%4s : empty\n"," "); + while (fds_blist_iter_next(&blist_it) == FDS_OK){ + read_field(&blist_it.field,indent+1, iemgr, snap, true); } return; } case FDS_ET_SUB_TEMPLATE_LIST: case FDS_ET_SUB_TEMPLATE_MULTILIST: { // Iteration through the subTemplate and subTemplateMulti lists - printf("%*s %s\n", WRITER_ORG_NAME_SPACE, org, field_name); - struct fds_stlist_iter stlist_iter; + printf("%*s %s", WRITER_ORG_NAME_SPACE, org, field_name); + struct fds_stlist_iter stlist_it; print_indent(indent); - fds_stlist_iter_init(&stlist_iter, field, snap, 0); - printf("- semantic: %d\n",stlist_iter.semantic); - while (fds_stlist_iter_next(&stlist_iter) == FDS_OK){ - read_record(&stlist_iter.rec, indent+1, iemgr); + fds_stlist_iter_init(&stlist_it, field, snap, 0); + // Before printing the first record, we need to print the semantic of the list + // only _next function returns state of the iterator + int ret = fds_stlist_iter_next(&stlist_it); + if (ret == FDS_OK || ret == FDS_EOC){ + printf(" (semantic: %s[%d])", fds_semantic2str(stlist_it.semantic), stlist_it.semantic); + } + if (ret == FDS_OK) { putchar('\n'); + read_record(&stlist_it.rec, indent+1, iemgr); + } else if (ret == FDS_EOC){ + printf(" - empty\n"); + } + while (fds_stlist_iter_next(&stlist_it) == FDS_OK){ + read_record(&stlist_it.rec, indent+1, iemgr); } - putchar('\b'); return; } default: @@ -310,8 +319,9 @@ void read_field(struct fds_drec_field *field, unsigned int indent, const fds_iem // Conversion was successful if (type == FDS_ET_STRING){ printf("\"%s\"", buffer); - } - else { + } else if (type == FDS_ET_OCTET_ARRAY){ + printf("0x%s",buffer); + } else { printf("%s", buffer); } diff --git a/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst index 31aeb871..b9c6ea78 100644 --- a/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst +++ b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst @@ -1,17 +1,16 @@ ======================== - ipfixcol2-dummy-output + ipfixcol2-viewer-output ======================== --------------------- -Dummy (output plugin) +Viewer (output plugin) --------------------- -:Author: Lukáš Huták (lukas.hutak@cesnet.cz) -:Author: Petr Velan (petr.velan@cesnet.cz) -:Date: 2018-09-20 +:Author: Jan Kala (xkalaj01@stud.fit.vutbr.cz) +:Date: 2018-12-01 :Copyright: Copyright © 2018 CESNET, z.s.p.o. :Version: 2.0 -:Manual section: 7 +:Manual section: X :Manual group: IPFIXcol collector Description From e1ba5fd639c53106beeb4d74345a174a6aba32de Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 20 Feb 2019 17:36:44 +0100 Subject: [PATCH 06/22] Viewer: output format improvements, added new readme The plugin now dependes on ipfixcol v2.1.0 (requires list iterators provided by libfds v0.2.0) --- src/plugins/output/viewer/README.rst | 83 +++- src/plugins/output/viewer/Reader.c | 447 +++++++++++++----- src/plugins/output/viewer/Reader.h | 103 ++-- .../viewer/doc/ipfixcol2-viewer-output.7.rst | 11 +- src/plugins/output/viewer/viewer.c | 14 +- 5 files changed, 488 insertions(+), 170 deletions(-) diff --git a/src/plugins/output/viewer/README.rst b/src/plugins/output/viewer/README.rst index fbda0ee1..8a43c55d 100644 --- a/src/plugins/output/viewer/README.rst +++ b/src/plugins/output/viewer/README.rst @@ -1,8 +1,19 @@ Viewer (output plugin) ===================== -The plugin provides a way how to observe functionality of collector. -Viewer module prints information about incoming IPFIX packets and data inside of them. +The plugin converts IPFIX Messages into plain text and prints them on standard output. + +The main goal of the plugin is to show content of received IPFIX Messages in human readable form. +Each IPFIX Message is broken down into IPFIX Sets and each IPFIX Set is further broken down into +(Options) Template/Data records and so on. Fields of the Data records are formatted according +to the expected data type, if their corresponding Information Element definitions are known to +the collector. Therefore, the output can be also used to determine missing Information Element +definitions. + +Biflow records and structured data types are also supported and appropriately formatted. +Output is not supposed to be further parsed because its format can be changed in the future. +However, if you are interested into processing Data Records, consider using other +plugins such as JSON, UniRec, etc. Example configuration --------------------- @@ -12,10 +23,74 @@ Example configuration Viewer output viewer - - + Parameters ---------- +Parameters are not supported by the plugin. + +Example output +-------------- + +Below you can see example output for anonymized IPFIX Messages. The message structure and +available fields of Data Record vary from exporter to exporter. + +.. code-block:: + + IPFIX Message header: + Version: 10 + Length: 176 + Export time: 1455534348 + Sequence no.: 0 + ODID: 1 + + Set Header: + Set ID: 2 (Template Set) + Length: 160 + Template Record (#1) + Template ID: 256 + Field Count: 27 + EN: 0 ID: 1 Size: 4 | iana:octetDeltaCount + EN: 0 ID: 2 Size: 4 | iana:packetDeltaCount + EN: 0 ID: 152 Size: 8 | iana:flowStartMilliseconds + EN: 0 ID: 153 Size: 8 | iana:flowEndMilliseconds + EN: 0 ID: 10 Size: 2 | iana:ingressInterface + EN: 0 ID: 60 Size: 1 | iana:ipVersion + EN: 0 ID: 8 Size: 4 | iana:sourceIPv4Address + EN: 0 ID: 12 Size: 4 | iana:destinationIPv4Address + EN: 0 ID: 5 Size: 1 | iana:ipClassOfService + EN: 0 ID: 192 Size: 1 | iana:ipTTL + EN: 0 ID: 4 Size: 1 | iana:protocolIdentifier + EN: 0 ID: 7 Size: 2 | iana:sourceTransportPort + EN: 0 ID: 11 Size: 2 | iana:destinationTransportPort + ... <~ output shortened for clarity ~> ... + + ------------------------------------------------------------------- + IPFIX Message header: + Version: 10 + Length: 216 + Export time: 1455534348 + Sequence no.: 0 + ODID: 1 + + Set Header: + Set ID: 256 (Data Set) + Length: 200 + Template ID: 256 + Data Record (#1) [Length: 95]: + EN: 0 ID: 1 iana:octetDeltaCount : 76 octets + EN: 0 ID: 2 iana:packetDeltaCount : 1 packets + EN: 0 ID: 152 iana:flowStartMilliseconds : 2016-02-15T11:05:48.150Z + EN: 0 ID: 153 iana:flowEndMilliseconds : 2016-02-15T11:05:48.150Z + EN: 0 ID: 10 iana:ingressInterface : 1 + EN: 0 ID: 60 iana:ipVersion : 4 + EN: 0 ID: 8 iana:sourceIPv4Address : 209.246.218.165 + EN: 0 ID: 12 iana:destinationIPv4Address : 93.113.168.59 + EN: 0 ID: 5 iana:ipClassOfService : 0 + EN: 0 ID: 192 iana:ipTTL : 122 hops + EN: 0 ID: 4 iana:protocolIdentifier : 17 + EN: 0 ID: 7 iana:sourceTransportPort : 62299 + EN: 0 ID: 11 iana:destinationTransportPort : 53 + ... <~ output shortened for clarity ~> ... diff --git a/src/plugins/output/viewer/Reader.c b/src/plugins/output/viewer/Reader.c index 1595b082..0310b2eb 100644 --- a/src/plugins/output/viewer/Reader.c +++ b/src/plugins/output/viewer/Reader.c @@ -1,6 +1,7 @@ /** * \file src/plugins/output/viewer/Reader.c * \author Jan Kala + * \author Lukas Hutak * \brief Viewer - output module for printing information about incoming packets on stdout (source file) * \date 2018 */ @@ -43,8 +44,9 @@ #include "Reader.h" #include -void read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr) { - +void +read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr) +{ const struct fds_ipfix_msg_hdr *ipfix_msg_hdr; ipfix_msg_hdr = (const struct fds_ipfix_msg_hdr*)ipx_msg_ipfix_get_packet(msg); @@ -53,12 +55,13 @@ void read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr) { } // Print packet header - printf("## Message header\n"); - printf("\tversion:\t%"PRIu16"\n",ntohs(ipfix_msg_hdr->version)); - printf("\tlength:\t\t%"PRIu16"\n",ntohs(ipfix_msg_hdr->length)); - printf("\texport time:\t%"PRIu32"\n",ntohl(ipfix_msg_hdr->export_time)); - printf("\tsequence no.:\t%"PRIu32"\n",ntohl(ipfix_msg_hdr->seq_num)); - printf("\tODID:\t\t%"PRIu32"\n", ntohl(ipfix_msg_hdr->odid)); + printf("--------------------------------------------------------------------------------\n"); + printf("IPFIX Message header:\n"); + printf("\tVersion: %"PRIu16"\n",ntohs(ipfix_msg_hdr->version)); + printf("\tLength: %"PRIu16"\n",ntohs(ipfix_msg_hdr->length)); + printf("\tExport time: %"PRIu32"\n",ntohl(ipfix_msg_hdr->export_time)); + printf("\tSequence no.: %"PRIu32"\n",ntohl(ipfix_msg_hdr->seq_num)); + printf("\tODID: %"PRIu32"\n", ntohl(ipfix_msg_hdr->odid)); // Get number of sets struct ipx_ipfix_set *sets; @@ -74,46 +77,63 @@ void read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr) { } } -void read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr, uint32_t *rec_i) { +void +read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr, uint32_t *rec_i) +{ uint8_t *set_end = (uint8_t *)set->ptr + ntohs(set->ptr->length); uint16_t set_id = ntohs(set->ptr->flowset_id); const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); + const char *set_type = ""; + if (set_id == FDS_IPFIX_SET_TMPLT) { + set_type = "Template Set"; + } else if (set_id == FDS_IPFIX_SET_OPTS_TMPLT) { + set_type = "Options Template Set"; + } else if (set_id >= FDS_IPFIX_SET_MIN_DSET) { + set_type = "Data Set"; + } - printf("** Set header\n"); - printf("\tset ID: %"PRIu16"\n",set_id); - printf("\tlength: %"PRIu16"\n",ntohs(set->ptr->length)); - - // Template set - if (set_id == FDS_IPFIX_SET_TMPLT || set_id ==FDS_IPFIX_SET_OPTS_TMPLT){ + printf("\n"); + printf("Set Header:\n"); + printf("\tSet ID: %"PRIu16" (%s)\n", set_id, set_type); + printf("\tLength: %"PRIu16"\n", ntohs(set->ptr->length)); + if (set_id == FDS_IPFIX_SET_TMPLT || set_id == FDS_IPFIX_SET_OPTS_TMPLT) { + // Template set struct fds_tset_iter tset_iter; fds_tset_iter_init(&tset_iter, set->ptr); + const char *rec_fmt = (set_id == FDS_IPFIX_SET_TMPLT) + ? "- Template Record (#%u)\n" + : "- Options Template Record (#%u)\n"; + unsigned int rec_cnt = 0; // Iteration through all templates in the set while (fds_tset_iter_next(&tset_iter) == FDS_OK){ // Read and print single template + printf(rec_fmt, ++rec_cnt); read_template_set(&tset_iter, set_id,iemgr); putchar('\n'); } return; } - // Data set - if (set_id >= FDS_IPFIX_SET_MIN_DSET){ - struct ipx_ipfix_record *ipfix_rec = ipx_msg_ipfix_get_drec(msg,*rec_i); + + if (set_id >= FDS_IPFIX_SET_MIN_DSET) { + // Data set + struct ipx_ipfix_record *ipfix_rec = ipx_msg_ipfix_get_drec(msg, *rec_i); if (ipfix_rec == NULL) return; - //All the records in the set has same template id, so we extract it from the first record and print it - printf("\ttemplate id: %"PRIu16"\n", ipfix_rec->rec.tmplt->id); + // All the records in the set has same template id, so we extract it from the first record and print it + printf("\tTemplate ID: %"PRIu16"\n", ipfix_rec->rec.tmplt->id); + unsigned int iter_cnt = 0; // Iteration through the records which belongs to the current set - while ((ipfix_rec != NULL) && (ipfix_rec->rec.data < set_end) && (*rec_i < rec_cnt)){ + while ((ipfix_rec != NULL) && (ipfix_rec->rec.data < set_end) && (*rec_i < rec_cnt)) { // Print record header - printf("-- Record [n.%"PRIu32" of %"PRIu32"]\n", *rec_i+1, rec_cnt); - + printf("- Data Record (#%u) [Length: %"PRIu16"]:\n", ++iter_cnt, ipfix_rec->rec.size); // Get the specific record and read all the fields - read_record(&ipfix_rec->rec, 1, iemgr); //add iemgr + read_record(&ipfix_rec->rec, 1, iemgr); putchar('\n'); + // Get the next record (*rec_i)++; ipfix_rec = ipx_msg_ipfix_get_drec(msg, *rec_i); @@ -125,7 +145,9 @@ void read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t printf("\t\n"); } -void read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const fds_iemgr_t *iemgr) { +void +read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const fds_iemgr_t *iemgr) +{ enum fds_template_type type; void *ptr; switch (set_id){ @@ -146,14 +168,15 @@ void read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const f struct fds_template *tmplt; if (fds_template_parse(type, ptr, &tmplt_size, &tmplt) != FDS_OK){ printf("*Template parsing error*\n"); - fds_template_destroy(tmplt); return; } // Printing out the header - printf("-- Template header\n"); - printf("\ttemplate id: %"PRIu16"\n", tmplt->id); - printf("\tfield count: %"PRIu16"\n", tmplt->fields_cnt_total); + printf("\tTemplate ID: %"PRIu16"\n", tmplt->id); + printf("\tField Count: %"PRIu16"\n", tmplt->fields_cnt_total); + if (type == FDS_TYPE_TEMPLATE_OPTS) { + printf("\tScope Field Count: %"PRIu16"\n", tmplt->fields_cnt_scope); + } // Using IEManager to fill the definitions of the fields in the template if(fds_template_ies_define(tmplt, iemgr , false) != FDS_OK){ @@ -162,56 +185,65 @@ void read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const f return; } - printf("\tfields:\n"); // Iteration through the fields and printing them out for (uint16_t i = 0; i < tmplt->fields_cnt_total ; ++i) { struct fds_tfield current = tmplt->fields[i]; printf("\t"); - printf("en:%*"PRIu32" ",WRITER_EN_SPACE, current.en); - printf("id:%*"PRIu16" ",WRITER_ID_SPACE, current.id); - printf("size:"); + printf("EN: %-*"PRIu32" ", WRITER_EN_SPACE, current.en); + printf("ID: %-*"PRIu16" ", WRITER_ID_SPACE, current.id); + printf("Size: "); // In case of variable length print keyword "var" - current.length == FDS_IPFIX_VAR_IE_LEN ? printf("%*s ", WRITER_SIZE_SPACE, "var") : printf("%*"PRIu16" ", WRITER_SIZE_SPACE,current.length); - - // Unknown field definition - if (current.def == NULL){ - printf(""); + current.length == FDS_IPFIX_VAR_IE_LEN + ? printf("%-*s ", WRITER_SIZE_SPACE, "var.") + : printf("%-*"PRIu16" ", WRITER_SIZE_SPACE, current.length); + + const char *pen_name = ""; + const char *field_name = ""; + if (current.def != NULL) { + // Known definition + pen_name = current.def->scope->name; + field_name = current.def->name; + } else { + // Field is unknown ... try to find at least vendor + const struct fds_iemgr_scope *scope_ptr = fds_iemgr_scope_find_pen(iemgr, current.en); + if (scope_ptr != NULL) { + pen_name = scope_ptr->name; + } } - // Known field definition - else { - printf("%*s ", WRITER_ORG_NAME_SPACE, current.def->scope->name); - printf("%s", current.def->name); + + printf("| %*s:%s", WRITER_ORG_NAME_SPACE, pen_name, field_name); + if ((current.flags & FDS_TFIELD_SCOPE) != 0) { + printf(" (scope)"); } putchar('\n'); } fds_template_destroy(tmplt); } -void print_indent(unsigned int n){ +void +print_indent(unsigned int n) +{ for (unsigned int i = 0; i < n; i++){ putchar('\t'); } } -void read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr) { - // Write info from header about the record template - print_indent(indent); - printf("[templateID: %-*"PRIu16" ", 10, rec->tmplt->id); - printf("field count: %-*"PRIu16" ", 10, rec->tmplt->fields_cnt_total); - printf("data length: %-*"PRIu16" ", 10, rec->tmplt->data_length); - printf("size: %"PRIu16"]\n", 10, rec->size); - +void +read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr) +{ // Iterate through all the fields in record struct fds_drec_iter iter; - fds_drec_iter_init(&iter, rec, 0); + fds_drec_iter_init(&iter, rec, FDS_DREC_PADDING_SHOW); while (fds_drec_iter_next(&iter) != FDS_EOC) { struct fds_drec_field field = iter.field; - read_field(&field,indent, iemgr, rec->snap, false); // add iemgr + read_field(&field, indent, iemgr, rec->snap); } } -const char * fds_semantic2str(enum fds_ipfix_list_semantics semantic){ +const char * +fds_semantic2str(enum fds_ipfix_list_semantics semantic) +{ switch (semantic){ case (FDS_IPFIX_LIST_ALL_OF): return "All of"; @@ -228,110 +260,90 @@ const char * fds_semantic2str(enum fds_ipfix_list_semantics semantic){ } } -void read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, const fds_tsnapshot_t *snap, bool in_basicList) { +void +read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ // Write info from header about field print_indent(indent); - if (!in_basicList) { - printf("en:%*"PRIu32" id:%*"PRIu16" ", WRITER_EN_SPACE, field->info->en, WRITER_ID_SPACE, field->info->id); - } + printf("EN: %-*"PRIu32" ID: %-*"PRIu16" ", WRITER_EN_SPACE, field->info->en, + WRITER_ID_SPACE, field->info->id); enum fds_iemgr_element_type type; - char *org; - char *field_name; + const char *org; + const char *field_name; const char *unit; // Get the organisation name, field name and unit of the data. - if (field->info->def == NULL){ + if (field->info->def == NULL) { type = FDS_ET_OCTET_ARRAY; - org = ""; field_name = ""; + org = ""; unit = ""; - } - else { + + // Try to find the scope + const struct fds_iemgr_scope *scope_ptr = fds_iemgr_scope_find_pen(iemgr, field->info->en); + if (scope_ptr != NULL) { + org = scope_ptr->name; + } + } else { type = field->info->def->data_type; org = field->info->def->scope->name; field_name = field->info->def->name; - if (field->info->def->data_unit != FDS_EU_NONE){ + if (field->info->def->data_unit != FDS_EU_NONE) { unit = fds_iemgr_unit2str(field->info->def->data_unit); - } - else{ + } else { unit = ""; } } - switch(type){ - case FDS_ET_BASIC_LIST: { - // Iteration through the basic list - struct fds_blist_iter blist_it; - - printf("%*s %s", WRITER_ORG_NAME_SPACE, org, field_name); - fds_blist_iter_init(&blist_it,field, iemgr); - // Before printing the frist record, we need to print semantics of list - // only function _next returns the state of the iterator - int ret = fds_blist_iter_next(&blist_it); - if (ret == FDS_OK || ret == FDS_EOC){ - printf(" (semantic: %s[%d])", fds_semantic2str(blist_it.semantic), blist_it.semantic); - } - if (ret == FDS_OK) { - putchar('\n'); - read_field(&blist_it.field,indent+1, iemgr, snap, true); - } else if(ret == FDS_EOC){ - printf(" - empty\n"); - } - while (fds_blist_iter_next(&blist_it) == FDS_OK){ - read_field(&blist_it.field,indent+1, iemgr, snap, true); - } - return; - } - case FDS_ET_SUB_TEMPLATE_LIST: - case FDS_ET_SUB_TEMPLATE_MULTILIST: { - // Iteration through the subTemplate and subTemplateMulti lists - printf("%*s %s", WRITER_ORG_NAME_SPACE, org, field_name); - struct fds_stlist_iter stlist_it; - print_indent(indent); - fds_stlist_iter_init(&stlist_it, field, snap, 0); - // Before printing the first record, we need to print the semantic of the list - // only _next function returns state of the iterator - int ret = fds_stlist_iter_next(&stlist_it); - if (ret == FDS_OK || ret == FDS_EOC){ - printf(" (semantic: %s[%d])", fds_semantic2str(stlist_it.semantic), stlist_it.semantic); - } - if (ret == FDS_OK) { - putchar('\n'); - read_record(&stlist_it.rec, indent+1, iemgr); - } else if (ret == FDS_EOC){ - printf(" - empty\n"); - } - while (fds_stlist_iter_next(&stlist_it) == FDS_OK){ - read_record(&stlist_it.rec, indent+1, iemgr); + if (fds_iemgr_is_type_list(type)) { + // Process lists + printf("%*s:%s", WRITER_ORG_NAME_SPACE, org, field_name); + switch (type) { + case FDS_ET_BASIC_LIST: + // Note: header description will be complete in the function + read_list_basic(field, indent, iemgr, snap); + break; + case FDS_ET_SUB_TEMPLATE_LIST: + printf(" (subTemplateList, see below)\n"); + read_list_stl(field, indent, iemgr, snap); + break; + case FDS_ET_SUB_TEMPLATE_MULTILIST: + printf(" (subTemplateMultiList, see below)\n"); + read_list_stml(field, indent, iemgr, snap); + break; + default: + printf("*Unsupported list type*\n"); + break; } + return; } - default: - printf("%*s %-*s : ", WRITER_ORG_NAME_SPACE,org, WRITER_FIELD_NAME_SPACE, field_name); - } + printf("%*s:%-*s : ", WRITER_ORG_NAME_SPACE, org, WRITER_FIELD_NAME_SPACE, field_name); // Read and write the data from the field char buffer[1024]; int res = fds_field2str_be(field->data, field->size, type, buffer, sizeof(buffer)); if(res >= 0){ // Conversion was successful - if (type == FDS_ET_STRING){ + if (type == FDS_ET_STRING) { printf("\"%s\"", buffer); - } else if (type == FDS_ET_OCTET_ARRAY){ + } else if (type == FDS_ET_OCTET_ARRAY) { printf("0x%s",buffer); } else { printf("%s", buffer); } - if (*unit != 0) + if (*unit != 0) { printf(" %s", unit); + } putchar('\n'); return; } - else if (res == FDS_ERR_BUFFER){ + else if (res == FDS_ERR_BUFFER) { // Buffer too small printf("\n"); return; @@ -343,3 +355,200 @@ void read_field(struct fds_drec_field *field, unsigned int indent, const fds_iem } } +void +read_list_basic(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ + printf(" (basicList"); + + struct fds_blist_iter it; + fds_blist_iter_init(&it, field, iemgr); + + // Print information about the list -> make sure that the field definition is filled + int rc = fds_blist_iter_next(&it); + if (rc != FDS_EOC && rc != FDS_OK) { + // Malformed + printf(")\n"); + print_indent(indent); + printf(" *Malformed data structure: %s*\n", fds_blist_iter_err(&it)); + return; + } + + uint32_t ie_en = it.field.info->en; + uint16_t ie_id = it.field.info->id; + const char *name_scope = ""; + const char *name_field = ""; + if (it.field.info->def != NULL) { + // Definition exists + name_scope = it.field.info->def->scope->name; + name_field = it.field.info->def->name; + } else { + // Definition not found... try to find the scope + const struct fds_iemgr_scope *scope_ptr = fds_iemgr_scope_find_pen(iemgr, ie_en); + if (scope_ptr != NULL) { + name_scope = scope_ptr->name; + } + } + + printf(", List Semantic: %s)\n", fds_semantic2str(it.semantic)); + //print_indent(indent); + //printf("> List fields: EN:%*"PRIu32" ID:%*"PRIu16" %*s:%-*s\n", + // WRITER_EN_SPACE, ie_en, WRITER_ID_SPACE, ie_id, + // WRITER_ORG_NAME_SPACE, name_scope, WRITER_FIELD_NAME_SPACE, name_field); + + // Re-initialize the iterator + fds_blist_iter_init(&it, field, iemgr); + unsigned int cnt_value = 0; + bool more_values = true; + + while (more_values) { + int rc = fds_blist_iter_next(&it); + switch (rc) { + case FDS_OK: // Process the record + break; + case FDS_EOC: // End of the list + more_values = false; + continue; + case FDS_ERR_FORMAT: // Something is wrong with the record + printf("*Unable to continue due to malformed data: %s*\n", fds_blist_iter_err(&it)); + return; + default: + printf("*Internal error: fds_blist_iter_next(): unexpected return code*\n"); + return; + } + + read_field(&it.field, indent + 1, iemgr, snap); + cnt_value++; + } + + if (cnt_value == 0) { + print_indent(indent + 1); + printf("EN: %-*"PRIu32" ID: %-*"PRIu16" ", WRITER_EN_SPACE, ie_en, WRITER_ID_SPACE, ie_id); + printf("%*s:%-*s : ", WRITER_ORG_NAME_SPACE, name_scope, WRITER_FIELD_NAME_SPACE, name_field); + printf("\n"); + } +} + +void +read_list_stl(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ + struct fds_stlist_iter it; + fds_stlist_iter_init(&it, field, snap, FDS_STL_REPORT); + print_indent(indent); + printf("> List semantic: %s, Template ID: %"PRIu16")\n", fds_semantic2str(it.semantic), it.tid); + + unsigned int cnt_rec = 0; + bool more_records = true; + + while (more_records) { + int rc = fds_stlist_iter_next(&it); + switch (rc) { + case FDS_OK: // Process the record + break; + case FDS_EOC: // No more records + more_records = false; + continue; + case FDS_ERR_NOTFOUND: // Template is not available + print_indent(indent); + printf(" *Template not available - unable to decode*\n"); + return; + case FDS_ERR_FORMAT: // Something is wrong with the record + print_indent(indent); + printf("*Unable to continue due to malformed data: %s*\n", fds_stlist_iter_err(&it)); + return; + default: + print_indent(indent); + printf("*Internal error: fds_stlist_iter_next(): unexpected return code*\n"); + return; + } + + print_indent(indent); + printf(" - Data Record (#%u) [Length: %"PRIu16"]\n", ++cnt_rec, it.rec.size); + read_record(&it.rec, indent + 1, iemgr); + } + + if (cnt_rec == 0) { + print_indent(indent + 1); + printf(" \n"); + } +} + +void +read_list_stml(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ + struct fds_stmlist_iter it; + fds_stmlist_iter_init(&it, field, snap, FDS_STL_REPORT); + print_indent(indent); + printf("> List semantic: %s\n", fds_semantic2str(it.semantic)); + + unsigned int cnt_block = 0; + bool more_blocks = true; + + // For each block in the list + while (more_blocks) { + int rc_block = fds_stmlist_iter_next_block(&it); + switch (rc_block) { + case FDS_OK: // Process the block + break; + case FDS_EOC: // No more blocks + more_blocks = false; + continue; + case FDS_ERR_NOTFOUND: // Unable to read this block -> skip it + break; + case FDS_ERR_FORMAT: // Something is wrong with the block + print_indent(indent); + printf("*Unable to continue due to malformed data: %s*\n", fds_stmlist_iter_err(&it)); + return; + default: + print_indent(indent); + printf("*Internal error: fds_stmlist_iter_next_block(): unexpected return code*\n"); + return; + } + + print_indent(indent); + printf("- Top-level list header (#%u) [Template ID: %"PRIu16"]\n", ++cnt_block, it.tid); + if (rc_block == FDS_ERR_NOTFOUND) { + print_indent(indent); + printf(" *Template not available - unable to decode*\n"); + continue; + } + + unsigned int cnt_rec = 0; + bool more_recs = true; + + while (more_recs) { + int rc_rec = fds_stmlist_iter_next_rec(&it); + switch (rc_rec) { + case FDS_OK: // Process the record + break; + case FDS_EOC: // No more records in the current block + more_recs = false; + continue; + case FDS_ERR_FORMAT: // Something is wrong with the record + print_indent(indent); + printf("*Unable to continue due to malformed data: %s*\n", fds_stmlist_iter_err(&it)); + return; + default: + print_indent(indent); + printf("*Internal error: fds_stmlist_iter_next_rec(): unexpected return code*\n"); + return; + } + + print_indent(indent); + printf(" - Data Record (#%u) [Length: %"PRIu16"]\n", ++cnt_rec, it.rec.size); + read_record(&it.rec, indent + 1, iemgr); + } + + if (cnt_rec == 0) { + print_indent(indent + 1); + printf(" \n"); + } + } + + if (cnt_block == 0) { + print_indent(indent); + printf(" \n"); + } +} \ No newline at end of file diff --git a/src/plugins/output/viewer/Reader.h b/src/plugins/output/viewer/Reader.h index 87ac2f6e..102a1dee 100644 --- a/src/plugins/output/viewer/Reader.h +++ b/src/plugins/output/viewer/Reader.h @@ -53,7 +53,7 @@ /** * \brief Spaces in output for size field */ -#define WRITER_SIZE_SPACE 7 +#define WRITER_SIZE_SPACE 6 /** * \brief Spaces in output for Field name */ @@ -64,61 +64,104 @@ #define WRITER_ORG_NAME_SPACE 12 /** - * \brief Reads the data inside of ipfix message + * \brief Print all data of an IPFIX Message * - * Function reads and prints header of the packet and then iterates through the records. - * Information printed from header of packet are: + * Function reads and prints the header of the packet and then iterates through the Sets and their + * records. Information printed from the header of the packet are: * version, length, export time, sequence number and Observation Domain ID - * \param[in] msg ipfix message which will be printed + * \param[in] msg IPFIX message which will be printed + * \param[in] iemgr Information Element manager */ void read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr); /** - * \brief Reads data inside the single record of IPFIX message + * \brief Print all values inside the single Data Record of an IPFIX message * - * Reads and prints the header of the record and then iterates through the fields. - * Information printed from header of record are: - * Template ID and number of the fields inside the record - * \param[in] rec record which will be printed + * Reads and prints the header of the Data Record and then iterates through its fields. + * \param[in] rec Record which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager */ void read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr); /** - * \brief Reads the data inside the field of IPFIX message + * \brief Print the value of the Data Record field * - * Reads and prints all the information about the data in the field - * if the detailed definition is known, data are printed in readable format - * as well as the information about the data (organisation name and name of the data). - * Otherwise data are printed in the raw format (hexadecimal) - * In both cases Enterprise number and ID will be printed. - * \param[in] field Field which will be printed + * Reads and prints all information about the data in the field. If the detailed definition is + * known, value is printed in human readable format. Otherwise data are printed in the raw + * format (hexadecimal). In both cases Enterprise number and ID will be printed. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record */ void -read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, const fds_tsnapshot_t *snap, bool in_basicList); +read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); /** - * \brief Reads set which contains template + * \brief Print the content of basicList data type * - * Reads template type of sets which has set id == 2 or set id == 3. Uses Information Element manager from - * parameters for parsing the data inside of the template. - * \param tset_iter Template iterator - * \param set_id ID of the whole set - * \param iemgr IE Manager + * Iterates through the list and prints all values. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record + */ +void +read_list_basic(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); + +/** + * \brief Print the content of subTemplateList data type + * + * Iterates through the list and prints all Data Records. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record + */ +void +read_list_stl(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); + +/** + * \brief Print the content of subTemplateMultiList data type + * + * Iterates through the list and prints all top-level lists and their content i.e. Data Records. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record + */ +void +read_list_stml(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); + +/** + * \brief Print the (Options) Template Record + * + * Reads and prints content of the (Options) Template Record. Uses Information Element manager + * for determine description of present Information Elements. + * \param[in] tset_iter Template iterator + * \param[in] set_id ID of the Set to which the record belongs + * \param[in] iemgr Information Element manager */ void read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const fds_iemgr_t *iemgr); /** - * \brief Reads single set + * \brief Print the IPFIX Set and its content * - * Reads single set and determines what kind of set it is and which read_xxx function to use for reading the fields. + * Reads and prints single IPFIX Set and determines what kind of IPFIX Set it is and which + * read_xxx function to use for reading its content. * - * \param sets_iter Sets iterator - * \param msg IPFix message - * \param iemgr IE Manager for parsing the fields - * \param rec_i Number of record + * \param[in] sets_iter Sets iterator + * \param[in] msg IPFIX message + * \param[in] iemgr Information Element manager + * \param[in/out] rec_i Number of processed Data Records */ void read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr, uint32_t *rec_i); diff --git a/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst index b9c6ea78..e779b726 100644 --- a/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst +++ b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst @@ -2,15 +2,16 @@ ipfixcol2-viewer-output ======================== ---------------------- +---------------------- Viewer (output plugin) ---------------------- +---------------------- :Author: Jan Kala (xkalaj01@stud.fit.vutbr.cz) -:Date: 2018-12-01 -:Copyright: Copyright © 2018 CESNET, z.s.p.o. +:Author: Lukáš Huták (lukas.hutak@cesnet.cz) +:Date: 2019-02-20 +:Copyright: Copyright © 2018-2019 CESNET, z.s.p.o. :Version: 2.0 -:Manual section: X +:Manual section: 7 :Manual group: IPFIXcol collector Description diff --git a/src/plugins/output/viewer/viewer.c b/src/plugins/output/viewer/viewer.c index 2935790a..4ca4d2d8 100644 --- a/src/plugins/output/viewer/viewer.c +++ b/src/plugins/output/viewer/viewer.c @@ -60,9 +60,9 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = { // Configuration flags (reserved for future use) .flags = 0, // Plugin version string (like "1.2.3") - .version = "0.1.0", + .version = "2.0.0", // Minimal IPFIXcol version string (like "1.2.3") - .ipx_min = "2.0.0" + .ipx_min = "2.1.0" }; /** Instance */ @@ -119,15 +119,5 @@ ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) ipx_msg_ipfix_t *ipfix_msg = ipx_msg_base2ipfix(msg); read_packet(ipfix_msg, iemgr); - - // This part of code is not used just because the delay was implemented but not used yet -// Delay between messages printing - can be setup in config file -// struct instance_data *data = (struct instance_data *) cfg; -// const struct timespec *delay = &data->config->sleep_time; -// if (delay->tv_sec != 0 || delay->tv_nsec != 0) { -// nanosleep(delay, NULL); -// } - return IPX_OK; - } From 1c275c8f9e484acd279a37cefe189809726c38ce Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 20 Feb 2019 17:41:52 +0100 Subject: [PATCH 07/22] Bumped version to v2.1.0 The collector now requires libfds v0.2.0 --- CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d50bf573..7a972f2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ endif() # Versions and other informations set(IPFIXCOL_VERSION_MAJOR 2) -set(IPFIXCOL_VERSION_MINOR 0) +set(IPFIXCOL_VERSION_MINOR 1) set(IPFIXCOL_VERSION_PATCH 0) set(IPFIXCOL_VERSION ${IPFIXCOL_VERSION_MAJOR}.${IPFIXCOL_VERSION_MINOR}.${IPFIXCOL_VERSION_PATCH}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f931cd15..609f035c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,7 +9,7 @@ configure_file( ) # Find libfds -find_package(LibFds 0.1.0 REQUIRED) +find_package(LibFds 0.2.0 REQUIRED) # Find rst2man if (ENABLE_DOC_MANPAGE) From b807a3cb66d4faf91df384bdb68a752911f2593a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hut=C3=A1k?= Date: Wed, 20 Feb 2019 17:51:34 +0100 Subject: [PATCH 08/22] README: add reference to viewer (output) plugin --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index c9ea2dbc..73f0b756 100644 --- a/README.rst +++ b/README.rst @@ -36,6 +36,8 @@ network interface and a port. Multiple instances of these plugins can run concur **Output plugins** - store or forward your flows. - `JSON `_ - convert flow records to JSON and send/store them +- `Viewer `_ - convert IPFIX into plain text and print + it on standard output - `dummy `_ - simple module example - `lnfstore `_ (*) - store all flows in nfdump compatible format for long-term preservation From d9c80bbf4877d4ca672ecf6592337ef9262122e3 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Fri, 1 Mar 2019 15:20:36 +0100 Subject: [PATCH 09/22] Core: add missing desription of an API function --- include/ipfixcol2/message_ipfix.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/ipfixcol2/message_ipfix.h b/include/ipfixcol2/message_ipfix.h index 43aa6b53..742ab36c 100644 --- a/include/ipfixcol2/message_ipfix.h +++ b/include/ipfixcol2/message_ipfix.h @@ -157,9 +157,9 @@ ipx_msg_ipfix_get_ctx(ipx_msg_ipfix_t *msg); /** * \brief Get reference to all (Data/Template/Options Template) sets in the message - * \param msg Message - * \param sets Pointer to the array of sets - * \param size Number of sets in the array + * \param[in] msg Message + * \param[out] sets Pointer to the array of sets + * \param[out] size Number of sets in the array */ IPX_API void ipx_msg_ipfix_get_sets(ipx_msg_ipfix_t *msg, struct ipx_ipfix_set **sets, size_t *size); From cc305e5b3e4cf58dd72803f18d0405f9d19d933e Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 5 Mar 2019 10:30:00 +0100 Subject: [PATCH 10/22] Time Check: initial implementation of the plugin --- src/plugins/output/CMakeLists.txt | 1 + src/plugins/output/timecheck/CMakeLists.txt | 28 ++ src/plugins/output/timecheck/README.rst | 51 ++++ .../doc/ipfixcol2-timecheck-output.7.rst | 20 ++ src/plugins/output/timecheck/src/config.c | 188 ++++++++++++++ src/plugins/output/timecheck/src/config.h | 73 ++++++ src/plugins/output/timecheck/src/timecheck.c | 242 ++++++++++++++++++ 7 files changed, 603 insertions(+) create mode 100644 src/plugins/output/timecheck/CMakeLists.txt create mode 100644 src/plugins/output/timecheck/README.rst create mode 100644 src/plugins/output/timecheck/doc/ipfixcol2-timecheck-output.7.rst create mode 100644 src/plugins/output/timecheck/src/config.c create mode 100644 src/plugins/output/timecheck/src/config.h create mode 100644 src/plugins/output/timecheck/src/timecheck.c diff --git a/src/plugins/output/CMakeLists.txt b/src/plugins/output/CMakeLists.txt index 0aafa523..10432b05 100644 --- a/src/plugins/output/CMakeLists.txt +++ b/src/plugins/output/CMakeLists.txt @@ -1,4 +1,5 @@ # List of output plugin to build and install add_subdirectory(dummy) add_subdirectory(json) +add_subdirectory(timecheck) add_subdirectory(viewer) \ No newline at end of file diff --git a/src/plugins/output/timecheck/CMakeLists.txt b/src/plugins/output/timecheck/CMakeLists.txt new file mode 100644 index 00000000..1fd46121 --- /dev/null +++ b/src/plugins/output/timecheck/CMakeLists.txt @@ -0,0 +1,28 @@ +# Create a linkable module +add_library(timecheck-output MODULE + src/config.c + src/config.h + src/timecheck.c +) + +install( + TARGETS timecheck-output + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-timecheck-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-timecheck-output.7") + + add_custom_command(TARGET timecheck-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${INSTALL_DIR_MAN}/man7" + ) +endif() diff --git a/src/plugins/output/timecheck/README.rst b/src/plugins/output/timecheck/README.rst new file mode 100644 index 00000000..65f5a78f --- /dev/null +++ b/src/plugins/output/timecheck/README.rst @@ -0,0 +1,51 @@ +Time Check (output plugin) +========================== + +The plugin checks that start and end timestamps of each flow record are relatively recent. +Based on configured parameters it reports flows with timestamps from the future (i.e. greater +than the current system time) and timestamps from the distant past. + +Timestamp anomalies are usually caused by missing clock synchronization (e.g. NTP) or invalid +implementation on side of the exporter. If the anomaly is detected, the plugin will print +details on the standard output. + +Only following standard IANA timestamps are supported: + +- ID 150 (flowStartSeconds) +- ID 151 (flowEndSeconds) +- ID 152 (flowStartMilliseconds) +- ID 153 (flowEndMilliseconds) +- ID 154 (flowStartMicroseconds) +- ID 155 (flowEndMicroseconds) +- ID 156 (flowStartNanoseconds) +- ID 157 (flowEndNanoseconds) + +Example configuration +--------------------- + +.. code-block:: xml + + + TimeCheck output + timecheck + + 600 + 0 + + + +Parameters +---------- + +:``devPast``: + Maximum allowed deviation between the current system time and timestamps from the past in + seconds. The value must be greater than active and inactive timeouts of exporters and must also + include a possible delay between expiration and processing on the collector. + For example, let's say that active timeout and inactive timeout are 5 minutes and 30 seconds, + respectively. Value 600 (i.e. 10 minutes) should be always enough for all flow data to be + received and processed at the collector. + +:``devFuture``: + Maximum allowed deviation between the current time and timestamps from the future in seconds. + The collector should never receive flows with timestamp from the future, therefore, the value + should be usually set to 0. diff --git a/src/plugins/output/timecheck/doc/ipfixcol2-timecheck-output.7.rst b/src/plugins/output/timecheck/doc/ipfixcol2-timecheck-output.7.rst new file mode 100644 index 00000000..a7b12b88 --- /dev/null +++ b/src/plugins/output/timecheck/doc/ipfixcol2-timecheck-output.7.rst @@ -0,0 +1,20 @@ +=========================== + ipfixcol2-timecheck-output +=========================== + +-------------------------- +Time Check (output plugin) +-------------------------- + +:Author: Lukáš Huták (lukas.hutak@cesnet.cz) +:Date: 2019-03-01 +:Copyright: Copyright © 2019 CESNET, z.s.p.o. +:Version: 2.0 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 diff --git a/src/plugins/output/timecheck/src/config.c b/src/plugins/output/timecheck/src/config.c new file mode 100644 index 00000000..a3690314 --- /dev/null +++ b/src/plugins/output/timecheck/src/config.c @@ -0,0 +1,188 @@ +/** + * \file src/plugins/output/timecheck/src/config.c + * \author Lukas Hutak + * \brief Parser of an XML configuration (source file) + * \date 2019 + */ + +/* Copyright (C) 2019 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include "config.h" + +/* + * + * ... + * + */ + +/** XML nodes */ +enum params_xml_nodes { + DEV_PAST = 1, + DEV_FUTURE +}; + +/** Definition of the \ node */ +static const struct fds_xml_args args_params[] = { + FDS_OPTS_ROOT("params"), + FDS_OPTS_ELEM(DEV_PAST, "devPast", FDS_OPTS_T_UINT, 0), + FDS_OPTS_ELEM(DEV_FUTURE, "devFuture", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_END +}; + +/** + * \brief Process \ node + * \param[in] ctx Plugin context + * \param[in] root XML context to process + * \param[in] cfg Parsed configuration + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT in case of failure + */ +static int +config_parser_root(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct instance_config *cfg) +{ + (void) ctx; + + const struct fds_xml_cont *content; + while (fds_xml_next(root, &content) != FDS_EOC) { + switch (content->id) { + case DEV_PAST: + assert(content->type == FDS_OPTS_T_UINT); + cfg->dev_past = content->val_uint; + break; + case DEV_FUTURE: + assert(content->type == FDS_OPTS_T_UINT); + cfg->dev_future = content->val_uint; + break; + default: + // Internal error + assert(false); + } + } + + return IPX_OK; +} + +/** + * \brief Set default parameters of the configuration + * \param[in] cfg Configuration + */ +static void +config_default_set(struct instance_config *cfg) +{ + cfg->dev_past = 0; + cfg->dev_future = 0; +} + +/** + * \brief Validate the parsed configuration + * \param[in] ctx Plugin context + * \param[in] cfg Configuration + * \return #IPX_OK or #IPX_ERR_FORMAT + */ +static int +config_validate(ipx_ctx_t *ctx, const struct instance_config *cfg) +{ + if (cfg->dev_past == 0) { + IPX_CTX_ERROR(ctx, "Maximum allowed deviation from current time and the timestamps " + "from the past cannot be zero!", '\0'); + return IPX_ERR_FORMAT; + } + + if (cfg->dev_past < 300) { + IPX_CTX_WARNING(ctx, "The configuration might cause many false warnings!"); + } + + return IPX_OK; +} + + +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params) +{ + struct instance_config *cfg = calloc(1, sizeof(*cfg)); + if (!cfg) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + return NULL; + } + + // Set default parameters + config_default_set(cfg); + + // Create an XML parser + fds_xml_t *parser = fds_xml_create(); + if (!parser) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + free(cfg); + return NULL; + } + + if (fds_xml_set_args(parser, args_params) != FDS_OK) { + IPX_CTX_ERROR(ctx, "Failed to parse the description of an XML document!", '\0'); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + fds_xml_ctx_t *params_ctx = fds_xml_parse_mem(parser, params, true); + if (params_ctx == NULL) { + IPX_CTX_ERROR(ctx, "Failed to parse the configuration: %s", fds_xml_last_err(parser)); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + // Parse parameters + int rc = config_parser_root(ctx, params_ctx, cfg); + fds_xml_destroy(parser); + if (rc != IPX_OK) { + free(cfg); + return NULL; + } + + if (config_validate(ctx, cfg) != IPX_OK) { + free(cfg); + return NULL; + } + + return cfg; +} + +void +config_destroy(struct instance_config *cfg) +{ + free(cfg); +} diff --git a/src/plugins/output/timecheck/src/config.h b/src/plugins/output/timecheck/src/config.h new file mode 100644 index 00000000..29f86b45 --- /dev/null +++ b/src/plugins/output/timecheck/src/config.h @@ -0,0 +1,73 @@ +/** + * \file src/plugins/output/timecheck/src/config.h + * \author Lukas Hutak + * \brief Parser of an XML configuration (header file) + * \date 2019 + */ + +/* Copyright (C) 2019 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef CONFIG_H +#define CONFIG_H + +#include +#include "stdint.h" + +/** Configuration of a instance of the plugin */ +struct instance_config { + /** Maximum allowed deviation between the current time and timestamps from the past (sec) */ + uint64_t dev_past; + /** Maximum allowed deviation between the current time and timestamps from the future (sec) */ + uint64_t dev_future; +}; + +/** + * \brief Parse configuration of the plugin + * \param[in] ctx Instance context + * \param[in] params XML parameters + * \return Pointer to the parse configuration of the instance on success + * \return NULL if arguments are not valid or if a memory allocation error has occurred + */ +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params); + +/** + * \brief Destroy parsed configuration + * \param[in] cfg Parsed configuration + */ +void +config_destroy(struct instance_config *cfg); + +#endif // CONFIG_H diff --git a/src/plugins/output/timecheck/src/timecheck.c b/src/plugins/output/timecheck/src/timecheck.c new file mode 100644 index 00000000..84604701 --- /dev/null +++ b/src/plugins/output/timecheck/src/timecheck.c @@ -0,0 +1,242 @@ +/** + * \file src/plugins/output/timecheck/src/timecheck.c + * \author Lukas Hutak + * \brief Time checker plugin for IPFIXcol 2 + * \date 2019 + */ + +/* Copyright (C) 2019 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include +#include +#include + +#include "config.h" +#include "../../../../core/message_ipfix.h" + +/** Private Enterprise Number of standard IEs from IANA */ +#define PEN_IANA 0 +/** Private Enterprise Number of Standard reverse IEs from IANA */ +#define PEN_IANA_REV 29305 + +/** Plugin description */ +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin type + .type = IPX_PT_OUTPUT, + // Plugin identification name + .name = "timecheck", + // Brief description of plugin + .dsc = "The plugin checks that timestamp elements in flows are relatively recent.", + // Configuration flags (reserved for future use) + .flags = 0, + // Plugin version string (like "1.2.3") + .version = "2.0.0", + // Minimal IPFIXcol version string (like "1.2.3") + .ipx_min = "2.0.0" +}; + +/** Instance */ +struct instance_data { + /** Parsed configuration of the instance */ + struct instance_config *config; + /** Current time (seconds since the Epoch) */ + uint64_t ts_now; + + /** Context reference (only for log!) */ + ipx_ctx_t *ctx; +}; + +// Function prototype +static void +timestamp_check(const struct instance_data *inst, ipx_msg_ipfix_t *msg, + const struct fds_drec_field *field); + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *params) +{ + // Create a private data + struct instance_data *data = calloc(1, sizeof(*data)); + if (!data) { + return IPX_ERR_DENIED; + } + + if ((data->config = config_parse(ctx, params)) == NULL) { + free(data); + return IPX_ERR_DENIED; + } + + data->ctx = ctx; + ipx_ctx_private_set(ctx, data); + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *cfg) +{ + (void) ctx; // Suppress warnings + + struct instance_data *data = (struct instance_data *) cfg; + config_destroy(data->config); + free(data); +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) +{ + (void) ctx; // Suppress warnings + struct instance_data *data = (struct instance_data *) cfg; + ipx_msg_ipfix_t *ipfix_msg = ipx_msg_base2ipfix(msg); + + // Update the current UTC time + data->ts_now = (uint64_t) time(NULL); + + // For each Data Record in the message + uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(ipfix_msg); + for (uint32_t i = 0; i < rec_cnt; ++i) { + // For each field in the Data Record + struct ipx_ipfix_record *rec = ipx_msg_ipfix_get_drec(ipfix_msg, i); + struct fds_drec_iter it; + + fds_drec_iter_init(&it, &rec->rec, 0); + while (fds_drec_iter_next(&it) != FDS_EOC) { + // Is it a timestamp? + const uint16_t field_id = it.field.info->id; + const uint32_t field_en = it.field.info->en; + + if (field_en != PEN_IANA && field_en != PEN_IANA_REV) { + // We don't check non-standard fields + continue; + } + + if (field_id < 150U || field_id > 157U) { + // We want to check only IE elements within the range 150 - 157 + continue; + } + + // Check if the value doesn't violate rules + timestamp_check(data, ipfix_msg, &it.field); + } + } + + return IPX_OK; +} + +/** + * \brief Timestamp check function + * + * Only IANA fields flowStart... and flowEnd... are supported! + * \param[in] inst Plugin instance parameters + * \param[in] msg IPFIX Message from which the timestamp comes from + * \param[in] field The timestamp to check + */ +void +timestamp_check(const struct instance_data *inst, ipx_msg_ipfix_t *msg, + const struct fds_drec_field *field) +{ + assert((field->info->en == PEN_IANA || field->info->en == PEN_IANA_REV) && "Non-IANA PEN!"); + enum fds_iemgr_element_type elem_type; + + switch (field->info->id) { + case 150U: // flowStartSeconds + case 151U: // flowEndSeconds + elem_type = FDS_ET_DATE_TIME_SECONDS; + break; + case 152U: // flowStartMilliseconds + case 153U: // flowEndMilliseconds + elem_type = FDS_ET_DATE_TIME_MILLISECONDS; + break; + case 154U: // flowStartMicroseconds + case 155U: // flowEndMicroseconds + elem_type = FDS_ET_DATE_TIME_MICROSECONDS; + break; + case 156U: // flowStartNanoseconds + case 157U: // flowEndNanoseconds + elem_type = FDS_ET_DATE_TIME_NANOSECONDS; + break; + default: + assert(false && "Unhandled switch option!"); + return; + } + + // Get the value + uint64_t ts_value; + int ret_code = fds_get_datetime_lp_be(field->data, field->size, elem_type, &ts_value); + if (ret_code != FDS_OK) { + IPX_CTX_WARNING(inst->ctx, "Timestamp conversion failed! Skipping...", '\0'); + return; + } + ts_value /= 1000U; // Convert from milliseconds to seconds (since the Epoch) + + // Check the deviation + const char *violation_type; + uint64_t ts_diff; + + if (ts_value <= inst->ts_now) { + // Timestamps is from the past + ts_diff = inst->ts_now - ts_value; + if (ts_diff <= inst->config->dev_past) { + return; + } + violation_type = "past"; + } else { + // Timestamp is from the future + ts_diff = ts_value - inst->ts_now; + if (ts_diff <= inst->config->dev_future) { + return; + } + violation_type = "future"; + } + + // Report the violation of rules + const struct ipx_msg_ctx *msg_ctx = ipx_msg_ipfix_get_ctx(msg); + const char *s_name = msg_ctx->session->ident; + const uint32_t s_odid = msg->ctx.odid; + + uint16_t diff_secs = ts_diff % 60U; + ts_diff /= 60U; + uint16_t diff_mins = ts_diff % 60U; + ts_diff /= 60U; + uint16_t diff_hrs = ts_diff % 24U; + ts_diff /= 24U; + + printf("%s [ODID: %"PRIu32"]: Timestamp (EN: %"PRIu32", ID: %"PRIu16") is " + "%"PRIu64" days, %"PRIu16" hours, %"PRIu16" minutes and %"PRIu16" seconds in the %s " + "(now: %"PRIu64", TS value: %"PRIu64" [seconds since the Epoch])\n", + s_name, s_odid, field->info->en, field->info->id, + ts_diff, diff_hrs, diff_mins, diff_secs, violation_type, + inst->ts_now, ts_value); +} \ No newline at end of file From d4ed108ce7097f8130e5bc0a4d03001389a47470 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 5 Mar 2019 10:44:34 +0100 Subject: [PATCH 11/22] Time Check: fix a compile time warning --- src/plugins/output/timecheck/src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/output/timecheck/src/config.c b/src/plugins/output/timecheck/src/config.c index a3690314..a0701d13 100644 --- a/src/plugins/output/timecheck/src/config.c +++ b/src/plugins/output/timecheck/src/config.c @@ -123,7 +123,7 @@ config_validate(ipx_ctx_t *ctx, const struct instance_config *cfg) } if (cfg->dev_past < 300) { - IPX_CTX_WARNING(ctx, "The configuration might cause many false warnings!"); + IPX_CTX_WARNING(ctx, "The configuration might cause many false warnings!", '\0'); } return IPX_OK; From 411a4aaff7e5916d95187e0eb7c71a67fce7eb95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hut=C3=A1k?= Date: Tue, 5 Mar 2019 11:45:10 +0100 Subject: [PATCH 12/22] README: add reference to Time Check plugin --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 73f0b756..15d12379 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,8 @@ network interface and a port. Multiple instances of these plugins can run concur - `JSON `_ - convert flow records to JSON and send/store them - `Viewer `_ - convert IPFIX into plain text and print it on standard output -- `dummy `_ - simple module example +- `Time Check `_ - flow timestamp check +- `Dummy `_ - simple module example - `lnfstore `_ (*) - store all flows in nfdump compatible format for long-term preservation - `UniRec `_ (*) - send flow records in UniRec format From 0ec65cf851184bf6d9b6dfa95e344613f78c9588 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Thu, 14 Mar 2019 15:13:40 +0100 Subject: [PATCH 13/22] UniRec output: added "string_trimmed" type for string trimming during conversion --- extra_plugins/output/unirec/src/map.c | 19 ++++++++++++++--- extra_plugins/output/unirec/src/map.h | 10 +++++++++ extra_plugins/output/unirec/src/translator.c | 21 +++++++++++++++++++ .../output/unirec/src/unirecplugin.c | 2 +- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/extra_plugins/output/unirec/src/map.c b/extra_plugins/output/unirec/src/map.c index 130ac8b9..d5b96768 100644 --- a/extra_plugins/output/unirec/src/map.c +++ b/extra_plugins/output/unirec/src/map.c @@ -52,6 +52,8 @@ #define DEF_SIZE 32 /** Size of error buffer */ #define ERR_SIZE 256 +/** Special type of UniRec string field that is trimmed before conversion from IPFIX */ +#define TYPE_STRING_TRIM "string_trimmed" /** Internal structure of the mapping database */ struct map_s { @@ -234,13 +236,14 @@ map_elem_get_internal(const char *elem) * \param[in] ur_name UniRec name * \param[in] ur_type UniRec type * \param[in] ur_type_str UniRec type string + * \param[in] cflags Additional conversion flags * \param[in] ie_defs Definition of IPFIX IEs (comma separated list of definitions) * \param[in] line_id Line ID (just for error messages) * \return #IPX_OK on success * \return #IPX_ERR_NOMEM or #IPX_ERR_FORMAT on failure */ static int -map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, +map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, uint32_t cflags, const char *ie_defs, size_t line_id) { char *defs_cpy = strdup(ie_defs); @@ -264,6 +267,7 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, rec.unirec.name = ur_name; rec.unirec.type = ur_type; rec.unirec.type_str = ur_type_str; + rec.unirec.flags = cflags; // Process IPFIX fields char *subsave_ptr = NULL; @@ -323,6 +327,7 @@ map_load_line(map_t *map, const char *line, size_t line_id) int rc = IPX_OK; char *ur_name = NULL; char *ur_type_str = NULL; + uint32_t conv_flags = 0; ur_field_type_t ur_type; char *line_cpy = strdup(line); @@ -356,7 +361,15 @@ map_load_line(map_t *map, const char *line, size_t line_id) goto end; } - int type = ur_get_field_type_from_str(token); + int type; + if (strcmp(token, TYPE_STRING_TRIM) == 0) { + // Special version of string + type = UR_TYPE_STRING; + conv_flags |= MAP_FLAGS_STR_TRIM; + } else { + type = ur_get_field_type_from_str(token); + } + if (type == UR_E_INVALID_TYPE) { snprintf(map->err_buffer, ERR_SIZE, "Line %zu: Invalid type '%s' of UniRec field '%s'", line_id, token, ur_name); @@ -382,7 +395,7 @@ map_load_line(map_t *map, const char *line, size_t line_id) ptrdiff_t offset = token - line_cpy; const char *ie_defs = line + offset; - rc = map_load_line_ie_defs(map, ur_name, ur_type, ur_type_str, ie_defs, line_id); + rc = map_load_line_ie_defs(map, ur_name, ur_type, ur_type_str, conv_flags, ie_defs, line_id); end: free(ur_type_str); diff --git a/extra_plugins/output/unirec/src/map.h b/extra_plugins/output/unirec/src/map.h index 420570e6..35c4cad1 100644 --- a/extra_plugins/output/unirec/src/map.h +++ b/extra_plugins/output/unirec/src/map.h @@ -61,6 +61,11 @@ enum MAP_SRC { MAP_SRC_INTERNAL_DBF }; +enum MAP_FLAGS { + /** Perform trim of an IPFIX string (convert only characters before the first '\0') */ + MAP_FLAGS_STR_TRIM = (1 << 0) +}; + /** IPFIX-to-UniRec mapping record */ struct map_rec { struct { @@ -85,6 +90,11 @@ struct map_rec { ur_field_type_t type; /** Data type (string, for log!) */ char *type_str; + /** + * \brief Additional conversion flags + * \note See ::MAP_FLAGS + */ + uint32_t flags; } unirec; }; diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index db3730ee..899bf9f6 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -316,6 +316,22 @@ translate_bytes(translator_t *trans, const struct translator_rec *rec, return 0; } +/** + * \brief Convert IPFIX string to trimmed UniRec string + * + * The function will copy only characters up to the first occurrence of '\0' (excluding). + * \copydetails translate_uint() + */ +static int +translator_string_trim(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + size_t copy_len = strnlen((const char *) field->data, field->size); + ur_set_var(trans->record.ur_tmplt, trans->record.data, ur_id, field->data, copy_len); + return 0; +} + /** * \brief Convert IPFIX boolean to UniRec char/(un)signed integer * \copydetails translate_uint() @@ -699,6 +715,11 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) switch (type_ur) { case UR_TYPE_STRING: + // String array + if (type_ipx == FDS_ET_STRING && (rec->unirec.flags & MAP_FLAGS_STR_TRIM) != 0) { + return translator_string_trim; + } + // Fall through case UR_TYPE_BYTES: // String/byte array if (type_ipx == FDS_ET_STRING || type_ipx == FDS_ET_OCTET_ARRAY) { diff --git a/extra_plugins/output/unirec/src/unirecplugin.c b/extra_plugins/output/unirec/src/unirecplugin.c index 48cdfa3d..e7279451 100644 --- a/extra_plugins/output/unirec/src/unirecplugin.c +++ b/extra_plugins/output/unirec/src/unirecplugin.c @@ -78,7 +78,7 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = { // Configuration flags (reserved for future use) .flags = 0, // Plugin version string (like "1.2.3") - .version = "2.0.0", + .version = "2.1.0", // Minimal IPFIXcol version string (like "1.2.3") .ipx_min = "2.0.0" }; From e5e994ad8a2cee5c790db6d3dc300ef41cec8b8a Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Thu, 14 Mar 2019 15:26:00 +0100 Subject: [PATCH 14/22] UniRec: updated README file (added note about "string_trimmed" type) --- extra_plugins/output/unirec/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extra_plugins/output/unirec/README.rst b/extra_plugins/output/unirec/README.rst index 4167022f..a10b155d 100644 --- a/extra_plugins/output/unirec/README.rst +++ b/extra_plugins/output/unirec/README.rst @@ -211,6 +211,8 @@ a line: ``"BYTES uint64 e0id1"``: the ```` element. - Second parameter specifies a data type of the UniRec field. List of all supported types is available in `UniRec documentation `_. + The plugin also supports additional virtual type "string_trimmed" that converts data to UniRec + string type, but only characters up to the first occurrence of '\\0' (excluding) are copied. - The third parameter is comma separated list of corresponding IPFIX Information Elements (IEs). In this case, "e0id1" means IPFIX IE with Private Enterprise ID 0 and IE ID 1 (which is "octetDeltaCount"). Instead of numeric identification an IE name in ":" format From cee8e4898e528bf9b056d15c8a78139c117f26de Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Fri, 21 Jun 2019 16:20:12 +0200 Subject: [PATCH 15/22] FDS output: initial version of the new plugin --- src/plugins/output/fds/CMakeLists.txt | 31 ++ src/plugins/output/fds/README.rst | 69 +++++ .../output/fds/doc/ipfixcol2-fds-output.7.rst | 20 ++ src/plugins/output/fds/src/Config.cpp | 190 ++++++++++++ src/plugins/output/fds/src/Config.hpp | 64 ++++ src/plugins/output/fds/src/Exception.hpp | 34 +++ src/plugins/output/fds/src/Storage.cpp | 286 ++++++++++++++++++ src/plugins/output/fds/src/Storage.hpp | 106 +++++++ src/plugins/output/fds/src/fds.cpp | 127 ++++++++ 9 files changed, 927 insertions(+) create mode 100644 src/plugins/output/fds/CMakeLists.txt create mode 100644 src/plugins/output/fds/README.rst create mode 100644 src/plugins/output/fds/doc/ipfixcol2-fds-output.7.rst create mode 100644 src/plugins/output/fds/src/Config.cpp create mode 100644 src/plugins/output/fds/src/Config.hpp create mode 100644 src/plugins/output/fds/src/Exception.hpp create mode 100644 src/plugins/output/fds/src/Storage.cpp create mode 100644 src/plugins/output/fds/src/Storage.hpp create mode 100644 src/plugins/output/fds/src/fds.cpp diff --git a/src/plugins/output/fds/CMakeLists.txt b/src/plugins/output/fds/CMakeLists.txt new file mode 100644 index 00000000..2c2a6a83 --- /dev/null +++ b/src/plugins/output/fds/CMakeLists.txt @@ -0,0 +1,31 @@ +# Create a linkable module +add_library(fds-output MODULE + src/Config.cpp + src/Config.hpp + src/Exception.hpp + src/fds.cpp + src/Storage.cpp + src/Storage.hpp +) + +install( + TARGETS fds-output + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-fds-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-fds-output.7") + + add_custom_command(TARGET fds-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${INSTALL_DIR_MAN}/man7" + ) +endif() diff --git a/src/plugins/output/fds/README.rst b/src/plugins/output/fds/README.rst new file mode 100644 index 00000000..09b57efe --- /dev/null +++ b/src/plugins/output/fds/README.rst @@ -0,0 +1,69 @@ +Flow Data Storage (output plugin) +================================= + +The plugin converts and stores IPFIX Data Records into FDS file format. The file +is based on IPFIX, therefore, it provides highly-effective way for long-term +storage and stores complete flow records (including all Enterprise-specific +fields, biflow, etc.) together with identification of the flow exporters who +exported these records. + +All data are stored into flat files, which are automatically rotated and renamed +every N minutes (by default 5 minutes). + +Example configuration +--------------------- + +.. code-block:: xml + + + FDS output + fds + + /tmp/ipfixcol2/fds/ + none + + 300 + yes + + + + +Parameters +---------- + +:``storagePath``: + The path element specifies the storage directory for data files. Keep on + mind that the path must exist in your system. Otherwise, no files are stored. + All files will be stored based on the configuration using the following + template: ``/YYYY/MM/DD/flows..fds`` where ``YYYY/MM/DD`` + means year/month/day and ```` represents a UTC timestamp in + format ``YYMMDDhhmmss``. + +:``compression``: + Data compression helps to significantly reduce size of output files. + Following compression algorithms are available: + + :``none``: Compression disabled [default] + :``lz4``: LZ4 compression (very fast, slightly worse compression ration) + :``zstd``: ZSTD compression (slightly slower, good compression ration) + +:``dumpInterval``: + Configuration of output files rotation. + + :``timeWindow``: + Specifies time interval in seconds to rotate files i.e. close the current + file and create a new one. [default: 300] + + :``align``: + Align file rotation with next N minute interval. For example, if enabled + and window size is 5 minutes long, files will be created at 0, 5, 10, etc. + [values: yes/no, default: yes] + +:``asyncIO``: + Allows to use asynchronous I/O for writing to the file. Usually when parts + of the file are being written, the process is blocked on synchronous I/O + and waits for the operation to complete. However, asynchronous I/O allows + the plugin to simultaneously write to file and process flow records, which + significantly improves overall performance. (Note: a pool of service + threads shared among instances of FDS plugin might be created). + [values: true/false, default: true] diff --git a/src/plugins/output/fds/doc/ipfixcol2-fds-output.7.rst b/src/plugins/output/fds/doc/ipfixcol2-fds-output.7.rst new file mode 100644 index 00000000..a978730e --- /dev/null +++ b/src/plugins/output/fds/doc/ipfixcol2-fds-output.7.rst @@ -0,0 +1,20 @@ +====================== + ipfixcol2-fds-output +====================== + +--------------------------------- +Flow Data Storage (output plugin) +--------------------------------- + +:Author: Lukáš Huták (lukas.hutak@cesnet.cz) +:Date: 2019-07-01 +:Copyright: Copyright © 2019 CESNET, z.s.p.o. +:Version: 2.0 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 diff --git a/src/plugins/output/fds/src/Config.cpp b/src/plugins/output/fds/src/Config.cpp new file mode 100644 index 00000000..a06d61e2 --- /dev/null +++ b/src/plugins/output/fds/src/Config.cpp @@ -0,0 +1,190 @@ +/** + * \file src/plugins/output/fds/src/Config.cpp + * \author Lukas Hutak + * \brief Parser of XML configuration (source file) + * \date 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "Config.hpp" +#include +#include + +/* + * + * ... + * ... + * + * ... + * ... + * + * ... + * + */ + +/// XML nodes +enum params_xml_nodes { + NODE_STORAGE = 1, + NODE_COMPRESS, + NODE_DUMP, + NODE_ASYNCIO, + + DUMP_WINDOW, + DUMP_ALIGN +}; + +/// Definition of the \ node +static const struct fds_xml_args args_dump[] = { + FDS_OPTS_ELEM(DUMP_WINDOW, "timeWindow", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM(DUMP_ALIGN, "align", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_END +}; + +/// Definition of the \ node +static const struct fds_xml_args args_params[] = { + FDS_OPTS_ROOT("params"), + FDS_OPTS_ELEM(NODE_STORAGE, "storagePath", FDS_OPTS_T_STRING, 0), + FDS_OPTS_ELEM(NODE_COMPRESS, "compression", FDS_OPTS_T_STRING, FDS_OPTS_P_OPT), + FDS_OPTS_NESTED(NODE_DUMP, "dumpInterval", args_dump, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM(NODE_ASYNCIO, "asyncIO", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_END +}; + +Config::Config(const char *params) +{ + set_default(); + + // Create XML parser + std::unique_ptr xml(fds_xml_create(), &fds_xml_destroy); + if (!xml) { + throw std::runtime_error("Failed to create an XML parser!"); + } + + if (fds_xml_set_args(xml.get(), args_params) != FDS_OK) { + throw std::runtime_error("Failed to parse the description of an XML document!"); + } + + fds_xml_ctx_t *params_ctx = fds_xml_parse_mem(xml.get(), params, true); + if (!params_ctx) { + std::string err = fds_xml_last_err(xml.get()); + throw std::runtime_error("Failed to parse the configuration: " + err); + } + + // Parse parameters and check configuration + try { + parse_root(params_ctx); + validate(); + } catch (std::exception &ex) { + throw std::runtime_error("Failed to parse the configuration: " + std::string(ex.what())); + } +} + +/** + * @brief Set default parameters + */ +void +Config::set_default() +{ + m_path.clear(); + m_calg = calg::NONE; + m_async = true; + + m_window.align = true; + m_window.size = WINDOW_SIZE; +} + +/** + * @brief Check if the configuration is valid + * @throw runtime_error if the configuration breaks some rules + */ +void +Config::validate() +{ + if (m_path.empty()) { + throw std::runtime_error("Storage path cannot be empty!"); + } + + if (m_window.size == 0) { + throw std::runtime_error("Window size cannot be zero!"); + } +} + +/** + * @brief Process \ node + * @param[in] ctx XML context to process + * @throw runtime_error if the parser fails + */ +void +Config::parse_root(fds_xml_ctx_t *ctx) +{ + const struct fds_xml_cont *content; + while (fds_xml_next(ctx, &content) != FDS_EOC) { + switch (content->id) { + case NODE_STORAGE: + // Storage path + assert(content->type == FDS_OPTS_T_STRING); + m_path = content->ptr_string; + break; + case NODE_COMPRESS: + // Compression method + assert(content->type == FDS_OPTS_T_STRING); + if (strcasecmp(content->ptr_string, "none") == 0) { + m_calg = calg::NONE; + } else if (strcasecmp(content->ptr_string, "lz4") == 0) { + m_calg = calg::LZ4; + } else if (strcasecmp(content->ptr_string, "zstd") == 0) { + m_calg = calg::ZSTD; + } else { + const std::string inv_str = content->ptr_string; + throw std::runtime_error("Unknown compression algorithm '" + inv_str + "'"); + } + break; + case NODE_ASYNCIO: + // Asynchronous I/O + assert(content->type == FDS_OPTS_T_BOOL); + m_async = content->val_bool; + break; + case NODE_DUMP: + // Dump window + assert(content->type == FDS_OPTS_T_CONTEXT); + parse_dump(content->ptr_ctx); + break; + default: + // Internal error + throw std::runtime_error("Unknown XML node"); + } + } +} + +/** + * @brief Auxiliary function for parsing \ options + * @param[in] ctx XML context to process + * @throw runtime_error if the parser fails + */ +void +Config::parse_dump(fds_xml_ctx_t *ctx) +{ + const struct fds_xml_cont *content; + while(fds_xml_next(ctx, &content) != FDS_EOC) { + switch (content->id) { + case DUMP_WINDOW: + // Window size + assert(content->type == FDS_OPTS_T_UINT); + if (content->val_uint > UINT32_MAX) { + throw std::runtime_error("Window size is too long!"); + } + m_window.size = static_cast(content->val_uint); + break; + case DUMP_ALIGN: + // Window alignment + assert(content->type == FDS_OPTS_T_BOOL); + m_window.align = content->val_bool; + break; + default: + // Internal error + throw std::runtime_error("Unknown XML node"); + } + } +} \ No newline at end of file diff --git a/src/plugins/output/fds/src/Config.hpp b/src/plugins/output/fds/src/Config.hpp new file mode 100644 index 00000000..876ef466 --- /dev/null +++ b/src/plugins/output/fds/src/Config.hpp @@ -0,0 +1,64 @@ +/** + * \file src/plugins/output/fds/src/Config.hpp + * \author Lukas Hutak + * \brief Parser of XML configuration (header file) + * \date 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPFIXCOL2_FDS_CONFIG_HPP +#define IPFIXCOL2_FDS_CONFIG_HPP + +#include +#include + +/** + * @brief Plugin configuration parser + */ +class Config { +public: + /** + * @brief Parse configuration of the plugin + * @param[in] params XML parameters to parse + * @throw runtime_exception on error + */ + Config(const char *params); + ~Config() = default; + + enum class calg { + NONE, ///< Do not use compression + LZ4, ///< LZ4 compression + ZSTD ///< ZSTD compression + }; + + /// Storage path + std::string m_path; + /// Compression algorithm + calg m_calg; + /// Asynchronous I/O enabled + bool m_async; + + struct { + bool align; ///< Enable/disable window alignment + uint32_t size; ///< Time window size + } m_window; ///< Window alignment + +private: + /// Default window size + static const uint32_t WINDOW_SIZE = 300U; + + void + set_default(); + void + validate(); + + void + parse_root(fds_xml_ctx_t *ctx); + void + parse_dump(fds_xml_ctx_t *ctx); +}; + + +#endif // IPFIXCOL2_FDS_CONFIG_HPP diff --git a/src/plugins/output/fds/src/Exception.hpp b/src/plugins/output/fds/src/Exception.hpp new file mode 100644 index 00000000..f5325c5f --- /dev/null +++ b/src/plugins/output/fds/src/Exception.hpp @@ -0,0 +1,34 @@ +/** + * \file src/plugins/output/fds/src/Exception.hpp + * \author Lukas Hutak + * \brief Plugin specific exception (header file) + * \date 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPFIXCOL2_FDS_EXCEPTION_HPP +#define IPFIXCOL2_FDS_EXCEPTION_HPP + +#include +#include + +/// Plugin specific exception +class FDS_exception : public std::runtime_error { +public: + /** + * @brief Constructor + * @param[in] str Error message + */ + FDS_exception(const std::string &str) : std::runtime_error(str) {}; + /** + * @brief Constructor + * @param[in] str Error message + */ + FDS_exception(const char *str) : std::runtime_error(str) {}; + // Default destructor + ~FDS_exception() = default; +}; + +#endif //IPFIXCOL2_FDS_EXCEPTION_HPP diff --git a/src/plugins/output/fds/src/Storage.cpp b/src/plugins/output/fds/src/Storage.cpp new file mode 100644 index 00000000..64655a66 --- /dev/null +++ b/src/plugins/output/fds/src/Storage.cpp @@ -0,0 +1,286 @@ +/** + * \file src/plugins/output/fds/src/Storage.cpp + * \author Lukas Hutak + * \brief FDS file storage (source file) + * \date June 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include +#include "Storage.hpp" + +Storage::Storage(ipx_ctx_t *ctx, const Config &cfg) : m_ctx(ctx), m_path(cfg.m_path) +{ + // Check if the directory exists + struct stat file_info; + memset(&file_info, 0, sizeof(file_info)); + if (stat(m_path.c_str(), &file_info) != 0 || !S_ISDIR(file_info.st_mode)) { + throw FDS_exception("Directory '" + m_path + "' doesn't exist or search permission is denied"); + } + + // Prepare flags for FDS file + m_flags = 0; + switch (cfg.m_calg) { + case Config::calg::LZ4: + m_flags |= FDS_FILE_LZ4; + break; + case Config::calg::ZSTD: + m_flags |= FDS_FILE_ZSTD; + break; + default: + break; + } + + if (!cfg.m_async) { + m_flags |= FDS_FILE_NOASYNC; + } + + m_flags |= FDS_FILE_APPEND; +} + +void +Storage::window_new(time_t ts) +{ + // Close the current window if exists + window_close(); + + // Open new file + const std::string new_file = filename_gen(ts); + std::unique_ptr new_file_cpy(strdup(new_file.c_str()), &free); + + char *dir2create; + if (!new_file_cpy || (dir2create = dirname(new_file_cpy.get())) == nullptr) { + throw FDS_exception("Failed to generate name of an output directory!"); + } + + if (ipx_utils_mkdir(dir2create, IPX_UTILS_MKDIR_DEF) != FDS_OK) { + throw FDS_exception("Failed to create directory '" + std::string(dir2create) + "'"); + } + + m_file.reset(fds_file_init()); + if (!m_file) { + throw FDS_exception("Failed to create FDS file handler!"); + } + + if (fds_file_open(m_file.get(), new_file.c_str(), m_flags) != FDS_OK) { + m_file.reset(); + const char *err_msg = "something"; //fds_file_error(m_file.get()); + throw FDS_exception("Failed to create file '" + new_file + "': " + err_msg); + } +} + +void +Storage::window_close() +{ + m_file.reset(); + m_session2params.clear(); +} + +void +Storage::process_msg(ipx_msg_ipfix_t *msg) +{ + if (!m_file) { + IPX_CTX_DEBUG(m_ctx, "Ignoring IPFIX Message due to undefined output file!", '\0'); + return; + } + + // Specify a Transport Session context + struct ipx_msg_ctx *msg_ctx = ipx_msg_ipfix_get_ctx(msg); + session_ctx &file_ctx = session_get(msg_ctx->session); + + auto hdr_ptr = reinterpret_cast(ipx_msg_ipfix_get_packet(msg)); + assert(ntohs(hdr_ptr->version) == FDS_IPFIX_VERSION && "Unexpected packet version"); + const uint32_t exp_time = ntohl(hdr_ptr->export_time); + + if (fds_file_write_ctx(m_file.get(), file_ctx.id, msg_ctx->odid, exp_time) != FDS_OK) { + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to configure file writer: " + std::string(err_msg)); + } + + // For each Data Record in the file + const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); + for (uint32_t i = 0; i < rec_cnt; ++i) { + ipx_ipfix_record *rec_ptr = ipx_msg_ipfix_get_drec(msg, i); + + // Insert the record // TODO: improve me! + const struct fds_template *rec_tmplt = rec_ptr->rec.tmplt; + uint16_t tmplt_id = rec_tmplt->id; + enum fds_template_type t_type; + const uint8_t *t_data; + uint16_t t_size; + + int rc = fds_file_write_tmplt_get(m_file.get(), tmplt_id, &t_type, &t_data, &t_size); + if (rc != FDS_OK && rc != FDS_ERR_NOTFOUND) { + // Something bad happened + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("fds_file_write_tmplt_get() failed: " + std::string(err_msg)); + } + + if (rc == FDS_ERR_NOTFOUND || t_type != rec_tmplt->type || t_size != rec_tmplt->raw.length + || memcmp(t_data, rec_tmplt->raw.data, rec_tmplt->raw.length) != 0) { + // Template not defined or the template are different + t_type = rec_tmplt->type; + t_data = rec_tmplt->raw.data; + t_size = rec_tmplt->raw.length; + + if (fds_file_write_tmplt_add(m_file.get(), t_type, t_data, t_size) != FDS_OK) { + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to add a template: " + std::string(err_msg)); + } + } + + // FIXME: check subTemplateList & subTemplateMultiList templates + + // Write the Data record + const uint8_t *rec_data = rec_ptr->rec.data; + const uint16_t rec_size = rec_ptr->rec.size; + if (fds_file_write_rec(m_file.get(), tmplt_id, rec_data, rec_size) != FDS_OK) { + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to add a Data Record: " + std::string(err_msg)); + } + } +} + +/** + * @brief Create a filename based for a user defined timestamp + * @note The timestamp will be expressed in Coordinated Universal Time (UTC) + * + * @param[in] ts Timestamp of the file + * @return New filename + * @throw FDS_exception if formatting functions fail. + */ +std::string +Storage::filename_gen(const time_t &ts) +{ + const char pattern[] = "%Y/%m/%d/flows.%Y%m%d%H%M%S.fds"; + constexpr size_t buffer_size = 64; + char buffer_data[buffer_size]; + + struct tm utc_time; + if (!gmtime_r(&ts, &utc_time)) { + throw FDS_exception("gmtime_r() failed"); + } + + if (strftime(buffer_data, buffer_size, pattern, &utc_time) == 0) { + throw FDS_exception("strftime() failed"); + } + + std::string new_path = m_path; + if (new_path.back() != '/') { + new_path += '/'; + } + + return new_path + buffer_data; +} + +/** + * @brief Convert IPv4 address to IPv4-mapped IPv6 address + * + * New IPv6 address has value corresponding to "::FFFF:\" + * @warning Size of @p in must be at least 4 bytes and @p out at least 16 bytes! + * @param[in] in IPv4 address + * @param[out] out IPv4-mapped IPv6 address + */ +void +Storage::ipv4toipv6(const uint8_t *in, uint8_t *out) +{ + memset(out, 0, 16U); + *(uint16_t *) &out[10] = UINT16_MAX; // 0xFFFF + memcpy(&out[12], in, 4U); // Copy IPv4 address +} + +/** + * @brief Get file identification of a Transport Session + * + * If the identification doesn't exist, the function will add it to the file and create + * a new internal record for it. + * + * @param[in] sptr Transport Session to find + * @return Internal description + * @throw FDS_exception if the function failed to add a new Transport Session + */ +struct Storage::session_ctx & +Storage::session_get(const struct ipx_session *sptr) +{ + auto res_it = m_session2params.find(sptr); + if (res_it != m_session2params.end()) { + // Found + return res_it->second; + } + + // Not found -> register a new session + assert(m_file != nullptr && "File must be opened!"); + + struct fds_file_session new_session; + fds_file_sid_t new_sid; + + session_ipx2fds(sptr, &new_session); + if (fds_file_session_add(m_file.get(), &new_session, &new_sid) != FDS_OK) { + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to register Transport Session '" + std::string(sptr->ident) + + "': " + err_msg); + } + + // Create a new session + struct session_ctx &ctx = m_session2params[sptr]; + ctx.id = new_sid; + return ctx; +} + +/** + * @brief Convert IPFIXcol representation of a Transport Session to FDS representation + * @param[in] ipx_desc IPFIXcol specific representation + * @param[out] fds_desc FDS specific representation + * @throw FDS_exception if conversion fails due to unsupported Transport Session type + */ +void +Storage::session_ipx2fds(const struct ipx_session *ipx_desc, struct fds_file_session *fds_desc) +{ + // Initialize Transport Session structure + memset(fds_desc, 0, sizeof *fds_desc); + + // Extract protocol type and description + const struct ipx_session_net *net_desc = nullptr; + switch (ipx_desc->type) { + case FDS_SESSION_UDP: + net_desc = &ipx_desc->udp.net; + fds_desc->proto = FDS_FILE_SESSION_UDP; + break; + case FDS_SESSION_TCP: + net_desc = &ipx_desc->tcp.net; + fds_desc->proto = FDS_FILE_SESSION_TCP; + break; + case FDS_SESSION_SCTP: + net_desc = &ipx_desc->sctp.net; + fds_desc->proto = FDS_FILE_SESSION_SCTP; + break; + default: + throw FDS_exception("Not implemented Transport Session type!"); + } + + // Convert ports + fds_desc->port_src = net_desc->port_src; + fds_desc->port_dst = net_desc->port_dst; + + // Convert IP addresses + if (net_desc->l3_proto == AF_INET) { + // IPv4 address + static_assert(sizeof(net_desc->addr_src.ipv4) >= 4U, "Invalid size"); + static_assert(sizeof(net_desc->addr_dst.ipv4) >= 4U, "Invalid size"); + ipv4toipv6(reinterpret_cast(&net_desc->addr_src.ipv4), fds_desc->ip_src); + ipv4toipv6(reinterpret_cast(&net_desc->addr_dst.ipv4), fds_desc->ip_dst); + } else { + // IPv6 address + static_assert(sizeof(fds_desc->ip_src) <= sizeof(net_desc->addr_src.ipv6), "Invalid size"); + static_assert(sizeof(fds_desc->ip_dst) <= sizeof(net_desc->addr_dst.ipv6), "Invalid size"); + memcpy(&fds_desc->ip_src, &net_desc->addr_src.ipv6, sizeof fds_desc->ip_src); + memcpy(&fds_desc->ip_dst, &net_desc->addr_dst.ipv6, sizeof fds_desc->ip_dst); + } +} + diff --git a/src/plugins/output/fds/src/Storage.hpp b/src/plugins/output/fds/src/Storage.hpp new file mode 100644 index 00000000..7c022a00 --- /dev/null +++ b/src/plugins/output/fds/src/Storage.hpp @@ -0,0 +1,106 @@ +/** + * \file src/plugins/output/fds/src/Storage.hpp + * \author Lukas Hutak + * \brief FDS file storage (header file) + * \date June 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPFIXCOL2_FDS_STORAGE_HPP +#define IPFIXCOL2_FDS_STORAGE_HPP + +#include +#include +#include +#include +#include + +#include "Exception.hpp" +#include "Config.hpp" + +/// Flow storage file +class Storage { +public: + /** + * @brief Create a flow storage file + * + * @note + * Output file for the current window MUST be specified using new_window() function. + * Otherwise, no flow records are stored. + * + * @param[in] ctx Plugin context (only for log) + * @param[in] cfg Configuration + * @throw FDS_exception if @p path directory doesn't exist in the system + */ + Storage(ipx_ctx_t *ctx, const Config &cfg); + virtual ~Storage() = default; + + // Disable copy constructors + Storage(const Storage &other) = delete; + Storage &operator=(const Storage &other) = delete; + + /** + * @brief Create a new time window + * + * @note Previous window is automatically closed, if exists. + * @param[in] ts Timestamp of the window + * @throw FDS_exception if the new window cannot be created + */ + void + window_new(time_t ts); + + /** + * @brief Close the current time window + * @note + * This can be also useful if a fatal error has occurred and we should not add more flow + * records to the file. + * @note + * No more Data Records will be added until a new window is created! + */ + void + window_close(); + + /** + * @brief Process IPFIX message + * + * Process all IPFIX Data Records in the message and store them to the file. + * @note If a time window is not opened, no Data Records are stored and no exception is thrown. + * @param[in] msg Message to process + * @throw FDS_exception if processing fails + */ + void + process_msg(ipx_msg_ipfix_t *msg); + +private: + /// Description parameters of a Transport Session + struct session_ctx { + /// FDS file ID + fds_file_sid_t id; + }; + + /// Plugin context only for logging! + ipx_ctx_t *m_ctx; + /// Storage path + std::string m_path; + /// Flags for opening file + uint32_t m_flags; + + /// Output FDS file + std::unique_ptr m_file = {nullptr, &fds_file_close}; + /// Mapping of Transport Sessions to FDS specific parameters + std::map m_session2params; + + std::string + filename_gen(const time_t &ts); + static void + ipv4toipv6(const uint8_t *in, uint8_t *out); + struct session_ctx & + session_get(const struct ipx_session *sptr); + void + session_ipx2fds(const struct ipx_session *ipx_desc, struct fds_file_session *fds_desc); +}; + + +#endif // IPFIXCOL2_FDS_STORAGE_HPP diff --git a/src/plugins/output/fds/src/fds.cpp b/src/plugins/output/fds/src/fds.cpp new file mode 100644 index 00000000..c0ecedbe --- /dev/null +++ b/src/plugins/output/fds/src/fds.cpp @@ -0,0 +1,127 @@ +/** + * \file src/plugins/output/dummy/dummy.c + * \author Lukas Hutak + * \brief Example output plugin for IPFIXcol 2 + * \date 2018 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include + +#include "Config.hpp" +#include "Storage.hpp" + +/// Plugin description +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin identification name + "fds", + // Brief description of plugin + "Flow Data Storage output plugin", + // Plugin type + IPX_PT_OUTPUT, + // Configuration flags (reserved for future use) + 0, + // Plugin version string (like "1.2.3") + "2.0.0", + // Minimal IPFIXcol version string (like "1.2.3") + "2.0.0" +}; + +/// Instance +struct Instance { + /// Parsed configuration + std::unique_ptr config_ptr = nullptr; + /// Storage file + std::unique_ptr storage_ptr = nullptr; + /// Start of the current window + time_t window_start = 0; +}; + +static void +window_check(struct Instance &inst) +{ + const Config &cfg = *inst.config_ptr; + + // Decide whether close file and create a new time window + time_t now = time(NULL); + if (difftime(now, inst.window_start) < cfg.m_window.size) { + // Nothing to do + return; + } + + if (cfg.m_window.align) { + const uint32_t window_size = cfg.m_window.size; + now /= window_size; + now *= window_size; + } + + inst.window_start = now; + inst.storage_ptr->window_new(now); +} + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *params) +{ + try { + // Parse configuration, try to create a storage and time window + std::unique_ptr instance(new Instance); + instance->config_ptr.reset(new Config(params)); + instance->storage_ptr.reset(new Storage(ctx, *instance->config_ptr)); + window_check(*instance); + // Everything seems OK + ipx_ctx_private_set(ctx, instance.release()); + } catch (const FDS_exception &ex) { + IPX_CTX_ERROR(ctx, "Initialization failed: %s", ex.what()); + return IPX_ERR_DENIED; + } catch (...) { + IPX_CTX_ERROR(ctx, "Unknown error has occurred!", '\0'); + return IPX_ERR_DENIED; + } + + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *cfg) +{ + (void) ctx; // Suppress warnings + + try { + auto inst = reinterpret_cast(cfg); + inst->storage_ptr.reset(); + inst->config_ptr.reset(); + delete inst; + } catch (...) { + IPX_CTX_ERROR(ctx, "Something bad happened during plugin destruction"); + } +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) +{ + auto *inst = reinterpret_cast(cfg); + + try { + // Check if the current time window should be closed + window_check(*inst); + ipx_msg_ipfix_t *msg_ipfix = ipx_msg_base2ipfix(msg); + inst->storage_ptr->process_msg(msg_ipfix); + } catch (const FDS_exception &ex) { + IPX_CTX_ERROR(ctx, "%s", ex.what()); + inst->storage_ptr->window_close(); + } catch (std::exception &ex) { + IPX_CTX_ERROR(ctx, "Unexpected error has occurred: %s", ex.what()); + inst->storage_ptr->window_close(); + } catch (...) { + IPX_CTX_ERROR(ctx, "Unknown error has occurred!"); + inst->storage_ptr->window_close(); + } + + return IPX_OK; +} From 8002be6c7456ee888a82e270e48a25aae56f0027 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Fri, 21 Jun 2019 16:22:20 +0200 Subject: [PATCH 16/22] Utils: add a new section with utilities useful for plugins --- include/CMakeLists.txt | 1 + include/ipfixcol2.h | 1 + include/ipfixcol2/utils.h | 70 ++++++++++++++++++++++++++++++ src/core/utils.c | 91 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 include/ipfixcol2/utils.h diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index e14eacfc..bc1eef5e 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -16,6 +16,7 @@ set(SUB_HEADERS ipfixcol2/message_session.h ipfixcol2/plugins.h ipfixcol2/session.h + ipfixcol2/utils.h ipfixcol2/verbose.h "${PROJECT_BINARY_DIR}/include/ipfixcol2/api.h" ) diff --git a/include/ipfixcol2.h b/include/ipfixcol2.h index 6ca3b2f4..d40fefbe 100644 --- a/include/ipfixcol2.h +++ b/include/ipfixcol2.h @@ -63,6 +63,7 @@ #include #include +#include #include diff --git a/include/ipfixcol2/utils.h b/include/ipfixcol2/utils.h new file mode 100644 index 00000000..e65fd143 --- /dev/null +++ b/include/ipfixcol2/utils.h @@ -0,0 +1,70 @@ +/** + * @file include/ipfixcol2/utils.h + * @author Lukas Hutak + * @brief Auxiliary utilities for plugins (header file) + * @date June 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPX_UTILS_H +#define IPX_UTILS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include // mkdir file permissions + +/** +* \defgroup ipxSource Transport session identification +* \ingroup publicAPIs +* \brief Transport session interface +* +* Data types and API functions for identification and management of Transport +* session identification. The Exporting Process uses the Transport Session to +* send messages from multiple _independent_ Observation Domains to the +* Collecting Process. Moreover, in case of SCTP session messages are also send +* over _independent_ streams. +* +* Following structures represents Transport session between Exporting process +* and Collecting Process. However, proper processing of flows also requires +* distinguishing Observation Domain IDs and Stream identifications out of +* scope of these structures. +* +* @{ +*/ + +/** + * @brief Default file permission of newly created directories + * @note Read/write/execute for a user and his group, read/execute for others. + */ +#define IPX_UTILS_MKDIR_DEF (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) + +/** + * @brief Create recursively a directory + * + * @note + * File permission @p mode only affects newly created directories. In other + * words, if a directory (or subdirectory) already exists, file permission + * bits @p mode are not applied. + * @note + * The function is implemented as "recursive" wrapper over standard mkdir + * function. See man 3 mkdir for more information. + * @param[in] path Full directory path to create + * @param[in] mode The file permission bits of the new directories + * (see default value #IPX_UTILS_MKDIR_DEF) + * @return #IPX_OK on success + * @return #IPX_ERR_DENIED otherwise and errno is set appropriately. + */ +IPX_API int +ipx_utils_mkdir(const char *path, mode_t mode); + +/**@}*/ + +#ifdef __cplusplus +} +#endif +#endif // IPX_UTILS_H diff --git a/src/core/utils.c b/src/core/utils.c index 56aad98f..1d9049a6 100644 --- a/src/core/utils.c +++ b/src/core/utils.c @@ -43,10 +43,15 @@ #undef _GNU_SOURCE #define _POSIX_C_SOURCE 200809L + +#include +#include +#include #include -#include + #include "utils.h" + int ipx_strerror_fn(int errnum, char *buffer, size_t buffer_size) { @@ -57,4 +62,86 @@ ipx_strerror_fn(int errnum, char *buffer, size_t buffer_size) snprintf(buffer, buffer_size, "strerror_r() failed: Unable process error code %d!", errnum); return IPX_ERR_ARG; -} \ No newline at end of file +} + +int +ipx_utils_mkdir(const char *path, mode_t mode) +{ + const char ch_slash = '/'; + bool add_slash = false; + + // Check the parameter + size_t len = strlen(path); + if (path[len - 1] != ch_slash) { + len++; // We have to add another slash + add_slash = true; + } + + if (len > PATH_MAX - 1) { + errno = ENAMETOOLONG; + return IPX_ERR_DENIED; + } + + // Make a copy + char *path_cpy = malloc((len + 1) * sizeof(char)); // +1 for '\0' + if (!path_cpy) { + errno = ENOMEM; + return IPX_ERR_DENIED; + } + + strcpy(path_cpy, path); + if (add_slash) { + path_cpy[len - 1] = ch_slash; + path_cpy[len] = '\0'; + } + + struct stat info; + char *pos; + + // Create directories from the beginning + for (pos = path_cpy + 1; *pos; pos++) { + // Find a slash + if (*pos != ch_slash) { + continue; + } + + *pos = '\0'; // Temporarily truncate pathname + + // Check if a subdirectory exists + if (stat(path_cpy, &info) == 0) { + // Check if the "info" is about directory + if (!S_ISDIR(info.st_mode)) { + free(path_cpy); + errno = ENOTDIR; + return IPX_ERR_DENIED; + } + + // Fix the pathname and continue with the next subdirectory + *pos = ch_slash; + continue; + } + + // Errno is filled by stat() + if (errno != ENOENT) { + int errno_cpy = errno; + free(path_cpy); + errno = errno_cpy; + return IPX_ERR_DENIED; + } + + // Required directory doesn't exist -> create new one + if (mkdir(path_cpy, mode) != 0 && errno != EEXIST) { + // Failed (by the way, EEXIST because of race condition i.e. + // multiple applications creating the same folder) + int errno_cpy = errno; + free(path_cpy); + errno = errno_cpy; + return IPX_ERR_DENIED; + } + + *pos = ch_slash; + } + + free(path_cpy); + return IPX_OK; +} From 2965d76fe740dbef48f40e1ca2624f38bc662296 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 2 Jul 2019 13:32:25 +0200 Subject: [PATCH 17/22] CMake: fix failing build due to unsupported CONCAT command (CMake version 2.8.12) Thanks to "dan0nl" for reporting the issue. --- pkg/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/CMakeLists.txt b/pkg/CMakeLists.txt index 9f3c4847..9d818c30 100644 --- a/pkg/CMakeLists.txt +++ b/pkg/CMakeLists.txt @@ -51,8 +51,7 @@ if (NOT CPACK_PACKAGE_CONTACT OR CPACK_PACKAGE_CONTACT STREQUAL "") if (GIT_USER_NAME AND GIT_USER_EMAIL) message(STATUS "Maintainer contact for packages is not specified " "- using a name and email from the git configuration") - string(CONCAT CPACK_PACKAGE_CONTACT - ${GIT_USER_NAME} " <" ${GIT_USER_EMAIL} ">") + set(CPACK_PACKAGE_CONTACT "${GIT_USER_NAME} <${GIT_USER_EMAIL}>") endif() endif() From 21897b87773b56e876e5d4c345c52100805cc7b0 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 2 Jul 2019 13:34:14 +0200 Subject: [PATCH 18/22] RPM & DEB packages: Update dependencies in RPM spec file and DEB control file Now it requires libfds >= 0.2.0 --- pkg/deb/templates/changelog | 2 +- pkg/deb/templates/control.in | 4 ++-- pkg/deb/templates/copyright.in | 2 +- pkg/rpm/ipfixcol2.spec.in | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/deb/templates/changelog b/pkg/deb/templates/changelog index d9137276..a9346490 100644 --- a/pkg/deb/templates/changelog +++ b/pkg/deb/templates/changelog @@ -1,4 +1,4 @@ -ipfixcol2 (2.0.0-1) unstable; urgency=low +ipfixcol2 (2.1.0-1) unstable; urgency=low * Initial release. diff --git a/pkg/deb/templates/control.in b/pkg/deb/templates/control.in index 1f21a7ac..1529f624 100644 --- a/pkg/deb/templates/control.in +++ b/pkg/deb/templates/control.in @@ -4,12 +4,12 @@ Homepage: http://www.liberouter.org/ Section: net Priority: optional Standards-Version: 3.9.8 -Build-Depends: debhelper (>= 10), cmake (>= 2.8.8), doxygen, make, +Build-Depends: debhelper (>= 10), cmake (>= 2.8.8), doxygen, make (>= 4.0), libfds-dev, gcc (>= 4.8), g++ (>= 4.8), pkg-config Package: @CPACK_PACKAGE_NAME@ Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, libfds (>= 0.1.0) +Depends: ${shlibs:Depends}, ${misc:Depends}, libfds (>= 0.2.0) Description: @CPACK_PACKAGE_DESCRIPTION_SUMMARY@ IPFIXcol is a flexible IPFIX (RFC 7011) flow data collector designed to be extensible by plugins. diff --git a/pkg/deb/templates/copyright.in b/pkg/deb/templates/copyright.in index 815986e1..62b37912 100644 --- a/pkg/deb/templates/copyright.in +++ b/pkg/deb/templates/copyright.in @@ -3,7 +3,7 @@ Upstream-Contact: @CPACK_PACKAGE_CONTACT@ Source: http://www.liberouter.org/ Files: * -Copyright: 2015-2017 CESNET, z.s.p.o. +Copyright: 2015-2019 CESNET, z.s.p.o. License: BSD-3-clause Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/pkg/rpm/ipfixcol2.spec.in b/pkg/rpm/ipfixcol2.spec.in index 026a3e31..63345520 100644 --- a/pkg/rpm/ipfixcol2.spec.in +++ b/pkg/rpm/ipfixcol2.spec.in @@ -13,7 +13,7 @@ Packager: @CPACK_PACKAGE_CONTACT@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} BuildRequires: gcc >= 4.8, gcc-c++ >= 4.8, cmake >= 2.8.8, make BuildRequires: libfds-devel, python2-docutils -Requires: libfds >= 0.1.0 +Requires: libfds >= 0.2.0 %description IPFIXcol is a flexible IPFIX (RFC 7011) flow data collector designed to From 9b23fd8cc30b7b710056df027d370c96f0df7836 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 3 Jul 2019 17:03:00 +0200 Subject: [PATCH 19/22] FDS output: improved detection of Template modifications Detection is newly based on detection of snapshot changes. If a template snapshot of a Data Record is different (compared to a previous Data Record from the same source and ODID), all Templates in the new snapshot are compared to Templates already added to the file. New or redefined Templates are added and Templates that not defined in the new snapshot are removed. --- src/plugins/output/fds/src/Storage.cpp | 199 +++++++++++++++++++++---- src/plugins/output/fds/src/Storage.hpp | 20 ++- 2 files changed, 188 insertions(+), 31 deletions(-) diff --git a/src/plugins/output/fds/src/Storage.cpp b/src/plugins/output/fds/src/Storage.cpp index 64655a66..30db3d25 100644 --- a/src/plugins/output/fds/src/Storage.cpp +++ b/src/plugins/output/fds/src/Storage.cpp @@ -8,6 +8,8 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include +#include #include #include #include @@ -69,9 +71,9 @@ Storage::window_new(time_t ts) } if (fds_file_open(m_file.get(), new_file.c_str(), m_flags) != FDS_OK) { + std::string err_msg = fds_file_error(m_file.get()); m_file.reset(); - const char *err_msg = "something"; //fds_file_error(m_file.get()); - throw FDS_exception("Failed to create file '" + new_file + "': " + err_msg); + throw FDS_exception("Failed to create/append file '" + new_file + "': " + err_msg); } } @@ -100,51 +102,188 @@ Storage::process_msg(ipx_msg_ipfix_t *msg) if (fds_file_write_ctx(m_file.get(), file_ctx.id, msg_ctx->odid, exp_time) != FDS_OK) { const char *err_msg = fds_file_error(m_file.get()); - throw FDS_exception("Failed to configure file writer: " + std::string(err_msg)); + throw FDS_exception("Failed to configure the writer: " + std::string(err_msg)); } + // Get info about the last seen Template snapshot + struct snap_info &snap_last = file_ctx.odid2snap[msg_ctx->odid]; + // For each Data Record in the file const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); for (uint32_t i = 0; i < rec_cnt; ++i) { ipx_ipfix_record *rec_ptr = ipx_msg_ipfix_get_drec(msg, i); - // Insert the record // TODO: improve me! - const struct fds_template *rec_tmplt = rec_ptr->rec.tmplt; - uint16_t tmplt_id = rec_tmplt->id; - enum fds_template_type t_type; - const uint8_t *t_data; - uint16_t t_size; + // Check if the templates has been changed (detected by change of template snapshots) + if (rec_ptr->rec.snap != snap_last.ptr) { + const char *session_name = msg_ctx->session->ident; + uint32_t session_odid = msg_ctx->odid; + IPX_CTX_DEBUG(m_ctx, "Template snapshot of '%s' [ODID %" PRIu32 "] has been changed. " + "Updating template definitions...", session_name, session_odid); - int rc = fds_file_write_tmplt_get(m_file.get(), tmplt_id, &t_type, &t_data, &t_size); - if (rc != FDS_OK && rc != FDS_ERR_NOTFOUND) { - // Something bad happened + tmplts_update(snap_last, rec_ptr->rec.snap); + } + + // Write the Data Record + const uint8_t *rec_data = rec_ptr->rec.data; + uint16_t rec_size = rec_ptr->rec.size; + uint16_t tmplt_id = rec_ptr->rec.tmplt->id; + + if (fds_file_write_rec(m_file.get(), tmplt_id, rec_data, rec_size) != FDS_OK) { const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to add a Data Record: " + std::string(err_msg)); + } + } +} + +/// Auxiliary data structure used in the snapshot iterator +struct tmplt_update_data { + /// Status of template processing + bool is_ok; + /// Plugin context (only for log!) + ipx_ctx_t *ctx; + + /// FDS file with specified context + fds_file_t *file; + /// Set of processed Templates in the snapshot + std::set ids; +}; + +/** + * @brief Callback function for updating definition of an IPFIX (Options) Template + * + * The function checks if the same Template is already defined in the current context of the file. + * If the Template is not present or it's different, the new Template definition is added to the + * file. + * @param[in] tmplt Template to process + * @param[in] data Auxiliary data structure \ref tmplt_update_data + * @return On success returns true. Otherwise returns false. + */ +static bool +tmplt_update_cb(const struct fds_template *tmplt, void *data) +{ + // Template type, raw data and size + enum fds_template_type t_type; + const uint8_t *t_data; + uint16_t t_size; + + auto info = reinterpret_cast(data); + + // No exceptions can be thrown in the C callback! + try { + uint16_t t_id = tmplt->id; + info->ids.emplace(t_id); + + // Get definition of the Template specified in the file + int res = fds_file_write_tmplt_get(info->file, t_id, &t_type, &t_data, &t_size); + + if (res != FDS_OK && res != FDS_ERR_NOTFOUND) { + // Something bad happened + const char *err_msg = fds_file_error(info->file); throw FDS_exception("fds_file_write_tmplt_get() failed: " + std::string(err_msg)); } - if (rc == FDS_ERR_NOTFOUND || t_type != rec_tmplt->type || t_size != rec_tmplt->raw.length - || memcmp(t_data, rec_tmplt->raw.data, rec_tmplt->raw.length) != 0) { - // Template not defined or the template are different - t_type = rec_tmplt->type; - t_data = rec_tmplt->raw.data; - t_size = rec_tmplt->raw.length; - - if (fds_file_write_tmplt_add(m_file.get(), t_type, t_data, t_size) != FDS_OK) { - const char *err_msg = fds_file_error(m_file.get()); - throw FDS_exception("Failed to add a template: " + std::string(err_msg)); - } + // Should we add/redefine the definition of the Template + if (res == FDS_OK + && tmplt->type == t_type + && tmplt->raw.length == t_size + && memcmp(tmplt->raw.data, t_data, t_size) == 0) { + // The same -> nothing to do + return info->is_ok; } - // FIXME: check subTemplateList & subTemplateMultiList templates + // Add the definition (i.e. templates are different or the template hasn't been defined) + IPX_CTX_DEBUG(info->ctx, "Adding/updating definition of Template ID %" PRIu16, t_id); - // Write the Data record - const uint8_t *rec_data = rec_ptr->rec.data; - const uint16_t rec_size = rec_ptr->rec.size; - if (fds_file_write_rec(m_file.get(), tmplt_id, rec_data, rec_size) != FDS_OK) { - const char *err_msg = fds_file_error(m_file.get()); - throw FDS_exception("Failed to add a Data Record: " + std::string(err_msg)); + t_type = tmplt->type; + t_data = tmplt->raw.data; + t_size = tmplt->raw.length; + + if (fds_file_write_tmplt_add(info->file, t_type, t_data, t_size) != FDS_OK) { + const char *err_msg = fds_file_error(info->file); + throw FDS_exception("fds_file_write_tmplt_add() failed: " + std::string(err_msg)); + } + + } catch (std::exception &ex) { + // Exceptions + IPX_CTX_ERROR(info->ctx, "Failure during update of Template ID %" PRIu16 ": %s", tmplt->id, + ex.what()); + info->is_ok = false; + } catch (...) { + // Other exceptions + IPX_CTX_ERROR(info->ctx, "Unknown exception thrown during template definition update", '\0'); + info->is_ok = false; + } + + return info->is_ok; +} + +/** + * @brief Update Template definitions for the current Transport Session and ODID + * + * The function compares Templates in the \p snap with Template definitions previously defined + * for the currently selected combination of Transport Session and ODID. For each different or + * previously undefined Template, its definition is added or updated. Definitions of Templates + * that were available in the previous snapshot but not available in the new one are removed. + * + * Finally, information (pointer, IDs) in \p info are updated to reflect the performed update. + * @warning + * Template definitions are always unique for a combination of Transport Session and ODID, + * therefore, appropriate file writer context MUST already set using fds_file_writer_ctx(). + * Parameters \p info and \p snap MUST also belong the same unique combination. + * @param[in] info Information about the last update of Templates (old snapshot ref. + list of IDs) + * @param[in] snap New Template snapshot with all valid Template definitions + */ +void +Storage::tmplts_update(struct snap_info &info, const fds_tsnapshot_t *snap) +{ + assert(info.ptr != snap && "Snapshots should be different"); + + // Prepare data for the callback function + struct tmplt_update_data data; + data.is_ok = true; + data.ctx = m_ctx; + data.file = m_file.get(); + data.ids.clear(); + + // Update templates + fds_tsnapshot_for(snap, &tmplt_update_cb, &data); + + // Check if the update failed + if (!data.is_ok) { + throw FDS_exception("Failed to update Template definitions"); + } + + // Check if there are any Template IDs that have been removed + std::set &ids_old = info.tmplt_ids; + std::set &ids_new = data.ids; + std::set ids2remove; + // Old Template IDs - New Templates IDs = Template IDs to remove + std::set_difference(ids_old.begin(), ids_old.end(), ids_new.begin(), ids_new.end(), + std::inserter(ids2remove, ids2remove.begin())); + + // Remove old templates that are not available in the new snapshot + for (uint16_t tid : ids2remove) { + IPX_CTX_DEBUG(m_ctx, "Removing definition of Template ID %" PRIu16, tid); + + int rc = fds_file_write_tmplt_remove(m_file.get(), tid); + if (rc == FDS_OK) { + continue; + } + + // Something bad happened + if (rc != FDS_ERR_NOTFOUND) { + std::string err_msg = fds_file_error(m_file.get()); + throw FDS_exception("fds_file_write_tmplt_remove() failed: " + err_msg); } + + // Weird, but not critical + IPX_CTX_WARNING(m_ctx, "Failed to remove undefined Template ID %" PRIu16 ". " + "Weird, this should not happen.", tid); } + + // Update information about the last update of Templates + info.ptr = snap; + std::swap(info.tmplt_ids, data.ids); } /** diff --git a/src/plugins/output/fds/src/Storage.hpp b/src/plugins/output/fds/src/Storage.hpp index 7c022a00..29b0528e 100644 --- a/src/plugins/output/fds/src/Storage.hpp +++ b/src/plugins/output/fds/src/Storage.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -74,10 +75,25 @@ class Storage { process_msg(ipx_msg_ipfix_t *msg); private: + /// Information about Templates in a snapshot + struct snap_info { + /// Last seen snapshot (might be already freed, do NOT dereference!) + const fds_tsnapshot_t *ptr; + /// Set of Template IDs in the snapshot + std::set tmplt_ids; + + snap_info() { + ptr = nullptr; + tmplt_ids.clear(); + } + }; + /// Description parameters of a Transport Session struct session_ctx { - /// FDS file ID + /// Session ID used in the FDS file fds_file_sid_t id; + /// Last seen snapshot for a specific ODID of the Transport Session + std::map odid2snap; }; /// Plugin context only for logging! @@ -100,6 +116,8 @@ class Storage { session_get(const struct ipx_session *sptr); void session_ipx2fds(const struct ipx_session *ipx_desc, struct fds_file_session *fds_desc); + void + tmplts_update(struct snap_info &info, const fds_tsnapshot_t *snap); }; From 1e8d2c41ccad87ab5cf6284d00b21086a86e565b Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 3 Jul 2019 17:09:46 +0200 Subject: [PATCH 20/22] FDS output: update dependency of collector version, improve handling of exceptions during a message process --- src/plugins/output/fds/src/fds.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/plugins/output/fds/src/fds.cpp b/src/plugins/output/fds/src/fds.cpp index c0ecedbe..292e17ab 100644 --- a/src/plugins/output/fds/src/fds.cpp +++ b/src/plugins/output/fds/src/fds.cpp @@ -30,7 +30,7 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = { // Plugin version string (like "1.2.3") "2.0.0", // Minimal IPFIXcol version string (like "1.2.3") - "2.0.0" + "2.1.0" }; /// Instance @@ -106,6 +106,7 @@ int ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) { auto *inst = reinterpret_cast(cfg); + bool failed = false; try { // Check if the current time window should be closed @@ -114,12 +115,19 @@ ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) inst->storage_ptr->process_msg(msg_ipfix); } catch (const FDS_exception &ex) { IPX_CTX_ERROR(ctx, "%s", ex.what()); - inst->storage_ptr->window_close(); + failed = true; } catch (std::exception &ex) { IPX_CTX_ERROR(ctx, "Unexpected error has occurred: %s", ex.what()); - inst->storage_ptr->window_close(); + failed = true; } catch (...) { IPX_CTX_ERROR(ctx, "Unknown error has occurred!"); + failed = true; + } + + if (failed) { + IPX_CTX_ERROR(ctx, "Due to the previous error(s), the output file is possibly corrupted. " + "Therefore, no flow records are stored until a new file is automatically opened " + "after current window expiration."); inst->storage_ptr->window_close(); } From 27f6dccd72f6c9f0c1d8655537b34b8192856635 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 10 Jul 2019 13:16:28 +0200 Subject: [PATCH 21/22] CMake: add the new fds output plugin to default build --- src/plugins/output/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/output/CMakeLists.txt b/src/plugins/output/CMakeLists.txt index 10432b05..b5eaef58 100644 --- a/src/plugins/output/CMakeLists.txt +++ b/src/plugins/output/CMakeLists.txt @@ -1,5 +1,6 @@ # List of output plugin to build and install add_subdirectory(dummy) +add_subdirectory(fds) add_subdirectory(json) add_subdirectory(timecheck) -add_subdirectory(viewer) \ No newline at end of file +add_subdirectory(viewer) From cea838ea278a688eb8b7f8d94665fb7459319cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hut=C3=A1k?= Date: Wed, 10 Jul 2019 14:02:20 +0200 Subject: [PATCH 22/22] README: add reference to the new fds output plugin --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 15d12379..6cf5acca 100644 --- a/README.rst +++ b/README.rst @@ -35,6 +35,7 @@ network interface and a port. Multiple instances of these plugins can run concur **Output plugins** - store or forward your flows. +- `FDS `_ - store all flows in FDS file format (efficient long-term storage) - `JSON `_ - convert flow records to JSON and send/store them - `Viewer `_ - convert IPFIX into plain text and print it on standard output