diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f824c45..7a972f2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,16 +14,16 @@ endif() # Versions and other informations set(IPFIXCOL_VERSION_MAJOR 2) -set(IPFIXCOL_VERSION_MINOR 0) +set(IPFIXCOL_VERSION_MINOR 1) set(IPFIXCOL_VERSION_PATCH 0) set(IPFIXCOL_VERSION ${IPFIXCOL_VERSION_MAJOR}.${IPFIXCOL_VERSION_MINOR}.${IPFIXCOL_VERSION_PATCH}) # Include modules include(CMakeModules/git_info.cmake) +include(CMakeModules/install_dirs.cmake) include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) -include(GNUInstallDirs) # Include custom FindXXX modules list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") diff --git a/CMakeModules/install_dirs.cmake b/CMakeModules/install_dirs.cmake new file mode 100644 index 00000000..65f845a1 --- /dev/null +++ b/CMakeModules/install_dirs.cmake @@ -0,0 +1,44 @@ +# The purpose of this file is to automatically determine install directories +# +# If no directories are defined, use GNU install directories by default. +# However, in case of RPM build, install directories are typically passed +# to CMake as definitions that overwrites the default paths. +# + +include(GNUInstallDirs) + +# Binary directories +set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR}) + +# Library directories +if (DEFINED LIB_INSTALL_DIR) + set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR}) +else() + set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR}) +endif() + +# Include directories +if (DEFINED INCLUDE_INSTALL_DIR) + set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR}) +else() + set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR}) +endif() + +# System configuration +if (DEFINED SYSCONF_INSTALL_DIR) + set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR}) +else() + set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR}) +endif() + +# Share files (architecture independend data) +if (DEFINED SHARE_INSTALL_PREFIX) + set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX}) +else() + set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR}) +endif() + +set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/") +set(INSTALL_DIR_MAN "${INSTALL_DIR_SHARE}/man/") +set(INSTALL_DIR_DOC "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/") + diff --git a/README.rst b/README.rst index c9ea2dbc..6cf5acca 100644 --- a/README.rst +++ b/README.rst @@ -35,8 +35,12 @@ network interface and a port. Multiple instances of these plugins can run concur **Output plugins** - store or forward your flows. +- `FDS `_ - store all flows in FDS file format (efficient long-term storage) - `JSON `_ - convert flow records to JSON and send/store them -- `dummy `_ - simple module example +- `Viewer `_ - convert IPFIX into plain text and print + it on standard output +- `Time Check `_ - flow timestamp check +- `Dummy `_ - simple module example - `lnfstore `_ (*) - store all flows in nfdump compatible format for long-term preservation - `UniRec `_ (*) - send flow records in UniRec format diff --git a/extra_plugins/output/unirec/CMakeLists.txt b/extra_plugins/output/unirec/CMakeLists.txt index 6ae51b28..5908e7ce 100644 --- a/extra_plugins/output/unirec/CMakeLists.txt +++ b/extra_plugins/output/unirec/CMakeLists.txt @@ -1,20 +1,20 @@ cmake_minimum_required(VERSION 2.8.8) -project(unirec) +project(ipfixcol2-unirec-output) # Description of the project set(UNIREC_DESCRIPTION "Output plugin for IPFIXcol2 that sends flow records in UniRec format into NEMEA modules." ) -set(UNIREC_VERSION_MAJOR 1) +set(UNIREC_VERSION_MAJOR 2) set(UNIREC_VERSION_MINOR 0) set(UNIREC_VERSION_PATCH 0) set(UNIREC_VERSION ${UNIREC_VERSION_MAJOR}.${UNIREC_VERSION_MINOR}.${UNIREC_VERSION_PATCH}) +include(CMakeModules/install_dirs.cmake) include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) -include(GNUInstallDirs) # Include custom FindXXX modules list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") @@ -39,6 +39,12 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden -std=gnu++1 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -Wall -Wextra -pedantic") +# Prepare SPEC file +configure_file( + "${PROJECT_SOURCE_DIR}/ipfixcol2-unirec-output.spec.in" + "${PROJECT_BINARY_DIR}/ipfixcol2-unirec-output.spec" +) + # Header files for source code building include_directories( "${IPFIXCOL2_INCLUDE_DIRS}" # IPFIXcol2 header files @@ -67,11 +73,11 @@ target_link_libraries(unirec-output install( TARGETS unirec-output - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) install( FILES config/unirec-elements.txt - DESTINATION "${CMAKE_INSTALL_FULL_SYSCONFDIR}/ipfixcol2/" + DESTINATION "${INSTALL_DIR_SYSCONF}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -92,6 +98,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) endif() diff --git a/extra_plugins/output/unirec/CMakeModules/install_dirs.cmake b/extra_plugins/output/unirec/CMakeModules/install_dirs.cmake new file mode 100644 index 00000000..65f845a1 --- /dev/null +++ b/extra_plugins/output/unirec/CMakeModules/install_dirs.cmake @@ -0,0 +1,44 @@ +# The purpose of this file is to automatically determine install directories +# +# If no directories are defined, use GNU install directories by default. +# However, in case of RPM build, install directories are typically passed +# to CMake as definitions that overwrites the default paths. +# + +include(GNUInstallDirs) + +# Binary directories +set(INSTALL_DIR_BIN ${CMAKE_INSTALL_FULL_BINDIR}) + +# Library directories +if (DEFINED LIB_INSTALL_DIR) + set(INSTALL_DIR_LIB ${LIB_INSTALL_DIR}) +else() + set(INSTALL_DIR_LIB ${CMAKE_INSTALL_FULL_LIBDIR}) +endif() + +# Include directories +if (DEFINED INCLUDE_INSTALL_DIR) + set(INSTALL_DIR_INCLUDE ${INCLUDE_INSTALL_DIR}) +else() + set(INSTALL_DIR_INCLUDE ${CMAKE_INSTALL_FULL_INCLUDEDIR}) +endif() + +# System configuration +if (DEFINED SYSCONF_INSTALL_DIR) + set(INSTALL_DIR_SYSCONF ${SYSCONF_INSTALL_DIR}) +else() + set(INSTALL_DIR_SYSCONF ${CMAKE_INSTALL_FULL_SYSCONFDIR}) +endif() + +# Share files (architecture independend data) +if (DEFINED SHARE_INSTALL_PREFIX) + set(INSTALL_DIR_SHARE ${SHARE_INSTALL_PREFIX}) +else() + set(INSTALL_DIR_SHARE ${CMAKE_INSTALL_FULL_DATAROOTDIR}) +endif() + +set(INSTALL_DIR_INFO "${INSTALL_DIR_SHARE}/info/") +set(INSTALL_DIR_MAN "${INSTALL_DIR_SHARE}/man/") +set(INSTALL_DIR_DOC "${INSTALL_DIR_SHARE}/doc/${CMAKE_PROJECT_NAME}/") + diff --git a/extra_plugins/output/unirec/README.rst b/extra_plugins/output/unirec/README.rst index 4167022f..a10b155d 100644 --- a/extra_plugins/output/unirec/README.rst +++ b/extra_plugins/output/unirec/README.rst @@ -211,6 +211,8 @@ a line: ``"BYTES uint64 e0id1"``: the ```` element. - Second parameter specifies a data type of the UniRec field. List of all supported types is available in `UniRec documentation `_. + The plugin also supports additional virtual type "string_trimmed" that converts data to UniRec + string type, but only characters up to the first occurrence of '\\0' (excluding) are copied. - The third parameter is comma separated list of corresponding IPFIX Information Elements (IEs). In this case, "e0id1" means IPFIX IE with Private Enterprise ID 0 and IE ID 1 (which is "octetDeltaCount"). Instead of numeric identification an IE name in ":" format diff --git a/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in b/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in new file mode 100644 index 00000000..71c348de --- /dev/null +++ b/extra_plugins/output/unirec/ipfixcol2-unirec-output.spec.in @@ -0,0 +1,47 @@ +Name: ipfixcol2-unirec-output +Version: @UNIREC_VERSION@ +Release: 1%{?dist} +Summary: Plugin for converting IPFIX data to UniRec format. + +License: BSD +URL: http://www.liberouter.org/ +Source0: %{name}-%{version}.tar.gz +Group: Liberouter +Vendor: CESNET, z.s.p.o. +Packager: + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} +BuildRequires: gcc >= 4.8, cmake >= 2.8.8, make +BuildRequires: ipfixcol2-devel, libfds-devel, python2-docutils +BuildRequires: libtrap-devel, unirec >= 2.3.0 +Requires: libtrap >= 0.12.0, ipfixcol2 >= 2.0.0, libfds >= 0.1.0 + +%description +Plugin for converting IPFIX data to UniRec format. + +%prep +%autosetup + +#%post +#/sbin/ldconfig + +#%postun +#/sbin/ldconfig + +# Build source code +%build +%cmake . +make %{?_smp_mflags} + +# Perform installation into build directory +%install +make install DESTDIR=%{buildroot} + +%files +%{_libdir}/ipfixcol2/*.so* +%{_mandir}/man7/*.7.gz +%config(noreplace) %{_sysconfdir}/ipfixcol2/unirec-elements.txt + +%changelog +* Wed Nov 21 2018 Lukas Hutak 2.0.0-1 +- Initial RPM release diff --git a/extra_plugins/output/unirec/src/map.c b/extra_plugins/output/unirec/src/map.c index 130ac8b9..d5b96768 100644 --- a/extra_plugins/output/unirec/src/map.c +++ b/extra_plugins/output/unirec/src/map.c @@ -52,6 +52,8 @@ #define DEF_SIZE 32 /** Size of error buffer */ #define ERR_SIZE 256 +/** Special type of UniRec string field that is trimmed before conversion from IPFIX */ +#define TYPE_STRING_TRIM "string_trimmed" /** Internal structure of the mapping database */ struct map_s { @@ -234,13 +236,14 @@ map_elem_get_internal(const char *elem) * \param[in] ur_name UniRec name * \param[in] ur_type UniRec type * \param[in] ur_type_str UniRec type string + * \param[in] cflags Additional conversion flags * \param[in] ie_defs Definition of IPFIX IEs (comma separated list of definitions) * \param[in] line_id Line ID (just for error messages) * \return #IPX_OK on success * \return #IPX_ERR_NOMEM or #IPX_ERR_FORMAT on failure */ static int -map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, +map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, uint32_t cflags, const char *ie_defs, size_t line_id) { char *defs_cpy = strdup(ie_defs); @@ -264,6 +267,7 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, rec.unirec.name = ur_name; rec.unirec.type = ur_type; rec.unirec.type_str = ur_type_str; + rec.unirec.flags = cflags; // Process IPFIX fields char *subsave_ptr = NULL; @@ -323,6 +327,7 @@ map_load_line(map_t *map, const char *line, size_t line_id) int rc = IPX_OK; char *ur_name = NULL; char *ur_type_str = NULL; + uint32_t conv_flags = 0; ur_field_type_t ur_type; char *line_cpy = strdup(line); @@ -356,7 +361,15 @@ map_load_line(map_t *map, const char *line, size_t line_id) goto end; } - int type = ur_get_field_type_from_str(token); + int type; + if (strcmp(token, TYPE_STRING_TRIM) == 0) { + // Special version of string + type = UR_TYPE_STRING; + conv_flags |= MAP_FLAGS_STR_TRIM; + } else { + type = ur_get_field_type_from_str(token); + } + if (type == UR_E_INVALID_TYPE) { snprintf(map->err_buffer, ERR_SIZE, "Line %zu: Invalid type '%s' of UniRec field '%s'", line_id, token, ur_name); @@ -382,7 +395,7 @@ map_load_line(map_t *map, const char *line, size_t line_id) ptrdiff_t offset = token - line_cpy; const char *ie_defs = line + offset; - rc = map_load_line_ie_defs(map, ur_name, ur_type, ur_type_str, ie_defs, line_id); + rc = map_load_line_ie_defs(map, ur_name, ur_type, ur_type_str, conv_flags, ie_defs, line_id); end: free(ur_type_str); diff --git a/extra_plugins/output/unirec/src/map.h b/extra_plugins/output/unirec/src/map.h index 420570e6..35c4cad1 100644 --- a/extra_plugins/output/unirec/src/map.h +++ b/extra_plugins/output/unirec/src/map.h @@ -61,6 +61,11 @@ enum MAP_SRC { MAP_SRC_INTERNAL_DBF }; +enum MAP_FLAGS { + /** Perform trim of an IPFIX string (convert only characters before the first '\0') */ + MAP_FLAGS_STR_TRIM = (1 << 0) +}; + /** IPFIX-to-UniRec mapping record */ struct map_rec { struct { @@ -85,6 +90,11 @@ struct map_rec { ur_field_type_t type; /** Data type (string, for log!) */ char *type_str; + /** + * \brief Additional conversion flags + * \note See ::MAP_FLAGS + */ + uint32_t flags; } unirec; }; diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index db3730ee..899bf9f6 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -316,6 +316,22 @@ translate_bytes(translator_t *trans, const struct translator_rec *rec, return 0; } +/** + * \brief Convert IPFIX string to trimmed UniRec string + * + * The function will copy only characters up to the first occurrence of '\0' (excluding). + * \copydetails translate_uint() + */ +static int +translator_string_trim(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + size_t copy_len = strnlen((const char *) field->data, field->size); + ur_set_var(trans->record.ur_tmplt, trans->record.data, ur_id, field->data, copy_len); + return 0; +} + /** * \brief Convert IPFIX boolean to UniRec char/(un)signed integer * \copydetails translate_uint() @@ -699,6 +715,11 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) switch (type_ur) { case UR_TYPE_STRING: + // String array + if (type_ipx == FDS_ET_STRING && (rec->unirec.flags & MAP_FLAGS_STR_TRIM) != 0) { + return translator_string_trim; + } + // Fall through case UR_TYPE_BYTES: // String/byte array if (type_ipx == FDS_ET_STRING || type_ipx == FDS_ET_OCTET_ARRAY) { diff --git a/extra_plugins/output/unirec/src/unirecplugin.c b/extra_plugins/output/unirec/src/unirecplugin.c index 48cdfa3d..e7279451 100644 --- a/extra_plugins/output/unirec/src/unirecplugin.c +++ b/extra_plugins/output/unirec/src/unirecplugin.c @@ -78,7 +78,7 @@ IPX_API struct ipx_plugin_info ipx_plugin_info = { // Configuration flags (reserved for future use) .flags = 0, // Plugin version string (like "1.2.3") - .version = "2.0.0", + .version = "2.1.0", // Minimal IPFIXcol version string (like "1.2.3") .ipx_min = "2.0.0" }; diff --git a/include/CMakeLists.txt b/include/CMakeLists.txt index e14eacfc..bc1eef5e 100644 --- a/include/CMakeLists.txt +++ b/include/CMakeLists.txt @@ -16,6 +16,7 @@ set(SUB_HEADERS ipfixcol2/message_session.h ipfixcol2/plugins.h ipfixcol2/session.h + ipfixcol2/utils.h ipfixcol2/verbose.h "${PROJECT_BINARY_DIR}/include/ipfixcol2/api.h" ) diff --git a/include/ipfixcol2.h b/include/ipfixcol2.h index 6ca3b2f4..d40fefbe 100644 --- a/include/ipfixcol2.h +++ b/include/ipfixcol2.h @@ -63,6 +63,7 @@ #include #include +#include #include diff --git a/include/ipfixcol2/message_ipfix.h b/include/ipfixcol2/message_ipfix.h index e2ecb78d..84a4a365 100644 --- a/include/ipfixcol2/message_ipfix.h +++ b/include/ipfixcol2/message_ipfix.h @@ -158,9 +158,9 @@ ipx_msg_ipfix_get_ctx(ipx_msg_ipfix_t *msg); /** * \brief Get reference to all (Data/Template/Options Template) sets in the message - * \param msg Message - * \param sets Pointer to the array of sets - * \param size Number of sets in the array + * \param[in] msg Message + * \param[out] sets Pointer to the array of sets + * \param[out] size Number of sets in the array */ IPX_API void ipx_msg_ipfix_get_sets(ipx_msg_ipfix_t *msg, struct ipx_ipfix_set **sets, size_t *size); diff --git a/include/ipfixcol2/utils.h b/include/ipfixcol2/utils.h new file mode 100644 index 00000000..e65fd143 --- /dev/null +++ b/include/ipfixcol2/utils.h @@ -0,0 +1,70 @@ +/** + * @file include/ipfixcol2/utils.h + * @author Lukas Hutak + * @brief Auxiliary utilities for plugins (header file) + * @date June 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPX_UTILS_H +#define IPX_UTILS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include // mkdir file permissions + +/** +* \defgroup ipxSource Transport session identification +* \ingroup publicAPIs +* \brief Transport session interface +* +* Data types and API functions for identification and management of Transport +* session identification. The Exporting Process uses the Transport Session to +* send messages from multiple _independent_ Observation Domains to the +* Collecting Process. Moreover, in case of SCTP session messages are also send +* over _independent_ streams. +* +* Following structures represents Transport session between Exporting process +* and Collecting Process. However, proper processing of flows also requires +* distinguishing Observation Domain IDs and Stream identifications out of +* scope of these structures. +* +* @{ +*/ + +/** + * @brief Default file permission of newly created directories + * @note Read/write/execute for a user and his group, read/execute for others. + */ +#define IPX_UTILS_MKDIR_DEF (S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) + +/** + * @brief Create recursively a directory + * + * @note + * File permission @p mode only affects newly created directories. In other + * words, if a directory (or subdirectory) already exists, file permission + * bits @p mode are not applied. + * @note + * The function is implemented as "recursive" wrapper over standard mkdir + * function. See man 3 mkdir for more information. + * @param[in] path Full directory path to create + * @param[in] mode The file permission bits of the new directories + * (see default value #IPX_UTILS_MKDIR_DEF) + * @return #IPX_OK on success + * @return #IPX_ERR_DENIED otherwise and errno is set appropriately. + */ +IPX_API int +ipx_utils_mkdir(const char *path, mode_t mode); + +/**@}*/ + +#ifdef __cplusplus +} +#endif +#endif // IPX_UTILS_H diff --git a/pkg/CMakeLists.txt b/pkg/CMakeLists.txt index db1abd9a..9d818c30 100644 --- a/pkg/CMakeLists.txt +++ b/pkg/CMakeLists.txt @@ -51,8 +51,7 @@ if (NOT CPACK_PACKAGE_CONTACT OR CPACK_PACKAGE_CONTACT STREQUAL "") if (GIT_USER_NAME AND GIT_USER_EMAIL) message(STATUS "Maintainer contact for packages is not specified " "- using a name and email from the git configuration") - string(CONCAT CPACK_PACKAGE_CONTACT - ${GIT_USER_NAME} " <" ${GIT_USER_EMAIL} ">") + set(CPACK_PACKAGE_CONTACT "${GIT_USER_NAME} <${GIT_USER_EMAIL}>") endif() endif() @@ -67,7 +66,7 @@ if (PKG_CONFIG_FOUND) @ONLY) install( FILES "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2.pc" - DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/pkgconfig/") + DESTINATION "${INSTALL_DIR_LIB}/pkgconfig/") endif() # Package builders diff --git a/pkg/deb/CMakeLists.txt b/pkg/deb/CMakeLists.txt index 56361566..129af061 100644 --- a/pkg/deb/CMakeLists.txt +++ b/pkg/deb/CMakeLists.txt @@ -51,8 +51,8 @@ file(COPY "templates/changelog" "templates/rules" "templates/compat" - "templates/ipfixcol.install" - "templates/ipfixcol-dev.install" + "templates/ipfixcol2.install" + "templates/ipfixcol2-dev.install" DESTINATION "${DEB_CFG_DIR}" ) diff --git a/pkg/deb/templates/changelog b/pkg/deb/templates/changelog index 5cae7c6e..a9346490 100644 --- a/pkg/deb/templates/changelog +++ b/pkg/deb/templates/changelog @@ -1,5 +1,5 @@ -ipfixcol (2.0.0-1) unstable; urgency=low +ipfixcol2 (2.1.0-1) unstable; urgency=low * Initial release. - -- Lukas Hutak Mon, 19 Jun 2017 14:24:56 +0200 + -- Lukas Hutak Thu, 25 Oct 2018 10:17:56 +0200 diff --git a/pkg/deb/templates/control.in b/pkg/deb/templates/control.in index dde00e1d..1529f624 100644 --- a/pkg/deb/templates/control.in +++ b/pkg/deb/templates/control.in @@ -4,12 +4,12 @@ Homepage: http://www.liberouter.org/ Section: net Priority: optional Standards-Version: 3.9.8 -Build-Depends: debhelper (>= 10), cmake (>= 2.8.8), doxygen, pkg-config, - libfds-dev +Build-Depends: debhelper (>= 10), cmake (>= 2.8.8), doxygen, make (>= 4.0), + libfds-dev, gcc (>= 4.8), g++ (>= 4.8), pkg-config Package: @CPACK_PACKAGE_NAME@ Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, libfds (>= 0.1.0) +Depends: ${shlibs:Depends}, ${misc:Depends}, libfds (>= 0.2.0) Description: @CPACK_PACKAGE_DESCRIPTION_SUMMARY@ IPFIXcol is a flexible IPFIX (RFC 7011) flow data collector designed to be extensible by plugins. diff --git a/pkg/deb/templates/copyright.in b/pkg/deb/templates/copyright.in index 815986e1..62b37912 100644 --- a/pkg/deb/templates/copyright.in +++ b/pkg/deb/templates/copyright.in @@ -3,7 +3,7 @@ Upstream-Contact: @CPACK_PACKAGE_CONTACT@ Source: http://www.liberouter.org/ Files: * -Copyright: 2015-2017 CESNET, z.s.p.o. +Copyright: 2015-2019 CESNET, z.s.p.o. License: BSD-3-clause Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/pkg/deb/templates/ipfixcol.install b/pkg/deb/templates/ipfixcol.install deleted file mode 100644 index bc7b6898..00000000 --- a/pkg/deb/templates/ipfixcol.install +++ /dev/null @@ -1 +0,0 @@ -/usr/bin/ diff --git a/pkg/deb/templates/ipfixcol-dev.install b/pkg/deb/templates/ipfixcol2-dev.install similarity index 100% rename from pkg/deb/templates/ipfixcol-dev.install rename to pkg/deb/templates/ipfixcol2-dev.install diff --git a/pkg/deb/templates/ipfixcol2.install b/pkg/deb/templates/ipfixcol2.install new file mode 100644 index 00000000..765c78be --- /dev/null +++ b/pkg/deb/templates/ipfixcol2.install @@ -0,0 +1,3 @@ +/usr/bin/ +/usr/lib/*/ipfixcol2/lib*.so* +/usr/share/man/ diff --git a/pkg/ipfixcol2.pc.in b/pkg/ipfixcol2.pc.in index d0633e78..5a02f70a 100644 --- a/pkg/ipfixcol2.pc.in +++ b/pkg/ipfixcol2.pc.in @@ -1,5 +1,4 @@ -prefix=@CMAKE_INSTALL_PREFIX@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +includedir=@INSTALL_DIR_INCLUDE@ Name: @PROJECT_NAME@ Description: @IPFIXCOL_DESCRIPTION@ diff --git a/pkg/rpm/ipfixcol2.spec.in b/pkg/rpm/ipfixcol2.spec.in index 026a3e31..63345520 100644 --- a/pkg/rpm/ipfixcol2.spec.in +++ b/pkg/rpm/ipfixcol2.spec.in @@ -13,7 +13,7 @@ Packager: @CPACK_PACKAGE_CONTACT@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} BuildRequires: gcc >= 4.8, gcc-c++ >= 4.8, cmake >= 2.8.8, make BuildRequires: libfds-devel, python2-docutils -Requires: libfds >= 0.1.0 +Requires: libfds >= 0.2.0 %description IPFIXcol is a flexible IPFIX (RFC 7011) flow data collector designed to diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f931cd15..609f035c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,7 +9,7 @@ configure_file( ) # Find libfds -find_package(LibFds 0.1.0 REQUIRED) +find_package(LibFds 0.2.0 REQUIRED) # Find rst2man if (ENABLE_DOC_MANPAGE) diff --git a/src/build_config.h.in b/src/build_config.h.in index ab352ec6..d1175e1b 100644 --- a/src/build_config.h.in +++ b/src/build_config.h.in @@ -68,13 +68,13 @@ /** \brief Path to the directory with configuration */ #define IPX_DEFAULT_CONFIG_DIR \ - "@CMAKE_INSTALL_FULL_SYSCONFDIR@/ipfixcol2/" + "@INSTALL_DIR_SYSCONF@/ipfixcol2/" /** \brief Path to default (if not specified) user configuration file */ #define IPX_DEFAULT_STARTUP_CONFIG \ IPX_DEFAULT_CONFIG_DIR "startup.xml" /** \brief Path to the directory with plugins */ #define IPX_DEFAULT_PLUGINS_DIR \ - "@CMAKE_INSTALL_FULL_LIBDIR@/ipfixcol2/" + "@INSTALL_DIR_LIB@/ipfixcol2/" /** \brief Compiler name and version */ #define IPX_BUILD_COMPILER \ diff --git a/src/core/utils.c b/src/core/utils.c index 56aad98f..1d9049a6 100644 --- a/src/core/utils.c +++ b/src/core/utils.c @@ -43,10 +43,15 @@ #undef _GNU_SOURCE #define _POSIX_C_SOURCE 200809L + +#include +#include +#include #include -#include + #include "utils.h" + int ipx_strerror_fn(int errnum, char *buffer, size_t buffer_size) { @@ -57,4 +62,86 @@ ipx_strerror_fn(int errnum, char *buffer, size_t buffer_size) snprintf(buffer, buffer_size, "strerror_r() failed: Unable process error code %d!", errnum); return IPX_ERR_ARG; -} \ No newline at end of file +} + +int +ipx_utils_mkdir(const char *path, mode_t mode) +{ + const char ch_slash = '/'; + bool add_slash = false; + + // Check the parameter + size_t len = strlen(path); + if (path[len - 1] != ch_slash) { + len++; // We have to add another slash + add_slash = true; + } + + if (len > PATH_MAX - 1) { + errno = ENAMETOOLONG; + return IPX_ERR_DENIED; + } + + // Make a copy + char *path_cpy = malloc((len + 1) * sizeof(char)); // +1 for '\0' + if (!path_cpy) { + errno = ENOMEM; + return IPX_ERR_DENIED; + } + + strcpy(path_cpy, path); + if (add_slash) { + path_cpy[len - 1] = ch_slash; + path_cpy[len] = '\0'; + } + + struct stat info; + char *pos; + + // Create directories from the beginning + for (pos = path_cpy + 1; *pos; pos++) { + // Find a slash + if (*pos != ch_slash) { + continue; + } + + *pos = '\0'; // Temporarily truncate pathname + + // Check if a subdirectory exists + if (stat(path_cpy, &info) == 0) { + // Check if the "info" is about directory + if (!S_ISDIR(info.st_mode)) { + free(path_cpy); + errno = ENOTDIR; + return IPX_ERR_DENIED; + } + + // Fix the pathname and continue with the next subdirectory + *pos = ch_slash; + continue; + } + + // Errno is filled by stat() + if (errno != ENOENT) { + int errno_cpy = errno; + free(path_cpy); + errno = errno_cpy; + return IPX_ERR_DENIED; + } + + // Required directory doesn't exist -> create new one + if (mkdir(path_cpy, mode) != 0 && errno != EEXIST) { + // Failed (by the way, EEXIST because of race condition i.e. + // multiple applications creating the same folder) + int errno_cpy = errno; + free(path_cpy); + errno = errno_cpy; + return IPX_ERR_DENIED; + } + + *pos = ch_slash; + } + + free(path_cpy); + return IPX_OK; +} diff --git a/src/plugins/input/dummy/CMakeLists.txt b/src/plugins/input/dummy/CMakeLists.txt index 8806d1d7..2e3f465f 100644 --- a/src/plugins/input/dummy/CMakeLists.txt +++ b/src/plugins/input/dummy/CMakeLists.txt @@ -7,5 +7,5 @@ add_library(dummy-input MODULE install( TARGETS dummy-input - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" -) \ No newline at end of file + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) diff --git a/src/plugins/input/tcp/CMakeLists.txt b/src/plugins/input/tcp/CMakeLists.txt index c817da60..6e743889 100644 --- a/src/plugins/input/tcp/CMakeLists.txt +++ b/src/plugins/input/tcp/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(tcp-input MODULE install( TARGETS tcp-input - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -23,6 +23,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/input/udp/CMakeLists.txt b/src/plugins/input/udp/CMakeLists.txt index ab132396..4ff410ee 100644 --- a/src/plugins/input/udp/CMakeLists.txt +++ b/src/plugins/input/udp/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(udp-input MODULE install( TARGETS udp-input - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -23,6 +23,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/intermediate/anonymization/CMakeLists.txt b/src/plugins/intermediate/anonymization/CMakeLists.txt index 10db29d5..8cf6dc77 100644 --- a/src/plugins/intermediate/anonymization/CMakeLists.txt +++ b/src/plugins/intermediate/anonymization/CMakeLists.txt @@ -11,7 +11,7 @@ add_library(anonymization-intermediate MODULE install( TARGETS anonymization-intermediate - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -27,6 +27,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/output/CMakeLists.txt b/src/plugins/output/CMakeLists.txt index 5d3af5fb..b5eaef58 100644 --- a/src/plugins/output/CMakeLists.txt +++ b/src/plugins/output/CMakeLists.txt @@ -1,3 +1,6 @@ # List of output plugin to build and install add_subdirectory(dummy) -add_subdirectory(json) \ No newline at end of file +add_subdirectory(fds) +add_subdirectory(json) +add_subdirectory(timecheck) +add_subdirectory(viewer) diff --git a/src/plugins/output/dummy/CMakeLists.txt b/src/plugins/output/dummy/CMakeLists.txt index 9f60a012..fa18064e 100644 --- a/src/plugins/output/dummy/CMakeLists.txt +++ b/src/plugins/output/dummy/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(dummy-output MODULE install( TARGETS dummy-output - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -23,6 +23,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/output/fds/CMakeLists.txt b/src/plugins/output/fds/CMakeLists.txt new file mode 100644 index 00000000..2c2a6a83 --- /dev/null +++ b/src/plugins/output/fds/CMakeLists.txt @@ -0,0 +1,31 @@ +# Create a linkable module +add_library(fds-output MODULE + src/Config.cpp + src/Config.hpp + src/Exception.hpp + src/fds.cpp + src/Storage.cpp + src/Storage.hpp +) + +install( + TARGETS fds-output + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-fds-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-fds-output.7") + + add_custom_command(TARGET fds-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${INSTALL_DIR_MAN}/man7" + ) +endif() diff --git a/src/plugins/output/fds/README.rst b/src/plugins/output/fds/README.rst new file mode 100644 index 00000000..09b57efe --- /dev/null +++ b/src/plugins/output/fds/README.rst @@ -0,0 +1,69 @@ +Flow Data Storage (output plugin) +================================= + +The plugin converts and stores IPFIX Data Records into FDS file format. The file +is based on IPFIX, therefore, it provides highly-effective way for long-term +storage and stores complete flow records (including all Enterprise-specific +fields, biflow, etc.) together with identification of the flow exporters who +exported these records. + +All data are stored into flat files, which are automatically rotated and renamed +every N minutes (by default 5 minutes). + +Example configuration +--------------------- + +.. code-block:: xml + + + FDS output + fds + + /tmp/ipfixcol2/fds/ + none + + 300 + yes + + + + +Parameters +---------- + +:``storagePath``: + The path element specifies the storage directory for data files. Keep on + mind that the path must exist in your system. Otherwise, no files are stored. + All files will be stored based on the configuration using the following + template: ``/YYYY/MM/DD/flows..fds`` where ``YYYY/MM/DD`` + means year/month/day and ```` represents a UTC timestamp in + format ``YYMMDDhhmmss``. + +:``compression``: + Data compression helps to significantly reduce size of output files. + Following compression algorithms are available: + + :``none``: Compression disabled [default] + :``lz4``: LZ4 compression (very fast, slightly worse compression ration) + :``zstd``: ZSTD compression (slightly slower, good compression ration) + +:``dumpInterval``: + Configuration of output files rotation. + + :``timeWindow``: + Specifies time interval in seconds to rotate files i.e. close the current + file and create a new one. [default: 300] + + :``align``: + Align file rotation with next N minute interval. For example, if enabled + and window size is 5 minutes long, files will be created at 0, 5, 10, etc. + [values: yes/no, default: yes] + +:``asyncIO``: + Allows to use asynchronous I/O for writing to the file. Usually when parts + of the file are being written, the process is blocked on synchronous I/O + and waits for the operation to complete. However, asynchronous I/O allows + the plugin to simultaneously write to file and process flow records, which + significantly improves overall performance. (Note: a pool of service + threads shared among instances of FDS plugin might be created). + [values: true/false, default: true] diff --git a/src/plugins/output/fds/doc/ipfixcol2-fds-output.7.rst b/src/plugins/output/fds/doc/ipfixcol2-fds-output.7.rst new file mode 100644 index 00000000..a978730e --- /dev/null +++ b/src/plugins/output/fds/doc/ipfixcol2-fds-output.7.rst @@ -0,0 +1,20 @@ +====================== + ipfixcol2-fds-output +====================== + +--------------------------------- +Flow Data Storage (output plugin) +--------------------------------- + +:Author: Lukáš Huták (lukas.hutak@cesnet.cz) +:Date: 2019-07-01 +:Copyright: Copyright © 2019 CESNET, z.s.p.o. +:Version: 2.0 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 diff --git a/src/plugins/output/fds/src/Config.cpp b/src/plugins/output/fds/src/Config.cpp new file mode 100644 index 00000000..a06d61e2 --- /dev/null +++ b/src/plugins/output/fds/src/Config.cpp @@ -0,0 +1,190 @@ +/** + * \file src/plugins/output/fds/src/Config.cpp + * \author Lukas Hutak + * \brief Parser of XML configuration (source file) + * \date 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "Config.hpp" +#include +#include + +/* + * + * ... + * ... + * + * ... + * ... + * + * ... + * + */ + +/// XML nodes +enum params_xml_nodes { + NODE_STORAGE = 1, + NODE_COMPRESS, + NODE_DUMP, + NODE_ASYNCIO, + + DUMP_WINDOW, + DUMP_ALIGN +}; + +/// Definition of the \ node +static const struct fds_xml_args args_dump[] = { + FDS_OPTS_ELEM(DUMP_WINDOW, "timeWindow", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM(DUMP_ALIGN, "align", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_END +}; + +/// Definition of the \ node +static const struct fds_xml_args args_params[] = { + FDS_OPTS_ROOT("params"), + FDS_OPTS_ELEM(NODE_STORAGE, "storagePath", FDS_OPTS_T_STRING, 0), + FDS_OPTS_ELEM(NODE_COMPRESS, "compression", FDS_OPTS_T_STRING, FDS_OPTS_P_OPT), + FDS_OPTS_NESTED(NODE_DUMP, "dumpInterval", args_dump, FDS_OPTS_P_OPT), + FDS_OPTS_ELEM(NODE_ASYNCIO, "asyncIO", FDS_OPTS_T_BOOL, FDS_OPTS_P_OPT), + FDS_OPTS_END +}; + +Config::Config(const char *params) +{ + set_default(); + + // Create XML parser + std::unique_ptr xml(fds_xml_create(), &fds_xml_destroy); + if (!xml) { + throw std::runtime_error("Failed to create an XML parser!"); + } + + if (fds_xml_set_args(xml.get(), args_params) != FDS_OK) { + throw std::runtime_error("Failed to parse the description of an XML document!"); + } + + fds_xml_ctx_t *params_ctx = fds_xml_parse_mem(xml.get(), params, true); + if (!params_ctx) { + std::string err = fds_xml_last_err(xml.get()); + throw std::runtime_error("Failed to parse the configuration: " + err); + } + + // Parse parameters and check configuration + try { + parse_root(params_ctx); + validate(); + } catch (std::exception &ex) { + throw std::runtime_error("Failed to parse the configuration: " + std::string(ex.what())); + } +} + +/** + * @brief Set default parameters + */ +void +Config::set_default() +{ + m_path.clear(); + m_calg = calg::NONE; + m_async = true; + + m_window.align = true; + m_window.size = WINDOW_SIZE; +} + +/** + * @brief Check if the configuration is valid + * @throw runtime_error if the configuration breaks some rules + */ +void +Config::validate() +{ + if (m_path.empty()) { + throw std::runtime_error("Storage path cannot be empty!"); + } + + if (m_window.size == 0) { + throw std::runtime_error("Window size cannot be zero!"); + } +} + +/** + * @brief Process \ node + * @param[in] ctx XML context to process + * @throw runtime_error if the parser fails + */ +void +Config::parse_root(fds_xml_ctx_t *ctx) +{ + const struct fds_xml_cont *content; + while (fds_xml_next(ctx, &content) != FDS_EOC) { + switch (content->id) { + case NODE_STORAGE: + // Storage path + assert(content->type == FDS_OPTS_T_STRING); + m_path = content->ptr_string; + break; + case NODE_COMPRESS: + // Compression method + assert(content->type == FDS_OPTS_T_STRING); + if (strcasecmp(content->ptr_string, "none") == 0) { + m_calg = calg::NONE; + } else if (strcasecmp(content->ptr_string, "lz4") == 0) { + m_calg = calg::LZ4; + } else if (strcasecmp(content->ptr_string, "zstd") == 0) { + m_calg = calg::ZSTD; + } else { + const std::string inv_str = content->ptr_string; + throw std::runtime_error("Unknown compression algorithm '" + inv_str + "'"); + } + break; + case NODE_ASYNCIO: + // Asynchronous I/O + assert(content->type == FDS_OPTS_T_BOOL); + m_async = content->val_bool; + break; + case NODE_DUMP: + // Dump window + assert(content->type == FDS_OPTS_T_CONTEXT); + parse_dump(content->ptr_ctx); + break; + default: + // Internal error + throw std::runtime_error("Unknown XML node"); + } + } +} + +/** + * @brief Auxiliary function for parsing \ options + * @param[in] ctx XML context to process + * @throw runtime_error if the parser fails + */ +void +Config::parse_dump(fds_xml_ctx_t *ctx) +{ + const struct fds_xml_cont *content; + while(fds_xml_next(ctx, &content) != FDS_EOC) { + switch (content->id) { + case DUMP_WINDOW: + // Window size + assert(content->type == FDS_OPTS_T_UINT); + if (content->val_uint > UINT32_MAX) { + throw std::runtime_error("Window size is too long!"); + } + m_window.size = static_cast(content->val_uint); + break; + case DUMP_ALIGN: + // Window alignment + assert(content->type == FDS_OPTS_T_BOOL); + m_window.align = content->val_bool; + break; + default: + // Internal error + throw std::runtime_error("Unknown XML node"); + } + } +} \ No newline at end of file diff --git a/src/plugins/output/fds/src/Config.hpp b/src/plugins/output/fds/src/Config.hpp new file mode 100644 index 00000000..876ef466 --- /dev/null +++ b/src/plugins/output/fds/src/Config.hpp @@ -0,0 +1,64 @@ +/** + * \file src/plugins/output/fds/src/Config.hpp + * \author Lukas Hutak + * \brief Parser of XML configuration (header file) + * \date 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPFIXCOL2_FDS_CONFIG_HPP +#define IPFIXCOL2_FDS_CONFIG_HPP + +#include +#include + +/** + * @brief Plugin configuration parser + */ +class Config { +public: + /** + * @brief Parse configuration of the plugin + * @param[in] params XML parameters to parse + * @throw runtime_exception on error + */ + Config(const char *params); + ~Config() = default; + + enum class calg { + NONE, ///< Do not use compression + LZ4, ///< LZ4 compression + ZSTD ///< ZSTD compression + }; + + /// Storage path + std::string m_path; + /// Compression algorithm + calg m_calg; + /// Asynchronous I/O enabled + bool m_async; + + struct { + bool align; ///< Enable/disable window alignment + uint32_t size; ///< Time window size + } m_window; ///< Window alignment + +private: + /// Default window size + static const uint32_t WINDOW_SIZE = 300U; + + void + set_default(); + void + validate(); + + void + parse_root(fds_xml_ctx_t *ctx); + void + parse_dump(fds_xml_ctx_t *ctx); +}; + + +#endif // IPFIXCOL2_FDS_CONFIG_HPP diff --git a/src/plugins/output/fds/src/Exception.hpp b/src/plugins/output/fds/src/Exception.hpp new file mode 100644 index 00000000..f5325c5f --- /dev/null +++ b/src/plugins/output/fds/src/Exception.hpp @@ -0,0 +1,34 @@ +/** + * \file src/plugins/output/fds/src/Exception.hpp + * \author Lukas Hutak + * \brief Plugin specific exception (header file) + * \date 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPFIXCOL2_FDS_EXCEPTION_HPP +#define IPFIXCOL2_FDS_EXCEPTION_HPP + +#include +#include + +/// Plugin specific exception +class FDS_exception : public std::runtime_error { +public: + /** + * @brief Constructor + * @param[in] str Error message + */ + FDS_exception(const std::string &str) : std::runtime_error(str) {}; + /** + * @brief Constructor + * @param[in] str Error message + */ + FDS_exception(const char *str) : std::runtime_error(str) {}; + // Default destructor + ~FDS_exception() = default; +}; + +#endif //IPFIXCOL2_FDS_EXCEPTION_HPP diff --git a/src/plugins/output/fds/src/Storage.cpp b/src/plugins/output/fds/src/Storage.cpp new file mode 100644 index 00000000..30db3d25 --- /dev/null +++ b/src/plugins/output/fds/src/Storage.cpp @@ -0,0 +1,425 @@ +/** + * \file src/plugins/output/fds/src/Storage.cpp + * \author Lukas Hutak + * \brief FDS file storage (source file) + * \date June 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Storage.hpp" + +Storage::Storage(ipx_ctx_t *ctx, const Config &cfg) : m_ctx(ctx), m_path(cfg.m_path) +{ + // Check if the directory exists + struct stat file_info; + memset(&file_info, 0, sizeof(file_info)); + if (stat(m_path.c_str(), &file_info) != 0 || !S_ISDIR(file_info.st_mode)) { + throw FDS_exception("Directory '" + m_path + "' doesn't exist or search permission is denied"); + } + + // Prepare flags for FDS file + m_flags = 0; + switch (cfg.m_calg) { + case Config::calg::LZ4: + m_flags |= FDS_FILE_LZ4; + break; + case Config::calg::ZSTD: + m_flags |= FDS_FILE_ZSTD; + break; + default: + break; + } + + if (!cfg.m_async) { + m_flags |= FDS_FILE_NOASYNC; + } + + m_flags |= FDS_FILE_APPEND; +} + +void +Storage::window_new(time_t ts) +{ + // Close the current window if exists + window_close(); + + // Open new file + const std::string new_file = filename_gen(ts); + std::unique_ptr new_file_cpy(strdup(new_file.c_str()), &free); + + char *dir2create; + if (!new_file_cpy || (dir2create = dirname(new_file_cpy.get())) == nullptr) { + throw FDS_exception("Failed to generate name of an output directory!"); + } + + if (ipx_utils_mkdir(dir2create, IPX_UTILS_MKDIR_DEF) != FDS_OK) { + throw FDS_exception("Failed to create directory '" + std::string(dir2create) + "'"); + } + + m_file.reset(fds_file_init()); + if (!m_file) { + throw FDS_exception("Failed to create FDS file handler!"); + } + + if (fds_file_open(m_file.get(), new_file.c_str(), m_flags) != FDS_OK) { + std::string err_msg = fds_file_error(m_file.get()); + m_file.reset(); + throw FDS_exception("Failed to create/append file '" + new_file + "': " + err_msg); + } +} + +void +Storage::window_close() +{ + m_file.reset(); + m_session2params.clear(); +} + +void +Storage::process_msg(ipx_msg_ipfix_t *msg) +{ + if (!m_file) { + IPX_CTX_DEBUG(m_ctx, "Ignoring IPFIX Message due to undefined output file!", '\0'); + return; + } + + // Specify a Transport Session context + struct ipx_msg_ctx *msg_ctx = ipx_msg_ipfix_get_ctx(msg); + session_ctx &file_ctx = session_get(msg_ctx->session); + + auto hdr_ptr = reinterpret_cast(ipx_msg_ipfix_get_packet(msg)); + assert(ntohs(hdr_ptr->version) == FDS_IPFIX_VERSION && "Unexpected packet version"); + const uint32_t exp_time = ntohl(hdr_ptr->export_time); + + if (fds_file_write_ctx(m_file.get(), file_ctx.id, msg_ctx->odid, exp_time) != FDS_OK) { + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to configure the writer: " + std::string(err_msg)); + } + + // Get info about the last seen Template snapshot + struct snap_info &snap_last = file_ctx.odid2snap[msg_ctx->odid]; + + // For each Data Record in the file + const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); + for (uint32_t i = 0; i < rec_cnt; ++i) { + ipx_ipfix_record *rec_ptr = ipx_msg_ipfix_get_drec(msg, i); + + // Check if the templates has been changed (detected by change of template snapshots) + if (rec_ptr->rec.snap != snap_last.ptr) { + const char *session_name = msg_ctx->session->ident; + uint32_t session_odid = msg_ctx->odid; + IPX_CTX_DEBUG(m_ctx, "Template snapshot of '%s' [ODID %" PRIu32 "] has been changed. " + "Updating template definitions...", session_name, session_odid); + + tmplts_update(snap_last, rec_ptr->rec.snap); + } + + // Write the Data Record + const uint8_t *rec_data = rec_ptr->rec.data; + uint16_t rec_size = rec_ptr->rec.size; + uint16_t tmplt_id = rec_ptr->rec.tmplt->id; + + if (fds_file_write_rec(m_file.get(), tmplt_id, rec_data, rec_size) != FDS_OK) { + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to add a Data Record: " + std::string(err_msg)); + } + } +} + +/// Auxiliary data structure used in the snapshot iterator +struct tmplt_update_data { + /// Status of template processing + bool is_ok; + /// Plugin context (only for log!) + ipx_ctx_t *ctx; + + /// FDS file with specified context + fds_file_t *file; + /// Set of processed Templates in the snapshot + std::set ids; +}; + +/** + * @brief Callback function for updating definition of an IPFIX (Options) Template + * + * The function checks if the same Template is already defined in the current context of the file. + * If the Template is not present or it's different, the new Template definition is added to the + * file. + * @param[in] tmplt Template to process + * @param[in] data Auxiliary data structure \ref tmplt_update_data + * @return On success returns true. Otherwise returns false. + */ +static bool +tmplt_update_cb(const struct fds_template *tmplt, void *data) +{ + // Template type, raw data and size + enum fds_template_type t_type; + const uint8_t *t_data; + uint16_t t_size; + + auto info = reinterpret_cast(data); + + // No exceptions can be thrown in the C callback! + try { + uint16_t t_id = tmplt->id; + info->ids.emplace(t_id); + + // Get definition of the Template specified in the file + int res = fds_file_write_tmplt_get(info->file, t_id, &t_type, &t_data, &t_size); + + if (res != FDS_OK && res != FDS_ERR_NOTFOUND) { + // Something bad happened + const char *err_msg = fds_file_error(info->file); + throw FDS_exception("fds_file_write_tmplt_get() failed: " + std::string(err_msg)); + } + + // Should we add/redefine the definition of the Template + if (res == FDS_OK + && tmplt->type == t_type + && tmplt->raw.length == t_size + && memcmp(tmplt->raw.data, t_data, t_size) == 0) { + // The same -> nothing to do + return info->is_ok; + } + + // Add the definition (i.e. templates are different or the template hasn't been defined) + IPX_CTX_DEBUG(info->ctx, "Adding/updating definition of Template ID %" PRIu16, t_id); + + t_type = tmplt->type; + t_data = tmplt->raw.data; + t_size = tmplt->raw.length; + + if (fds_file_write_tmplt_add(info->file, t_type, t_data, t_size) != FDS_OK) { + const char *err_msg = fds_file_error(info->file); + throw FDS_exception("fds_file_write_tmplt_add() failed: " + std::string(err_msg)); + } + + } catch (std::exception &ex) { + // Exceptions + IPX_CTX_ERROR(info->ctx, "Failure during update of Template ID %" PRIu16 ": %s", tmplt->id, + ex.what()); + info->is_ok = false; + } catch (...) { + // Other exceptions + IPX_CTX_ERROR(info->ctx, "Unknown exception thrown during template definition update", '\0'); + info->is_ok = false; + } + + return info->is_ok; +} + +/** + * @brief Update Template definitions for the current Transport Session and ODID + * + * The function compares Templates in the \p snap with Template definitions previously defined + * for the currently selected combination of Transport Session and ODID. For each different or + * previously undefined Template, its definition is added or updated. Definitions of Templates + * that were available in the previous snapshot but not available in the new one are removed. + * + * Finally, information (pointer, IDs) in \p info are updated to reflect the performed update. + * @warning + * Template definitions are always unique for a combination of Transport Session and ODID, + * therefore, appropriate file writer context MUST already set using fds_file_writer_ctx(). + * Parameters \p info and \p snap MUST also belong the same unique combination. + * @param[in] info Information about the last update of Templates (old snapshot ref. + list of IDs) + * @param[in] snap New Template snapshot with all valid Template definitions + */ +void +Storage::tmplts_update(struct snap_info &info, const fds_tsnapshot_t *snap) +{ + assert(info.ptr != snap && "Snapshots should be different"); + + // Prepare data for the callback function + struct tmplt_update_data data; + data.is_ok = true; + data.ctx = m_ctx; + data.file = m_file.get(); + data.ids.clear(); + + // Update templates + fds_tsnapshot_for(snap, &tmplt_update_cb, &data); + + // Check if the update failed + if (!data.is_ok) { + throw FDS_exception("Failed to update Template definitions"); + } + + // Check if there are any Template IDs that have been removed + std::set &ids_old = info.tmplt_ids; + std::set &ids_new = data.ids; + std::set ids2remove; + // Old Template IDs - New Templates IDs = Template IDs to remove + std::set_difference(ids_old.begin(), ids_old.end(), ids_new.begin(), ids_new.end(), + std::inserter(ids2remove, ids2remove.begin())); + + // Remove old templates that are not available in the new snapshot + for (uint16_t tid : ids2remove) { + IPX_CTX_DEBUG(m_ctx, "Removing definition of Template ID %" PRIu16, tid); + + int rc = fds_file_write_tmplt_remove(m_file.get(), tid); + if (rc == FDS_OK) { + continue; + } + + // Something bad happened + if (rc != FDS_ERR_NOTFOUND) { + std::string err_msg = fds_file_error(m_file.get()); + throw FDS_exception("fds_file_write_tmplt_remove() failed: " + err_msg); + } + + // Weird, but not critical + IPX_CTX_WARNING(m_ctx, "Failed to remove undefined Template ID %" PRIu16 ". " + "Weird, this should not happen.", tid); + } + + // Update information about the last update of Templates + info.ptr = snap; + std::swap(info.tmplt_ids, data.ids); +} + +/** + * @brief Create a filename based for a user defined timestamp + * @note The timestamp will be expressed in Coordinated Universal Time (UTC) + * + * @param[in] ts Timestamp of the file + * @return New filename + * @throw FDS_exception if formatting functions fail. + */ +std::string +Storage::filename_gen(const time_t &ts) +{ + const char pattern[] = "%Y/%m/%d/flows.%Y%m%d%H%M%S.fds"; + constexpr size_t buffer_size = 64; + char buffer_data[buffer_size]; + + struct tm utc_time; + if (!gmtime_r(&ts, &utc_time)) { + throw FDS_exception("gmtime_r() failed"); + } + + if (strftime(buffer_data, buffer_size, pattern, &utc_time) == 0) { + throw FDS_exception("strftime() failed"); + } + + std::string new_path = m_path; + if (new_path.back() != '/') { + new_path += '/'; + } + + return new_path + buffer_data; +} + +/** + * @brief Convert IPv4 address to IPv4-mapped IPv6 address + * + * New IPv6 address has value corresponding to "::FFFF:\" + * @warning Size of @p in must be at least 4 bytes and @p out at least 16 bytes! + * @param[in] in IPv4 address + * @param[out] out IPv4-mapped IPv6 address + */ +void +Storage::ipv4toipv6(const uint8_t *in, uint8_t *out) +{ + memset(out, 0, 16U); + *(uint16_t *) &out[10] = UINT16_MAX; // 0xFFFF + memcpy(&out[12], in, 4U); // Copy IPv4 address +} + +/** + * @brief Get file identification of a Transport Session + * + * If the identification doesn't exist, the function will add it to the file and create + * a new internal record for it. + * + * @param[in] sptr Transport Session to find + * @return Internal description + * @throw FDS_exception if the function failed to add a new Transport Session + */ +struct Storage::session_ctx & +Storage::session_get(const struct ipx_session *sptr) +{ + auto res_it = m_session2params.find(sptr); + if (res_it != m_session2params.end()) { + // Found + return res_it->second; + } + + // Not found -> register a new session + assert(m_file != nullptr && "File must be opened!"); + + struct fds_file_session new_session; + fds_file_sid_t new_sid; + + session_ipx2fds(sptr, &new_session); + if (fds_file_session_add(m_file.get(), &new_session, &new_sid) != FDS_OK) { + const char *err_msg = fds_file_error(m_file.get()); + throw FDS_exception("Failed to register Transport Session '" + std::string(sptr->ident) + + "': " + err_msg); + } + + // Create a new session + struct session_ctx &ctx = m_session2params[sptr]; + ctx.id = new_sid; + return ctx; +} + +/** + * @brief Convert IPFIXcol representation of a Transport Session to FDS representation + * @param[in] ipx_desc IPFIXcol specific representation + * @param[out] fds_desc FDS specific representation + * @throw FDS_exception if conversion fails due to unsupported Transport Session type + */ +void +Storage::session_ipx2fds(const struct ipx_session *ipx_desc, struct fds_file_session *fds_desc) +{ + // Initialize Transport Session structure + memset(fds_desc, 0, sizeof *fds_desc); + + // Extract protocol type and description + const struct ipx_session_net *net_desc = nullptr; + switch (ipx_desc->type) { + case FDS_SESSION_UDP: + net_desc = &ipx_desc->udp.net; + fds_desc->proto = FDS_FILE_SESSION_UDP; + break; + case FDS_SESSION_TCP: + net_desc = &ipx_desc->tcp.net; + fds_desc->proto = FDS_FILE_SESSION_TCP; + break; + case FDS_SESSION_SCTP: + net_desc = &ipx_desc->sctp.net; + fds_desc->proto = FDS_FILE_SESSION_SCTP; + break; + default: + throw FDS_exception("Not implemented Transport Session type!"); + } + + // Convert ports + fds_desc->port_src = net_desc->port_src; + fds_desc->port_dst = net_desc->port_dst; + + // Convert IP addresses + if (net_desc->l3_proto == AF_INET) { + // IPv4 address + static_assert(sizeof(net_desc->addr_src.ipv4) >= 4U, "Invalid size"); + static_assert(sizeof(net_desc->addr_dst.ipv4) >= 4U, "Invalid size"); + ipv4toipv6(reinterpret_cast(&net_desc->addr_src.ipv4), fds_desc->ip_src); + ipv4toipv6(reinterpret_cast(&net_desc->addr_dst.ipv4), fds_desc->ip_dst); + } else { + // IPv6 address + static_assert(sizeof(fds_desc->ip_src) <= sizeof(net_desc->addr_src.ipv6), "Invalid size"); + static_assert(sizeof(fds_desc->ip_dst) <= sizeof(net_desc->addr_dst.ipv6), "Invalid size"); + memcpy(&fds_desc->ip_src, &net_desc->addr_src.ipv6, sizeof fds_desc->ip_src); + memcpy(&fds_desc->ip_dst, &net_desc->addr_dst.ipv6, sizeof fds_desc->ip_dst); + } +} + diff --git a/src/plugins/output/fds/src/Storage.hpp b/src/plugins/output/fds/src/Storage.hpp new file mode 100644 index 00000000..29b0528e --- /dev/null +++ b/src/plugins/output/fds/src/Storage.hpp @@ -0,0 +1,124 @@ +/** + * \file src/plugins/output/fds/src/Storage.hpp + * \author Lukas Hutak + * \brief FDS file storage (header file) + * \date June 2019 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef IPFIXCOL2_FDS_STORAGE_HPP +#define IPFIXCOL2_FDS_STORAGE_HPP + +#include +#include +#include +#include +#include +#include + +#include "Exception.hpp" +#include "Config.hpp" + +/// Flow storage file +class Storage { +public: + /** + * @brief Create a flow storage file + * + * @note + * Output file for the current window MUST be specified using new_window() function. + * Otherwise, no flow records are stored. + * + * @param[in] ctx Plugin context (only for log) + * @param[in] cfg Configuration + * @throw FDS_exception if @p path directory doesn't exist in the system + */ + Storage(ipx_ctx_t *ctx, const Config &cfg); + virtual ~Storage() = default; + + // Disable copy constructors + Storage(const Storage &other) = delete; + Storage &operator=(const Storage &other) = delete; + + /** + * @brief Create a new time window + * + * @note Previous window is automatically closed, if exists. + * @param[in] ts Timestamp of the window + * @throw FDS_exception if the new window cannot be created + */ + void + window_new(time_t ts); + + /** + * @brief Close the current time window + * @note + * This can be also useful if a fatal error has occurred and we should not add more flow + * records to the file. + * @note + * No more Data Records will be added until a new window is created! + */ + void + window_close(); + + /** + * @brief Process IPFIX message + * + * Process all IPFIX Data Records in the message and store them to the file. + * @note If a time window is not opened, no Data Records are stored and no exception is thrown. + * @param[in] msg Message to process + * @throw FDS_exception if processing fails + */ + void + process_msg(ipx_msg_ipfix_t *msg); + +private: + /// Information about Templates in a snapshot + struct snap_info { + /// Last seen snapshot (might be already freed, do NOT dereference!) + const fds_tsnapshot_t *ptr; + /// Set of Template IDs in the snapshot + std::set tmplt_ids; + + snap_info() { + ptr = nullptr; + tmplt_ids.clear(); + } + }; + + /// Description parameters of a Transport Session + struct session_ctx { + /// Session ID used in the FDS file + fds_file_sid_t id; + /// Last seen snapshot for a specific ODID of the Transport Session + std::map odid2snap; + }; + + /// Plugin context only for logging! + ipx_ctx_t *m_ctx; + /// Storage path + std::string m_path; + /// Flags for opening file + uint32_t m_flags; + + /// Output FDS file + std::unique_ptr m_file = {nullptr, &fds_file_close}; + /// Mapping of Transport Sessions to FDS specific parameters + std::map m_session2params; + + std::string + filename_gen(const time_t &ts); + static void + ipv4toipv6(const uint8_t *in, uint8_t *out); + struct session_ctx & + session_get(const struct ipx_session *sptr); + void + session_ipx2fds(const struct ipx_session *ipx_desc, struct fds_file_session *fds_desc); + void + tmplts_update(struct snap_info &info, const fds_tsnapshot_t *snap); +}; + + +#endif // IPFIXCOL2_FDS_STORAGE_HPP diff --git a/src/plugins/output/fds/src/fds.cpp b/src/plugins/output/fds/src/fds.cpp new file mode 100644 index 00000000..292e17ab --- /dev/null +++ b/src/plugins/output/fds/src/fds.cpp @@ -0,0 +1,135 @@ +/** + * \file src/plugins/output/dummy/dummy.c + * \author Lukas Hutak + * \brief Example output plugin for IPFIXcol 2 + * \date 2018 + * + * Copyright(c) 2019 CESNET z.s.p.o. + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include +#include +#include +#include +#include + +#include "Config.hpp" +#include "Storage.hpp" + +/// Plugin description +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin identification name + "fds", + // Brief description of plugin + "Flow Data Storage output plugin", + // Plugin type + IPX_PT_OUTPUT, + // Configuration flags (reserved for future use) + 0, + // Plugin version string (like "1.2.3") + "2.0.0", + // Minimal IPFIXcol version string (like "1.2.3") + "2.1.0" +}; + +/// Instance +struct Instance { + /// Parsed configuration + std::unique_ptr config_ptr = nullptr; + /// Storage file + std::unique_ptr storage_ptr = nullptr; + /// Start of the current window + time_t window_start = 0; +}; + +static void +window_check(struct Instance &inst) +{ + const Config &cfg = *inst.config_ptr; + + // Decide whether close file and create a new time window + time_t now = time(NULL); + if (difftime(now, inst.window_start) < cfg.m_window.size) { + // Nothing to do + return; + } + + if (cfg.m_window.align) { + const uint32_t window_size = cfg.m_window.size; + now /= window_size; + now *= window_size; + } + + inst.window_start = now; + inst.storage_ptr->window_new(now); +} + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *params) +{ + try { + // Parse configuration, try to create a storage and time window + std::unique_ptr instance(new Instance); + instance->config_ptr.reset(new Config(params)); + instance->storage_ptr.reset(new Storage(ctx, *instance->config_ptr)); + window_check(*instance); + // Everything seems OK + ipx_ctx_private_set(ctx, instance.release()); + } catch (const FDS_exception &ex) { + IPX_CTX_ERROR(ctx, "Initialization failed: %s", ex.what()); + return IPX_ERR_DENIED; + } catch (...) { + IPX_CTX_ERROR(ctx, "Unknown error has occurred!", '\0'); + return IPX_ERR_DENIED; + } + + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *cfg) +{ + (void) ctx; // Suppress warnings + + try { + auto inst = reinterpret_cast(cfg); + inst->storage_ptr.reset(); + inst->config_ptr.reset(); + delete inst; + } catch (...) { + IPX_CTX_ERROR(ctx, "Something bad happened during plugin destruction"); + } +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) +{ + auto *inst = reinterpret_cast(cfg); + bool failed = false; + + try { + // Check if the current time window should be closed + window_check(*inst); + ipx_msg_ipfix_t *msg_ipfix = ipx_msg_base2ipfix(msg); + inst->storage_ptr->process_msg(msg_ipfix); + } catch (const FDS_exception &ex) { + IPX_CTX_ERROR(ctx, "%s", ex.what()); + failed = true; + } catch (std::exception &ex) { + IPX_CTX_ERROR(ctx, "Unexpected error has occurred: %s", ex.what()); + failed = true; + } catch (...) { + IPX_CTX_ERROR(ctx, "Unknown error has occurred!"); + failed = true; + } + + if (failed) { + IPX_CTX_ERROR(ctx, "Due to the previous error(s), the output file is possibly corrupted. " + "Therefore, no flow records are stored until a new file is automatically opened " + "after current window expiration."); + inst->storage_ptr->window_close(); + } + + return IPX_OK; +} diff --git a/src/plugins/output/json/CMakeLists.txt b/src/plugins/output/json/CMakeLists.txt index 74175acc..1dd569c7 100644 --- a/src/plugins/output/json/CMakeLists.txt +++ b/src/plugins/output/json/CMakeLists.txt @@ -19,7 +19,7 @@ add_library(json-output MODULE install( TARGETS json-output - LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" ) if (ENABLE_DOC_MANPAGE) @@ -35,6 +35,6 @@ if (ENABLE_DOC_MANPAGE) install( FILES "${DST_FILE}" - DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + DESTINATION "${INSTALL_DIR_MAN}/man7" ) -endif() \ No newline at end of file +endif() diff --git a/src/plugins/output/timecheck/CMakeLists.txt b/src/plugins/output/timecheck/CMakeLists.txt new file mode 100644 index 00000000..1fd46121 --- /dev/null +++ b/src/plugins/output/timecheck/CMakeLists.txt @@ -0,0 +1,28 @@ +# Create a linkable module +add_library(timecheck-output MODULE + src/config.c + src/config.h + src/timecheck.c +) + +install( + TARGETS timecheck-output + LIBRARY DESTINATION "${INSTALL_DIR_LIB}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-timecheck-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-timecheck-output.7") + + add_custom_command(TARGET timecheck-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${INSTALL_DIR_MAN}/man7" + ) +endif() diff --git a/src/plugins/output/timecheck/README.rst b/src/plugins/output/timecheck/README.rst new file mode 100644 index 00000000..65f5a78f --- /dev/null +++ b/src/plugins/output/timecheck/README.rst @@ -0,0 +1,51 @@ +Time Check (output plugin) +========================== + +The plugin checks that start and end timestamps of each flow record are relatively recent. +Based on configured parameters it reports flows with timestamps from the future (i.e. greater +than the current system time) and timestamps from the distant past. + +Timestamp anomalies are usually caused by missing clock synchronization (e.g. NTP) or invalid +implementation on side of the exporter. If the anomaly is detected, the plugin will print +details on the standard output. + +Only following standard IANA timestamps are supported: + +- ID 150 (flowStartSeconds) +- ID 151 (flowEndSeconds) +- ID 152 (flowStartMilliseconds) +- ID 153 (flowEndMilliseconds) +- ID 154 (flowStartMicroseconds) +- ID 155 (flowEndMicroseconds) +- ID 156 (flowStartNanoseconds) +- ID 157 (flowEndNanoseconds) + +Example configuration +--------------------- + +.. code-block:: xml + + + TimeCheck output + timecheck + + 600 + 0 + + + +Parameters +---------- + +:``devPast``: + Maximum allowed deviation between the current system time and timestamps from the past in + seconds. The value must be greater than active and inactive timeouts of exporters and must also + include a possible delay between expiration and processing on the collector. + For example, let's say that active timeout and inactive timeout are 5 minutes and 30 seconds, + respectively. Value 600 (i.e. 10 minutes) should be always enough for all flow data to be + received and processed at the collector. + +:``devFuture``: + Maximum allowed deviation between the current time and timestamps from the future in seconds. + The collector should never receive flows with timestamp from the future, therefore, the value + should be usually set to 0. diff --git a/src/plugins/output/timecheck/doc/ipfixcol2-timecheck-output.7.rst b/src/plugins/output/timecheck/doc/ipfixcol2-timecheck-output.7.rst new file mode 100644 index 00000000..a7b12b88 --- /dev/null +++ b/src/plugins/output/timecheck/doc/ipfixcol2-timecheck-output.7.rst @@ -0,0 +1,20 @@ +=========================== + ipfixcol2-timecheck-output +=========================== + +-------------------------- +Time Check (output plugin) +-------------------------- + +:Author: Lukáš Huták (lukas.hutak@cesnet.cz) +:Date: 2019-03-01 +:Copyright: Copyright © 2019 CESNET, z.s.p.o. +:Version: 2.0 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 diff --git a/src/plugins/output/timecheck/src/config.c b/src/plugins/output/timecheck/src/config.c new file mode 100644 index 00000000..a0701d13 --- /dev/null +++ b/src/plugins/output/timecheck/src/config.c @@ -0,0 +1,188 @@ +/** + * \file src/plugins/output/timecheck/src/config.c + * \author Lukas Hutak + * \brief Parser of an XML configuration (source file) + * \date 2019 + */ + +/* Copyright (C) 2019 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include "config.h" + +/* + * + * ... + * + */ + +/** XML nodes */ +enum params_xml_nodes { + DEV_PAST = 1, + DEV_FUTURE +}; + +/** Definition of the \ node */ +static const struct fds_xml_args args_params[] = { + FDS_OPTS_ROOT("params"), + FDS_OPTS_ELEM(DEV_PAST, "devPast", FDS_OPTS_T_UINT, 0), + FDS_OPTS_ELEM(DEV_FUTURE, "devFuture", FDS_OPTS_T_UINT, FDS_OPTS_P_OPT), + FDS_OPTS_END +}; + +/** + * \brief Process \ node + * \param[in] ctx Plugin context + * \param[in] root XML context to process + * \param[in] cfg Parsed configuration + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT in case of failure + */ +static int +config_parser_root(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct instance_config *cfg) +{ + (void) ctx; + + const struct fds_xml_cont *content; + while (fds_xml_next(root, &content) != FDS_EOC) { + switch (content->id) { + case DEV_PAST: + assert(content->type == FDS_OPTS_T_UINT); + cfg->dev_past = content->val_uint; + break; + case DEV_FUTURE: + assert(content->type == FDS_OPTS_T_UINT); + cfg->dev_future = content->val_uint; + break; + default: + // Internal error + assert(false); + } + } + + return IPX_OK; +} + +/** + * \brief Set default parameters of the configuration + * \param[in] cfg Configuration + */ +static void +config_default_set(struct instance_config *cfg) +{ + cfg->dev_past = 0; + cfg->dev_future = 0; +} + +/** + * \brief Validate the parsed configuration + * \param[in] ctx Plugin context + * \param[in] cfg Configuration + * \return #IPX_OK or #IPX_ERR_FORMAT + */ +static int +config_validate(ipx_ctx_t *ctx, const struct instance_config *cfg) +{ + if (cfg->dev_past == 0) { + IPX_CTX_ERROR(ctx, "Maximum allowed deviation from current time and the timestamps " + "from the past cannot be zero!", '\0'); + return IPX_ERR_FORMAT; + } + + if (cfg->dev_past < 300) { + IPX_CTX_WARNING(ctx, "The configuration might cause many false warnings!", '\0'); + } + + return IPX_OK; +} + + +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params) +{ + struct instance_config *cfg = calloc(1, sizeof(*cfg)); + if (!cfg) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + return NULL; + } + + // Set default parameters + config_default_set(cfg); + + // Create an XML parser + fds_xml_t *parser = fds_xml_create(); + if (!parser) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + free(cfg); + return NULL; + } + + if (fds_xml_set_args(parser, args_params) != FDS_OK) { + IPX_CTX_ERROR(ctx, "Failed to parse the description of an XML document!", '\0'); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + fds_xml_ctx_t *params_ctx = fds_xml_parse_mem(parser, params, true); + if (params_ctx == NULL) { + IPX_CTX_ERROR(ctx, "Failed to parse the configuration: %s", fds_xml_last_err(parser)); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + // Parse parameters + int rc = config_parser_root(ctx, params_ctx, cfg); + fds_xml_destroy(parser); + if (rc != IPX_OK) { + free(cfg); + return NULL; + } + + if (config_validate(ctx, cfg) != IPX_OK) { + free(cfg); + return NULL; + } + + return cfg; +} + +void +config_destroy(struct instance_config *cfg) +{ + free(cfg); +} diff --git a/src/plugins/output/timecheck/src/config.h b/src/plugins/output/timecheck/src/config.h new file mode 100644 index 00000000..29f86b45 --- /dev/null +++ b/src/plugins/output/timecheck/src/config.h @@ -0,0 +1,73 @@ +/** + * \file src/plugins/output/timecheck/src/config.h + * \author Lukas Hutak + * \brief Parser of an XML configuration (header file) + * \date 2019 + */ + +/* Copyright (C) 2019 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef CONFIG_H +#define CONFIG_H + +#include +#include "stdint.h" + +/** Configuration of a instance of the plugin */ +struct instance_config { + /** Maximum allowed deviation between the current time and timestamps from the past (sec) */ + uint64_t dev_past; + /** Maximum allowed deviation between the current time and timestamps from the future (sec) */ + uint64_t dev_future; +}; + +/** + * \brief Parse configuration of the plugin + * \param[in] ctx Instance context + * \param[in] params XML parameters + * \return Pointer to the parse configuration of the instance on success + * \return NULL if arguments are not valid or if a memory allocation error has occurred + */ +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params); + +/** + * \brief Destroy parsed configuration + * \param[in] cfg Parsed configuration + */ +void +config_destroy(struct instance_config *cfg); + +#endif // CONFIG_H diff --git a/src/plugins/output/timecheck/src/timecheck.c b/src/plugins/output/timecheck/src/timecheck.c new file mode 100644 index 00000000..84604701 --- /dev/null +++ b/src/plugins/output/timecheck/src/timecheck.c @@ -0,0 +1,242 @@ +/** + * \file src/plugins/output/timecheck/src/timecheck.c + * \author Lukas Hutak + * \brief Time checker plugin for IPFIXcol 2 + * \date 2019 + */ + +/* Copyright (C) 2019 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include +#include +#include + +#include "config.h" +#include "../../../../core/message_ipfix.h" + +/** Private Enterprise Number of standard IEs from IANA */ +#define PEN_IANA 0 +/** Private Enterprise Number of Standard reverse IEs from IANA */ +#define PEN_IANA_REV 29305 + +/** Plugin description */ +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin type + .type = IPX_PT_OUTPUT, + // Plugin identification name + .name = "timecheck", + // Brief description of plugin + .dsc = "The plugin checks that timestamp elements in flows are relatively recent.", + // Configuration flags (reserved for future use) + .flags = 0, + // Plugin version string (like "1.2.3") + .version = "2.0.0", + // Minimal IPFIXcol version string (like "1.2.3") + .ipx_min = "2.0.0" +}; + +/** Instance */ +struct instance_data { + /** Parsed configuration of the instance */ + struct instance_config *config; + /** Current time (seconds since the Epoch) */ + uint64_t ts_now; + + /** Context reference (only for log!) */ + ipx_ctx_t *ctx; +}; + +// Function prototype +static void +timestamp_check(const struct instance_data *inst, ipx_msg_ipfix_t *msg, + const struct fds_drec_field *field); + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *params) +{ + // Create a private data + struct instance_data *data = calloc(1, sizeof(*data)); + if (!data) { + return IPX_ERR_DENIED; + } + + if ((data->config = config_parse(ctx, params)) == NULL) { + free(data); + return IPX_ERR_DENIED; + } + + data->ctx = ctx; + ipx_ctx_private_set(ctx, data); + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *cfg) +{ + (void) ctx; // Suppress warnings + + struct instance_data *data = (struct instance_data *) cfg; + config_destroy(data->config); + free(data); +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) +{ + (void) ctx; // Suppress warnings + struct instance_data *data = (struct instance_data *) cfg; + ipx_msg_ipfix_t *ipfix_msg = ipx_msg_base2ipfix(msg); + + // Update the current UTC time + data->ts_now = (uint64_t) time(NULL); + + // For each Data Record in the message + uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(ipfix_msg); + for (uint32_t i = 0; i < rec_cnt; ++i) { + // For each field in the Data Record + struct ipx_ipfix_record *rec = ipx_msg_ipfix_get_drec(ipfix_msg, i); + struct fds_drec_iter it; + + fds_drec_iter_init(&it, &rec->rec, 0); + while (fds_drec_iter_next(&it) != FDS_EOC) { + // Is it a timestamp? + const uint16_t field_id = it.field.info->id; + const uint32_t field_en = it.field.info->en; + + if (field_en != PEN_IANA && field_en != PEN_IANA_REV) { + // We don't check non-standard fields + continue; + } + + if (field_id < 150U || field_id > 157U) { + // We want to check only IE elements within the range 150 - 157 + continue; + } + + // Check if the value doesn't violate rules + timestamp_check(data, ipfix_msg, &it.field); + } + } + + return IPX_OK; +} + +/** + * \brief Timestamp check function + * + * Only IANA fields flowStart... and flowEnd... are supported! + * \param[in] inst Plugin instance parameters + * \param[in] msg IPFIX Message from which the timestamp comes from + * \param[in] field The timestamp to check + */ +void +timestamp_check(const struct instance_data *inst, ipx_msg_ipfix_t *msg, + const struct fds_drec_field *field) +{ + assert((field->info->en == PEN_IANA || field->info->en == PEN_IANA_REV) && "Non-IANA PEN!"); + enum fds_iemgr_element_type elem_type; + + switch (field->info->id) { + case 150U: // flowStartSeconds + case 151U: // flowEndSeconds + elem_type = FDS_ET_DATE_TIME_SECONDS; + break; + case 152U: // flowStartMilliseconds + case 153U: // flowEndMilliseconds + elem_type = FDS_ET_DATE_TIME_MILLISECONDS; + break; + case 154U: // flowStartMicroseconds + case 155U: // flowEndMicroseconds + elem_type = FDS_ET_DATE_TIME_MICROSECONDS; + break; + case 156U: // flowStartNanoseconds + case 157U: // flowEndNanoseconds + elem_type = FDS_ET_DATE_TIME_NANOSECONDS; + break; + default: + assert(false && "Unhandled switch option!"); + return; + } + + // Get the value + uint64_t ts_value; + int ret_code = fds_get_datetime_lp_be(field->data, field->size, elem_type, &ts_value); + if (ret_code != FDS_OK) { + IPX_CTX_WARNING(inst->ctx, "Timestamp conversion failed! Skipping...", '\0'); + return; + } + ts_value /= 1000U; // Convert from milliseconds to seconds (since the Epoch) + + // Check the deviation + const char *violation_type; + uint64_t ts_diff; + + if (ts_value <= inst->ts_now) { + // Timestamps is from the past + ts_diff = inst->ts_now - ts_value; + if (ts_diff <= inst->config->dev_past) { + return; + } + violation_type = "past"; + } else { + // Timestamp is from the future + ts_diff = ts_value - inst->ts_now; + if (ts_diff <= inst->config->dev_future) { + return; + } + violation_type = "future"; + } + + // Report the violation of rules + const struct ipx_msg_ctx *msg_ctx = ipx_msg_ipfix_get_ctx(msg); + const char *s_name = msg_ctx->session->ident; + const uint32_t s_odid = msg->ctx.odid; + + uint16_t diff_secs = ts_diff % 60U; + ts_diff /= 60U; + uint16_t diff_mins = ts_diff % 60U; + ts_diff /= 60U; + uint16_t diff_hrs = ts_diff % 24U; + ts_diff /= 24U; + + printf("%s [ODID: %"PRIu32"]: Timestamp (EN: %"PRIu32", ID: %"PRIu16") is " + "%"PRIu64" days, %"PRIu16" hours, %"PRIu16" minutes and %"PRIu16" seconds in the %s " + "(now: %"PRIu64", TS value: %"PRIu64" [seconds since the Epoch])\n", + s_name, s_odid, field->info->en, field->info->id, + ts_diff, diff_hrs, diff_mins, diff_secs, violation_type, + inst->ts_now, ts_value); +} \ No newline at end of file diff --git a/src/plugins/output/viewer/CMakeLists.txt b/src/plugins/output/viewer/CMakeLists.txt new file mode 100644 index 00000000..5e0341f4 --- /dev/null +++ b/src/plugins/output/viewer/CMakeLists.txt @@ -0,0 +1,31 @@ +# Create a linkable module + +add_library(viewer-output MODULE + viewer.c + config.c + config.h + Reader.h + Reader.c +) + +install( + TARGETS viewer-output + LIBRARY DESTINATION "${CMAKE_INSTALL_FULL_LIBDIR}/ipfixcol2/" +) + +if (ENABLE_DOC_MANPAGE) + # Build a manual page + set(SRC_FILE "${CMAKE_CURRENT_SOURCE_DIR}/doc/ipfixcol2-viewer-output.7.rst") + set(DST_FILE "${CMAKE_CURRENT_BINARY_DIR}/ipfixcol2-viewer-output.7") + + add_custom_command(TARGET viewer-output PRE_BUILD + COMMAND ${RST2MAN_EXECUTABLE} --syntax-highlight=none ${SRC_FILE} ${DST_FILE} + DEPENDS ${SRC_FILE} + VERBATIM + ) + + install( + FILES "${DST_FILE}" + DESTINATION "${CMAKE_INSTALL_FULL_MANDIR}/man7" + ) +endif() \ No newline at end of file diff --git a/src/plugins/output/viewer/README.rst b/src/plugins/output/viewer/README.rst new file mode 100644 index 00000000..8a43c55d --- /dev/null +++ b/src/plugins/output/viewer/README.rst @@ -0,0 +1,96 @@ +Viewer (output plugin) +===================== + +The plugin converts IPFIX Messages into plain text and prints them on standard output. + +The main goal of the plugin is to show content of received IPFIX Messages in human readable form. +Each IPFIX Message is broken down into IPFIX Sets and each IPFIX Set is further broken down into +(Options) Template/Data records and so on. Fields of the Data records are formatted according +to the expected data type, if their corresponding Information Element definitions are known to +the collector. Therefore, the output can be also used to determine missing Information Element +definitions. + +Biflow records and structured data types are also supported and appropriately formatted. +Output is not supposed to be further parsed because its format can be changed in the future. +However, if you are interested into processing Data Records, consider using other +plugins such as JSON, UniRec, etc. + +Example configuration +--------------------- + +.. code-block:: xml + + + Viewer output + viewer + + + +Parameters +---------- + +Parameters are not supported by the plugin. + +Example output +-------------- + +Below you can see example output for anonymized IPFIX Messages. The message structure and +available fields of Data Record vary from exporter to exporter. + +.. code-block:: + + IPFIX Message header: + Version: 10 + Length: 176 + Export time: 1455534348 + Sequence no.: 0 + ODID: 1 + + Set Header: + Set ID: 2 (Template Set) + Length: 160 + Template Record (#1) + Template ID: 256 + Field Count: 27 + EN: 0 ID: 1 Size: 4 | iana:octetDeltaCount + EN: 0 ID: 2 Size: 4 | iana:packetDeltaCount + EN: 0 ID: 152 Size: 8 | iana:flowStartMilliseconds + EN: 0 ID: 153 Size: 8 | iana:flowEndMilliseconds + EN: 0 ID: 10 Size: 2 | iana:ingressInterface + EN: 0 ID: 60 Size: 1 | iana:ipVersion + EN: 0 ID: 8 Size: 4 | iana:sourceIPv4Address + EN: 0 ID: 12 Size: 4 | iana:destinationIPv4Address + EN: 0 ID: 5 Size: 1 | iana:ipClassOfService + EN: 0 ID: 192 Size: 1 | iana:ipTTL + EN: 0 ID: 4 Size: 1 | iana:protocolIdentifier + EN: 0 ID: 7 Size: 2 | iana:sourceTransportPort + EN: 0 ID: 11 Size: 2 | iana:destinationTransportPort + ... <~ output shortened for clarity ~> ... + + ------------------------------------------------------------------- + IPFIX Message header: + Version: 10 + Length: 216 + Export time: 1455534348 + Sequence no.: 0 + ODID: 1 + + Set Header: + Set ID: 256 (Data Set) + Length: 200 + Template ID: 256 + Data Record (#1) [Length: 95]: + EN: 0 ID: 1 iana:octetDeltaCount : 76 octets + EN: 0 ID: 2 iana:packetDeltaCount : 1 packets + EN: 0 ID: 152 iana:flowStartMilliseconds : 2016-02-15T11:05:48.150Z + EN: 0 ID: 153 iana:flowEndMilliseconds : 2016-02-15T11:05:48.150Z + EN: 0 ID: 10 iana:ingressInterface : 1 + EN: 0 ID: 60 iana:ipVersion : 4 + EN: 0 ID: 8 iana:sourceIPv4Address : 209.246.218.165 + EN: 0 ID: 12 iana:destinationIPv4Address : 93.113.168.59 + EN: 0 ID: 5 iana:ipClassOfService : 0 + EN: 0 ID: 192 iana:ipTTL : 122 hops + EN: 0 ID: 4 iana:protocolIdentifier : 17 + EN: 0 ID: 7 iana:sourceTransportPort : 62299 + EN: 0 ID: 11 iana:destinationTransportPort : 53 + ... <~ output shortened for clarity ~> ... diff --git a/src/plugins/output/viewer/Reader.c b/src/plugins/output/viewer/Reader.c new file mode 100644 index 00000000..0310b2eb --- /dev/null +++ b/src/plugins/output/viewer/Reader.c @@ -0,0 +1,554 @@ +/** + * \file src/plugins/output/viewer/Reader.c + * \author Jan Kala + * \author Lukas Hutak + * \brief Viewer - output module for printing information about incoming packets on stdout (source file) + * \date 2018 + */ +/* Copyright (C) 2018 CESNET, z.s.p.o. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided with the +* distribution. +* 3. Neither the name of the Company nor the names of its contributors +* may be used to endorse or promote products derived from this +* software without specific prior written permission. +* +* ALTERNATIVELY, provided that this notice is retained in full, this +* product may be distributed under the terms of the GNU General Public +* License (GPL) version 2 or later, in which case the provisions +* of the GPL apply INSTEAD OF those given above. +* +* This software is provided ``as is'', and any express or implied +* warranties, including, but not limited to, the implied warranties of +* merchantability and fitness for a particular purpose are disclaimed. +* In no event shall the company or contributors be liable for any +* direct, indirect, incidental, special, exemplary, or consequential +* damages (including, but not limited to, procurement of substitute +* goods or services; loss of use, data, or profits; or business +* interruption) however caused and on any theory of liability, whether +* in contract, strict liability, or tort (including negligence or +* otherwise) arising in any way out of the use of this software, even +* if advised of the possibility of such damage. +* +*/ + +#include +#include +#include "Reader.h" +#include + +void +read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr) +{ + const struct fds_ipfix_msg_hdr *ipfix_msg_hdr; + ipfix_msg_hdr = (const struct fds_ipfix_msg_hdr*)ipx_msg_ipfix_get_packet(msg); + + if (ipfix_msg_hdr->length < FDS_IPFIX_MSG_HDR_LEN){ + return; + } + + // Print packet header + printf("--------------------------------------------------------------------------------\n"); + printf("IPFIX Message header:\n"); + printf("\tVersion: %"PRIu16"\n",ntohs(ipfix_msg_hdr->version)); + printf("\tLength: %"PRIu16"\n",ntohs(ipfix_msg_hdr->length)); + printf("\tExport time: %"PRIu32"\n",ntohl(ipfix_msg_hdr->export_time)); + printf("\tSequence no.: %"PRIu32"\n",ntohl(ipfix_msg_hdr->seq_num)); + printf("\tODID: %"PRIu32"\n", ntohl(ipfix_msg_hdr->odid)); + + // Get number of sets + struct ipx_ipfix_set *sets; + size_t set_cnt; + ipx_msg_ipfix_get_sets(msg, &sets, &set_cnt); + + // Record counter of total records in IPFIX message + uint32_t rec_i = 0; + + // Iteration through all the sets + for (uint32_t i = 0; i < set_cnt; ++i){ + read_set(&sets[i], msg, iemgr, &rec_i); + } +} + +void +read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr, uint32_t *rec_i) +{ + uint8_t *set_end = (uint8_t *)set->ptr + ntohs(set->ptr->length); + uint16_t set_id = ntohs(set->ptr->flowset_id); + + const uint32_t rec_cnt = ipx_msg_ipfix_get_drec_cnt(msg); + const char *set_type = ""; + if (set_id == FDS_IPFIX_SET_TMPLT) { + set_type = "Template Set"; + } else if (set_id == FDS_IPFIX_SET_OPTS_TMPLT) { + set_type = "Options Template Set"; + } else if (set_id >= FDS_IPFIX_SET_MIN_DSET) { + set_type = "Data Set"; + } + + printf("\n"); + printf("Set Header:\n"); + printf("\tSet ID: %"PRIu16" (%s)\n", set_id, set_type); + printf("\tLength: %"PRIu16"\n", ntohs(set->ptr->length)); + + if (set_id == FDS_IPFIX_SET_TMPLT || set_id == FDS_IPFIX_SET_OPTS_TMPLT) { + // Template set + struct fds_tset_iter tset_iter; + fds_tset_iter_init(&tset_iter, set->ptr); + const char *rec_fmt = (set_id == FDS_IPFIX_SET_TMPLT) + ? "- Template Record (#%u)\n" + : "- Options Template Record (#%u)\n"; + unsigned int rec_cnt = 0; + + // Iteration through all templates in the set + while (fds_tset_iter_next(&tset_iter) == FDS_OK){ + // Read and print single template + printf(rec_fmt, ++rec_cnt); + read_template_set(&tset_iter, set_id,iemgr); + putchar('\n'); + } + return; + } + + if (set_id >= FDS_IPFIX_SET_MIN_DSET) { + // Data set + struct ipx_ipfix_record *ipfix_rec = ipx_msg_ipfix_get_drec(msg, *rec_i); + if (ipfix_rec == NULL) return; + + // All the records in the set has same template id, so we extract it from the first record and print it + printf("\tTemplate ID: %"PRIu16"\n", ipfix_rec->rec.tmplt->id); + unsigned int iter_cnt = 0; + + // Iteration through the records which belongs to the current set + while ((ipfix_rec != NULL) && (ipfix_rec->rec.data < set_end) && (*rec_i < rec_cnt)) { + // Print record header + printf("- Data Record (#%u) [Length: %"PRIu16"]:\n", ++iter_cnt, ipfix_rec->rec.size); + // Get the specific record and read all the fields + read_record(&ipfix_rec->rec, 1, iemgr); + putchar('\n'); + + // Get the next record + (*rec_i)++; + ipfix_rec = ipx_msg_ipfix_get_drec(msg, *rec_i); + } + return; + } + + //Unknown set ID + printf("\t\n"); +} + +void +read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const fds_iemgr_t *iemgr) +{ + enum fds_template_type type; + void *ptr; + switch (set_id){ + case FDS_IPFIX_SET_TMPLT: + type = FDS_TYPE_TEMPLATE; + ptr = tset_iter->ptr.trec; + break; + case FDS_IPFIX_SET_OPTS_TMPLT: + type = FDS_TYPE_TEMPLATE_OPTS; + ptr = tset_iter->ptr.opts_trec; + break; + default: + printf("\t\n"); + return; + } + // Filling the template structure with data from raw packet + uint16_t tmplt_size = tset_iter->size; + struct fds_template *tmplt; + if (fds_template_parse(type, ptr, &tmplt_size, &tmplt) != FDS_OK){ + printf("*Template parsing error*\n"); + return; + } + + // Printing out the header + printf("\tTemplate ID: %"PRIu16"\n", tmplt->id); + printf("\tField Count: %"PRIu16"\n", tmplt->fields_cnt_total); + if (type == FDS_TYPE_TEMPLATE_OPTS) { + printf("\tScope Field Count: %"PRIu16"\n", tmplt->fields_cnt_scope); + } + + // Using IEManager to fill the definitions of the fields in the template + if(fds_template_ies_define(tmplt, iemgr , false) != FDS_OK){ + printf("*Error while assigning element definitions in template*\n"); + fds_template_destroy(tmplt); + return; + } + + // Iteration through the fields and printing them out + for (uint16_t i = 0; i < tmplt->fields_cnt_total ; ++i) { + struct fds_tfield current = tmplt->fields[i]; + printf("\t"); + printf("EN: %-*"PRIu32" ", WRITER_EN_SPACE, current.en); + printf("ID: %-*"PRIu16" ", WRITER_ID_SPACE, current.id); + printf("Size: "); + // In case of variable length print keyword "var" + current.length == FDS_IPFIX_VAR_IE_LEN + ? printf("%-*s ", WRITER_SIZE_SPACE, "var.") + : printf("%-*"PRIu16" ", WRITER_SIZE_SPACE, current.length); + + const char *pen_name = ""; + const char *field_name = ""; + if (current.def != NULL) { + // Known definition + pen_name = current.def->scope->name; + field_name = current.def->name; + } else { + // Field is unknown ... try to find at least vendor + const struct fds_iemgr_scope *scope_ptr = fds_iemgr_scope_find_pen(iemgr, current.en); + if (scope_ptr != NULL) { + pen_name = scope_ptr->name; + } + } + + printf("| %*s:%s", WRITER_ORG_NAME_SPACE, pen_name, field_name); + if ((current.flags & FDS_TFIELD_SCOPE) != 0) { + printf(" (scope)"); + } + putchar('\n'); + } + fds_template_destroy(tmplt); +} + +void +print_indent(unsigned int n) +{ + for (unsigned int i = 0; i < n; i++){ + putchar('\t'); + } +} + +void +read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr) +{ + // Iterate through all the fields in record + struct fds_drec_iter iter; + fds_drec_iter_init(&iter, rec, FDS_DREC_PADDING_SHOW); + + while (fds_drec_iter_next(&iter) != FDS_EOC) { + struct fds_drec_field field = iter.field; + read_field(&field, indent, iemgr, rec->snap); + } +} + +const char * +fds_semantic2str(enum fds_ipfix_list_semantics semantic) +{ + switch (semantic){ + case (FDS_IPFIX_LIST_ALL_OF): + return "All of"; + case (FDS_IPFIX_LIST_EXACTLY_ONE_OF): + return "Exactly one of"; + case (FDS_IPFIX_LIST_ORDERED): + return "Ordered"; + case (FDS_IPFIX_LIST_NONE_OF): + return "None of"; + case (FDS_IPFIX_LIST_ONE_OR_MORE_OF): + return "One or more of"; + default: + return "Undefined"; + } +} + +void +read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ + // Write info from header about field + print_indent(indent); + printf("EN: %-*"PRIu32" ID: %-*"PRIu16" ", WRITER_EN_SPACE, field->info->en, + WRITER_ID_SPACE, field->info->id); + + enum fds_iemgr_element_type type; + const char *org; + const char *field_name; + const char *unit; + + // Get the organisation name, field name and unit of the data. + if (field->info->def == NULL) { + type = FDS_ET_OCTET_ARRAY; + field_name = ""; + org = ""; + unit = ""; + + // Try to find the scope + const struct fds_iemgr_scope *scope_ptr = fds_iemgr_scope_find_pen(iemgr, field->info->en); + if (scope_ptr != NULL) { + org = scope_ptr->name; + } + } else { + type = field->info->def->data_type; + org = field->info->def->scope->name; + field_name = field->info->def->name; + if (field->info->def->data_unit != FDS_EU_NONE) { + unit = fds_iemgr_unit2str(field->info->def->data_unit); + } else { + unit = ""; + } + } + + if (fds_iemgr_is_type_list(type)) { + // Process lists + printf("%*s:%s", WRITER_ORG_NAME_SPACE, org, field_name); + switch (type) { + case FDS_ET_BASIC_LIST: + // Note: header description will be complete in the function + read_list_basic(field, indent, iemgr, snap); + break; + case FDS_ET_SUB_TEMPLATE_LIST: + printf(" (subTemplateList, see below)\n"); + read_list_stl(field, indent, iemgr, snap); + break; + case FDS_ET_SUB_TEMPLATE_MULTILIST: + printf(" (subTemplateMultiList, see below)\n"); + read_list_stml(field, indent, iemgr, snap); + break; + default: + printf("*Unsupported list type*\n"); + break; + } + + return; + } + + printf("%*s:%-*s : ", WRITER_ORG_NAME_SPACE, org, WRITER_FIELD_NAME_SPACE, field_name); + // Read and write the data from the field + char buffer[1024]; + int res = fds_field2str_be(field->data, field->size, type, buffer, sizeof(buffer)); + + if(res >= 0){ + // Conversion was successful + if (type == FDS_ET_STRING) { + printf("\"%s\"", buffer); + } else if (type == FDS_ET_OCTET_ARRAY) { + printf("0x%s",buffer); + } else { + printf("%s", buffer); + } + + if (*unit != 0) { + printf(" %s", unit); + } + putchar('\n'); + + return; + } + else if (res == FDS_ERR_BUFFER) { + // Buffer too small + printf("\n"); + return; + } + else { + // Any other error + printf("*Invalid value*\n"); + return; + } +} + +void +read_list_basic(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ + printf(" (basicList"); + + struct fds_blist_iter it; + fds_blist_iter_init(&it, field, iemgr); + + // Print information about the list -> make sure that the field definition is filled + int rc = fds_blist_iter_next(&it); + if (rc != FDS_EOC && rc != FDS_OK) { + // Malformed + printf(")\n"); + print_indent(indent); + printf(" *Malformed data structure: %s*\n", fds_blist_iter_err(&it)); + return; + } + + uint32_t ie_en = it.field.info->en; + uint16_t ie_id = it.field.info->id; + const char *name_scope = ""; + const char *name_field = ""; + if (it.field.info->def != NULL) { + // Definition exists + name_scope = it.field.info->def->scope->name; + name_field = it.field.info->def->name; + } else { + // Definition not found... try to find the scope + const struct fds_iemgr_scope *scope_ptr = fds_iemgr_scope_find_pen(iemgr, ie_en); + if (scope_ptr != NULL) { + name_scope = scope_ptr->name; + } + } + + printf(", List Semantic: %s)\n", fds_semantic2str(it.semantic)); + //print_indent(indent); + //printf("> List fields: EN:%*"PRIu32" ID:%*"PRIu16" %*s:%-*s\n", + // WRITER_EN_SPACE, ie_en, WRITER_ID_SPACE, ie_id, + // WRITER_ORG_NAME_SPACE, name_scope, WRITER_FIELD_NAME_SPACE, name_field); + + // Re-initialize the iterator + fds_blist_iter_init(&it, field, iemgr); + unsigned int cnt_value = 0; + bool more_values = true; + + while (more_values) { + int rc = fds_blist_iter_next(&it); + switch (rc) { + case FDS_OK: // Process the record + break; + case FDS_EOC: // End of the list + more_values = false; + continue; + case FDS_ERR_FORMAT: // Something is wrong with the record + printf("*Unable to continue due to malformed data: %s*\n", fds_blist_iter_err(&it)); + return; + default: + printf("*Internal error: fds_blist_iter_next(): unexpected return code*\n"); + return; + } + + read_field(&it.field, indent + 1, iemgr, snap); + cnt_value++; + } + + if (cnt_value == 0) { + print_indent(indent + 1); + printf("EN: %-*"PRIu32" ID: %-*"PRIu16" ", WRITER_EN_SPACE, ie_en, WRITER_ID_SPACE, ie_id); + printf("%*s:%-*s : ", WRITER_ORG_NAME_SPACE, name_scope, WRITER_FIELD_NAME_SPACE, name_field); + printf("\n"); + } +} + +void +read_list_stl(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ + struct fds_stlist_iter it; + fds_stlist_iter_init(&it, field, snap, FDS_STL_REPORT); + print_indent(indent); + printf("> List semantic: %s, Template ID: %"PRIu16")\n", fds_semantic2str(it.semantic), it.tid); + + unsigned int cnt_rec = 0; + bool more_records = true; + + while (more_records) { + int rc = fds_stlist_iter_next(&it); + switch (rc) { + case FDS_OK: // Process the record + break; + case FDS_EOC: // No more records + more_records = false; + continue; + case FDS_ERR_NOTFOUND: // Template is not available + print_indent(indent); + printf(" *Template not available - unable to decode*\n"); + return; + case FDS_ERR_FORMAT: // Something is wrong with the record + print_indent(indent); + printf("*Unable to continue due to malformed data: %s*\n", fds_stlist_iter_err(&it)); + return; + default: + print_indent(indent); + printf("*Internal error: fds_stlist_iter_next(): unexpected return code*\n"); + return; + } + + print_indent(indent); + printf(" - Data Record (#%u) [Length: %"PRIu16"]\n", ++cnt_rec, it.rec.size); + read_record(&it.rec, indent + 1, iemgr); + } + + if (cnt_rec == 0) { + print_indent(indent + 1); + printf(" \n"); + } +} + +void +read_list_stml(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap) +{ + struct fds_stmlist_iter it; + fds_stmlist_iter_init(&it, field, snap, FDS_STL_REPORT); + print_indent(indent); + printf("> List semantic: %s\n", fds_semantic2str(it.semantic)); + + unsigned int cnt_block = 0; + bool more_blocks = true; + + // For each block in the list + while (more_blocks) { + int rc_block = fds_stmlist_iter_next_block(&it); + switch (rc_block) { + case FDS_OK: // Process the block + break; + case FDS_EOC: // No more blocks + more_blocks = false; + continue; + case FDS_ERR_NOTFOUND: // Unable to read this block -> skip it + break; + case FDS_ERR_FORMAT: // Something is wrong with the block + print_indent(indent); + printf("*Unable to continue due to malformed data: %s*\n", fds_stmlist_iter_err(&it)); + return; + default: + print_indent(indent); + printf("*Internal error: fds_stmlist_iter_next_block(): unexpected return code*\n"); + return; + } + + print_indent(indent); + printf("- Top-level list header (#%u) [Template ID: %"PRIu16"]\n", ++cnt_block, it.tid); + if (rc_block == FDS_ERR_NOTFOUND) { + print_indent(indent); + printf(" *Template not available - unable to decode*\n"); + continue; + } + + unsigned int cnt_rec = 0; + bool more_recs = true; + + while (more_recs) { + int rc_rec = fds_stmlist_iter_next_rec(&it); + switch (rc_rec) { + case FDS_OK: // Process the record + break; + case FDS_EOC: // No more records in the current block + more_recs = false; + continue; + case FDS_ERR_FORMAT: // Something is wrong with the record + print_indent(indent); + printf("*Unable to continue due to malformed data: %s*\n", fds_stmlist_iter_err(&it)); + return; + default: + print_indent(indent); + printf("*Internal error: fds_stmlist_iter_next_rec(): unexpected return code*\n"); + return; + } + + print_indent(indent); + printf(" - Data Record (#%u) [Length: %"PRIu16"]\n", ++cnt_rec, it.rec.size); + read_record(&it.rec, indent + 1, iemgr); + } + + if (cnt_rec == 0) { + print_indent(indent + 1); + printf(" \n"); + } + } + + if (cnt_block == 0) { + print_indent(indent); + printf(" \n"); + } +} \ No newline at end of file diff --git a/src/plugins/output/viewer/Reader.h b/src/plugins/output/viewer/Reader.h new file mode 100644 index 00000000..102a1dee --- /dev/null +++ b/src/plugins/output/viewer/Reader.h @@ -0,0 +1,169 @@ +/** + * \file src/plugins/output/viewer/Reader.h + * \author Jan Kala + * \brief Viewer - output module for printing information about incoming packets on stdout (header file) + * \date 2018 + */ +/* Copyright (C) 2018 CESNET, z.s.p.o. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in +* the documentation and/or other materials provided with the +* distribution. +* 3. Neither the name of the Company nor the names of its contributors +* may be used to endorse or promote products derived from this +* software without specific prior written permission. +* +* ALTERNATIVELY, provided that this notice is retained in full, this +* product may be distributed under the terms of the GNU General Public +* License (GPL) version 2 or later, in which case the provisions +* of the GPL apply INSTEAD OF those given above. +* +* This software is provided ``as is'', and any express or implied +* warranties, including, but not limited to, the implied warranties of +* merchantability and fitness for a particular purpose are disclaimed. +* In no event shall the company or contributors be liable for any +* direct, indirect, incidental, special, exemplary, or consequential +* damages (including, but not limited to, procurement of substitute +* goods or services; loss of use, data, or profits; or business +* interruption) however caused and on any theory of liability, whether +* in contract, strict liability, or tort (including negligence or +* otherwise) arising in any way out of the use of this software, even +* if advised of the possibility of such damage. +* +*/ +#ifndef IPFIXCOL_READER_H +#define IPFIXCOL_READER_H + +#include + +/** + * \brief spaces in output for Enterprise number field + */ +#define WRITER_EN_SPACE 8 +/** + * \brief Spaces in output for ID field + */ +#define WRITER_ID_SPACE 6 +/** + * \brief Spaces in output for size field + */ +#define WRITER_SIZE_SPACE 6 +/** + * \brief Spaces in output for Field name + */ +#define WRITER_FIELD_NAME_SPACE 35 +/** + * \brief Spaces in output for Organization name + */ +#define WRITER_ORG_NAME_SPACE 12 + +/** + * \brief Print all data of an IPFIX Message + * + * Function reads and prints the header of the packet and then iterates through the Sets and their + * records. Information printed from the header of the packet are: + * version, length, export time, sequence number and Observation Domain ID + * \param[in] msg IPFIX message which will be printed + * \param[in] iemgr Information Element manager + */ +void +read_packet(ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr); + +/** + * \brief Print all values inside the single Data Record of an IPFIX message + * + * Reads and prints the header of the Data Record and then iterates through its fields. + * \param[in] rec Record which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + */ +void +read_record(struct fds_drec *rec, unsigned int indent, const fds_iemgr_t *iemgr); + +/** + * \brief Print the value of the Data Record field + * + * Reads and prints all information about the data in the field. If the detailed definition is + * known, value is printed in human readable format. Otherwise data are printed in the raw + * format (hexadecimal). In both cases Enterprise number and ID will be printed. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record + */ +void +read_field(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); + +/** + * \brief Print the content of basicList data type + * + * Iterates through the list and prints all values. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record + */ +void +read_list_basic(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); + +/** + * \brief Print the content of subTemplateList data type + * + * Iterates through the list and prints all Data Records. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record + */ +void +read_list_stl(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); + +/** + * \brief Print the content of subTemplateMultiList data type + * + * Iterates through the list and prints all top-level lists and their content i.e. Data Records. + * \param[in] field Field which will be printed + * \param[in] indent Additional output indentation + * \param[in] iemgr Information Element manager + * \param[in] snap Template snapshot of the Data Record + */ +void +read_list_stml(struct fds_drec_field *field, unsigned int indent, const fds_iemgr_t *iemgr, + const fds_tsnapshot_t *snap); + +/** + * \brief Print the (Options) Template Record + * + * Reads and prints content of the (Options) Template Record. Uses Information Element manager + * for determine description of present Information Elements. + * \param[in] tset_iter Template iterator + * \param[in] set_id ID of the Set to which the record belongs + * \param[in] iemgr Information Element manager + */ +void +read_template_set(struct fds_tset_iter *tset_iter, uint16_t set_id, const fds_iemgr_t *iemgr); + +/** + * \brief Print the IPFIX Set and its content + * + * Reads and prints single IPFIX Set and determines what kind of IPFIX Set it is and which + * read_xxx function to use for reading its content. + * + * \param[in] sets_iter Sets iterator + * \param[in] msg IPFIX message + * \param[in] iemgr Information Element manager + * \param[in/out] rec_i Number of processed Data Records + */ +void +read_set(struct ipx_ipfix_set *set, ipx_msg_ipfix_t *msg, const fds_iemgr_t *iemgr, uint32_t *rec_i); + +#endif //IPFIXCOL_READER_H diff --git a/src/plugins/output/viewer/config.c b/src/plugins/output/viewer/config.c new file mode 100644 index 00000000..8d35e89c --- /dev/null +++ b/src/plugins/output/viewer/config.c @@ -0,0 +1,154 @@ +/** + * \file src/plugins/output/viewer/config.c + * \author Jan Kala + * \brief Parser of an XML configuration for viewer module (source file) + * \date 2018 + */ + +/* Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include "config.h" + +/* + * + * ... + * + */ + +/** XML nodes */ +enum params_xml_nodes { + NODE_DELAY = 1 +}; + +/** Definition of the \ node */ +static const struct fds_xml_args args_params[] = { + FDS_OPTS_ROOT("params"), + FDS_OPTS_END +}; + +/** + * \brief Process \ node + * \param[in] ctx Plugin context + * \param[in] root XML context to process + * \param[in] cfg Parsed configuration + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT in case of failure + */ +static int +config_parser_root(ipx_ctx_t *ctx, fds_xml_ctx_t *root, struct instance_config *cfg) +{ + (void) ctx; + + const struct fds_xml_cont *content; + while(fds_xml_next(root, &content) != FDS_EOC) { + switch (content->id) { + case NODE_DELAY: + // Delay between messages [microseconds] + assert(content->type == FDS_OPTS_T_UINT); + cfg->sleep_time.tv_nsec = (content->val_uint % 1000000LL) * 1000LL; + cfg->sleep_time.tv_sec = content->val_uint / 1000000LL; + break; + default: + // Internal error + assert(false); + break; + } + } + + return IPX_OK; +} + +/** + * \brief Set default parameters of the configuration + * \param[in] cfg Configuration + */ +static void +config_default_set(struct instance_config *cfg) +{ + (void) cfg; +} + +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params) +{ + struct instance_config *cfg = calloc(1, sizeof(*cfg)); + if (!cfg) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + return NULL; + } + + // Set default parameters + config_default_set(cfg); + + // Create an XML parser + fds_xml_t *parser = fds_xml_create(); + if (!parser) { + IPX_CTX_ERROR(ctx, "Memory allocation error (%s:%d)", __FILE__, __LINE__); + free(cfg); + } + + if (fds_xml_set_args(parser, args_params) != IPX_OK) { + IPX_CTX_ERROR(ctx, "Failed to parse the description of an XML document!", '\0'); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + fds_xml_ctx_t *params_ctx = fds_xml_parse_mem(parser, params, true); + if (params_ctx == NULL) { + IPX_CTX_ERROR(ctx, "Failed to parse the configuration: %s", fds_xml_last_err(parser)); + fds_xml_destroy(parser); + free(cfg); + return NULL; + } + + // Parse parameters + int rc = config_parser_root(ctx, params_ctx, cfg); + fds_xml_destroy(parser); + if (rc != IPX_OK) { + free(cfg); + return NULL; + } + + return cfg; +} + +void +config_destroy(struct instance_config *cfg) +{ + free(cfg); +} diff --git a/src/plugins/output/viewer/config.h b/src/plugins/output/viewer/config.h new file mode 100644 index 00000000..46f864ae --- /dev/null +++ b/src/plugins/output/viewer/config.h @@ -0,0 +1,71 @@ +/** + * \file src/plugins/output/dummy/config.h + * \author Jan Kala + * \brief Parser of an XML configuration for viewer module (header file) + * \date 2018 + */ + +/* Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef CONFIG_H +#define CONFIG_H + +#include +#include "stdint.h" + +/** Configuration of a instance of the dummy plugin */ +struct instance_config { + /** Sleep time */ + struct timespec sleep_time; +}; + +/** + * \brief Parse configuration of the plugin + * \param[in] ctx Instance context + * \param[in] params XML parameters + * \return Pointer to the parse configuration of the instance on success + * \return NULL if arguments are not valid or if a memory allocation error has occurred + */ +struct instance_config * +config_parse(ipx_ctx_t *ctx, const char *params); + +/** + * \brief Destroy parsed configuration + * \param[in] cfg Parsed configuration + */ +void +config_destroy(struct instance_config *cfg); + +#endif // CONFIG_H diff --git a/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst new file mode 100644 index 00000000..e779b726 --- /dev/null +++ b/src/plugins/output/viewer/doc/ipfixcol2-viewer-output.7.rst @@ -0,0 +1,21 @@ +======================== + ipfixcol2-viewer-output +======================== + +---------------------- +Viewer (output plugin) +---------------------- + +:Author: Jan Kala (xkalaj01@stud.fit.vutbr.cz) +:Author: Lukáš Huták (lukas.hutak@cesnet.cz) +:Date: 2019-02-20 +:Copyright: Copyright © 2018-2019 CESNET, z.s.p.o. +:Version: 2.0 +:Manual section: 7 +:Manual group: IPFIXcol collector + +Description +----------- + +.. include:: ../README.rst + :start-line: 3 diff --git a/src/plugins/output/viewer/viewer.c b/src/plugins/output/viewer/viewer.c new file mode 100644 index 00000000..4ca4d2d8 --- /dev/null +++ b/src/plugins/output/viewer/viewer.c @@ -0,0 +1,123 @@ +/** + * \file src/plugins/output/viewer/viewer.c + * \author Jan Kala + * \brief Viewer output plugin for IPFIXcol 2 + * \date 2018 + */ + +/* Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include +#include + +#include "config.h" +#include "../../input/dummy/config.h" +#include "../../../core/message_ipfix.h" +#include "Reader.h" + +/** Plugin description */ +IPX_API struct ipx_plugin_info ipx_plugin_info = { + // Plugin type + .type = IPX_PT_OUTPUT, + // Plugin identification name + .name = "viewer", + // Brief description of plugin + .dsc = "Output plugin for ptinting information about incoming IPFIX messages.", + // Configuration flags (reserved for future use) + .flags = 0, + // Plugin version string (like "1.2.3") + .version = "2.0.0", + // Minimal IPFIXcol version string (like "1.2.3") + .ipx_min = "2.1.0" +}; + +/** Instance */ +struct instance_data { + /** Parsed configuration of the instance */ + struct instance_config *config; +}; + +int +ipx_plugin_init(ipx_ctx_t *ctx, const char *params) +{ + // Create a private data + struct instance_data *data = calloc(1, sizeof(*data)); + if (!data) { + return IPX_ERR_DENIED; + } + + if ((data->config = config_parse(ctx, params)) == NULL) { + free(data); + return IPX_ERR_DENIED; + } + + ipx_ctx_private_set(ctx, data); + + // Subscribe to receive IPFIX messages and Transport Session events + uint16_t new_mask = IPX_MSG_IPFIX | IPX_MSG_SESSION; + ipx_ctx_subscribe(ctx, &new_mask, NULL); + return IPX_OK; +} + +void +ipx_plugin_destroy(ipx_ctx_t *ctx, void *cfg) +{ + (void) ctx; // Suppress warnings + + struct instance_data *data = (struct instance_data *) cfg; + config_destroy(data->config); + free(data); +} + +int +ipx_plugin_process(ipx_ctx_t *ctx, void *cfg, ipx_msg_t *msg) +{ + (void) cfg; + //Check of the type + int type = ipx_msg_get_type(msg); + if (type != IPX_MSG_IPFIX) { + return IPX_OK; + } + + const fds_iemgr_t *iemgr = ipx_ctx_iemgr_get(ctx); + + //Convert the message to the IPFIX message and read it + ipx_msg_ipfix_t *ipfix_msg = ipx_msg_base2ipfix(msg); + read_packet(ipfix_msg, iemgr); + + return IPX_OK; +}