From d1fa93bafda04836e9ea82506397d04e2ba32c91 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 27 Nov 2018 12:05:08 +0100 Subject: [PATCH] NetFlow converters: added NetFlow v5/v9 to IPFIX converters, NetFlow v9 iterators, etc. (not tested, unit tests will follow) --- include/ipfixcol2/api.h.in | 2 + src/core/CMakeLists.txt | 10 +- src/core/netflow2ipfix/netflow2ipfix.h | 180 +++ src/core/netflow2ipfix/netflow5.c | 498 +++++++ src/core/netflow2ipfix/netflow9.c | 1301 +++++++++++++++++++ src/core/netflow2ipfix/netflow9_templates.c | 187 +++ src/core/netflow2ipfix/netflow9_templates.h | 228 ++++ src/core/netflow2ipfix/netflow_parsers.c | 340 +++++ src/core/netflow2ipfix/netflow_parsers.h | 308 +++++ src/core/netflow2ipfix/netflow_structs.h | 348 +++++ 10 files changed, 3401 insertions(+), 1 deletion(-) create mode 100644 src/core/netflow2ipfix/netflow2ipfix.h create mode 100644 src/core/netflow2ipfix/netflow5.c create mode 100644 src/core/netflow2ipfix/netflow9.c create mode 100644 src/core/netflow2ipfix/netflow9_templates.c create mode 100644 src/core/netflow2ipfix/netflow9_templates.h create mode 100644 src/core/netflow2ipfix/netflow_parsers.c create mode 100644 src/core/netflow2ipfix/netflow_parsers.h create mode 100644 src/core/netflow2ipfix/netflow_structs.h diff --git a/include/ipfixcol2/api.h.in b/include/ipfixcol2/api.h.in index 00c70d2f..6bf6431b 100644 --- a/include/ipfixcol2/api.h.in +++ b/include/ipfixcol2/api.h.in @@ -105,6 +105,8 @@ extern "C" { // Note: Following codes preservers the same numbering as libfds /** Status code for success */ #define IPX_OK (0) +/** Status code for the end of a context */ +#define IPX_EOC (-1) /** Status code for ready operation */ #define IPX_READY (-11) /** Status code for memory allocation error */ diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 42cbc33e..08fe4fd7 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -17,6 +17,14 @@ set(CORE_SOURCE configurator/plugin_mgr.hpp configurator/model.cpp configurator/model.hpp + netflow2ipfix/netflow2ipfix.h + netflow2ipfix/netflow5.c + netflow2ipfix/netflow9.c + netflow2ipfix/netflow9_templates.c + netflow2ipfix/netflow9_templates.h + netflow2ipfix/netflow_parsers.c + netflow2ipfix/netflow_parsers.h + netflow2ipfix/netflow_structs.h api.c context.c context.h @@ -76,4 +84,4 @@ set_target_properties(ipfixcol2 PROPERTIES # by default, hide all symbols install( TARGETS ipfixcol2 DESTINATION bin -) \ No newline at end of file +) diff --git a/src/core/netflow2ipfix/netflow2ipfix.h b/src/core/netflow2ipfix/netflow2ipfix.h new file mode 100644 index 00000000..21b9388a --- /dev/null +++ b/src/core/netflow2ipfix/netflow2ipfix.h @@ -0,0 +1,180 @@ +/** + * \file src/core/netflow2ipfix/netflow2ipfix.h + * \author Lukas Hutak + * \brief Main NetFlow v5/v9 to IPFIX converter functions (header file) + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef IPFIXCOL2_NETFLOW2IPFIX_H +#define IPFIXCOL2_NETFLOW2IPFIX_H + +#include +#include +#include + +/** + * \defgroup nf5_to_ipfix NetFlow v5 to IPFIX + * \brief Conversion from NetFlow v5 Messages to IPFIX Messages + * + * The converter helps to convert a stream of NetFlow Messages from a NetFlow exporter to stream + * of IPFIX Messages. Messages are processed individually and should be passed to the + * converter in the order send by the exporter. If it is necessary to convert streams from + * multiple exporters at time, you MUST create an independent instance for each stream. + * + * @{ + */ + +/** Auxiliary definition of NetFlow v5 to IPFIX converter internals */ +typedef struct ipx_nf5_conv ipx_nf5_conv_t; + +/** + * \brief Initialize NetFlow v5 to IPFIX converter + * + * \note + * Template refresh interval (\p tmplt_refresh) refers to exporter timestamps in NetFlow + * Messages to convert, not to wall-clock time. + * \param[in] ident Instance identification (only for log messages!) + * \param[in] vlevel Verbosity level of the converter (i.e. amount of log messages) + * \param[in] tmplt_refresh Template refresh interval (seconds, 0 == disabled) + * \param[in] odid Observation Domain ID of IPFIX Messages (e.g. 0) + * \return Pointer to the converter or NULL (memory allocation error) + */ +ipx_nf5_conv_t * +ipx_nf5_conv_init(const char *ident, enum ipx_verb_level vlevel, uint32_t tmplt_refresh, + uint32_t odid); + +/** + * \brief Destroy NetFlow v5 to IPFIX converter + * \param[in] conv Converter to destroy + */ +void +ipx_nf5_conv_destroy(ipx_nf5_conv_t *conv); + +/** + * \brief Convert NetFlow v5 message to IPFIX message + * + * The function accepts a message wrapper \p wrapper that should hold a NetFlow v5 Message. + * If the NetFlow Message is successfully converted, a content of the wrapper is replaced + * with the IPFIX Message and the original NetFlow Message is not accessible anymore and it is + * freed. + * + * \note + * In case of an error (i.e. return code different from #IPX_OK) the original NetFlow Message + * in the wrapper is untouched. + * \param[in] conv Message converter + * \param[in] wrapper Message wrapper + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT in case of invalid NetFlow Message format + * \return #IPX_ERR_NOMEM in case of a memory allocation error + */ +int +ipx_nf5_conv_process(ipx_nf5_conv_t *conv, ipx_msg_ipfix_t *wrapper); + +/** + * @} + */ + +/** + * \defgroup nf9_to_ipfix NetFlow v9 to IPFIX + * \brief Conversion from NetFlow v9 Messages to IPFIX Messages + * + * The converter helps to convert a stream of NetFlow Messages from combination of a NetFlow + * exporter and a Source ID (a.k.a. Observation Domain ID) to stream of IPFIX Messages. Messages + * are processed individually and should be passed to the converter in the order send by the + * exporter. If it is necessary to convert streams from the same exporter with different Source + * IDs or from multiple exporters at time, you MUST create an independent instance for each + * stream (i.e. combination of an Exporter and Source ID). + * + * \note + * In the context of IPFIX protocol, the Source ID is referred as Observation Domain ID (ODID) + * + * @{ + */ + +/** Auxiliary definition of NetFlow v9 to IPFIX converter internals */ +typedef struct ipx_nf9_conv ipx_nf9_conv_t; + +/** + * \brief Initialize NetFlow v9 to IPFIX converter + * + * \param[in] ident Instance identification (only for log messages!) + * \param[in] vlevel Verbosity level of the converter (i.e. amount of log messages) + * \return Pointer to the converter or NULL (memory allocation error) + */ +ipx_nf9_conv_t * +ipx_nf9_conv_init(const char *ident, enum ipx_verb_level vlevel); + +/** + * \brief Destroy NetFlow v9 to IPFIX converter + * \param[in] conv Converter to destroy + */ +void +ipx_nf9_conv_destroy(ipx_nf9_conv_t *conv); + +/** + * \brief Convert NetFlow v9 Message to IPFIX Message + * + * The function accepts a message wrapper \p wrapper that should hold a NetFlow v9 Message. + * If the NetFlow Message is successfully converted, a content of the wrapper is replaced + * with the IPFIX Message and the original NetFlow Message is not accessible anymore and it is + * freed. + * + * \note + * In case of an error (i.e. return code different from #IPX_OK) the original NetFlow Message + * in the wrapper is untouched. + * \note + * After conversion the IPFIX Message is not ready to be used for accessing flow records, + * it MUST be processed by the IPFIX Parser first. + * \param[in] conv Message converter + * \param[in] wrapper Message wrapper + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT in case of invalid NetFlow Message format + * \return #IPX_ERR_NOMEM in case of a memory allocation error + */ +int +ipx_nf9_conv_process(ipx_nf9_conv_t *conv, ipx_msg_ipfix_t *wrapper); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif // IPFIXCOL2_NETFLOW2IPFIX_H diff --git a/src/core/netflow2ipfix/netflow5.c b/src/core/netflow2ipfix/netflow5.c new file mode 100644 index 00000000..8e3ce6f8 --- /dev/null +++ b/src/core/netflow2ipfix/netflow5.c @@ -0,0 +1,498 @@ +/** + * \file src/core/netflow2ipfix/netflow5.c + * \author Lukas Hutak + * \brief Converter from NetFlow v5 to IPFIX Message (source code) + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "netflow_structs.h" +#include "netflow2ipfix.h" +#include "../message_ipfix.h" +#include "../verbose.h" + +// Simple static asserts to prevent unexpected structure modifications! +static_assert(IPX_NF5_MSG_HDR_LEN == 24U, "NetFlow v5 header size is not valid!"); +static_assert(IPX_NF5_MSG_REC_LEN == 48U, "NetFlow v5 record size is not valid!"); + +/** Auxiliary definition to represent size of 1 byte */ +#define BYTES_1 (1U) +/** Auxiliary definition to represent size of 2 bytes */ +#define BYTES_2 (2U) +/** Auxiliary definition to represent size of 4 bytes */ +#define BYTES_4 (4U) +/** Auxiliary definition to represent size of 8 bytes */ +#define BYTES_8 (8U) + +/** + * \brief IPFIX template set of modified NetFlow v5 record + * + * The set consists of Template Set header and a single Template definition. + * \note + * All values are in "host byte order" and MUST be converted to "network byte order" before + * they can be used! + * \warning + * "first" and "last" timestamps (relative to exporter time SysUpTime) are replaced with + * absolute timestamps in milliseconds! + */ +static const uint16_t nf5_tmpl_set[] = { + // IPFIX Set header + IPFIX Template header (ID 256) + FDS_IPFIX_SET_TMPLT, 0, // Size and field count will be filled later + FDS_IPFIX_SET_MIN_DSET, 0, + // Template fields + 8U, BYTES_4, // iana:sourceIPv4Address + 12U, BYTES_4, // iana:destinationIPv4Address + 15U, BYTES_4, // iana:ipNextHopIPv4Address + 10U, BYTES_2, // iana:ingressInterface + 14U, BYTES_2, // iana:egressInterface + 2U, BYTES_4, // iana:packetDeltaCount + 1U, BYTES_4, // iana:octetDeltaCount + 152U, BYTES_8, // iana:flowStartMilliseconds (absolute timestamp in milliseconds) + 153U, BYTES_8, // iana:flowEndMilliseconds (absolute timestamp in milliseconds) + 7U, BYTES_2, // iana:sourceTransportPort + 11U, BYTES_2, // iana:destinationTransportPort + 210U, BYTES_1, // iana:paddingOctets + 6U, BYTES_1, // iana:tcpControlBits + 4U, BYTES_1, // iana:protocolIdentifier + 5U, BYTES_1, // iana:ipClassOfService + 16U, BYTES_2, // iana:bgpSourceAsNumber + 17U, BYTES_2, // iana:bgpDestinationAsNumber + 9U, BYTES_1, // iana:sourceIPv4PrefixLength + 13U, BYTES_1, // iana:destinationIPv4PrefixLength + 210U, BYTES_2 // iana:paddingOctets +}; + +/** Number of fields (including header fields) in the Template Set */ +#define NF5_TSET_ITEMS (sizeof(nf5_tmpl_set) / sizeof(nf5_tmpl_set[0])) +static_assert(NF5_TSET_ITEMS % 2 == 0, "Number of fields MUST be even!"); + +/** Offset of "first" timestamp in the new IPFIX Message */ +#define IPX_FIRST_OFFSET (offsetof(struct ipx_nf5_rec, ts_first)) +/** Offset of "last" timestamp in the new IPFIX Message */ +#define IPX_LAST_OFFSET (IPX_FIRST_OFFSET + sizeof(uint64_t)) + +/** Start position of the first part to copy in the original NetFlow v5 Message */ +#define PART1_NF_POS(x) ((uint8_t *) (x)) +/** Start position of the first part to copy in the new IPFIX Message */ +#define PART1_IPX_POS(x) ((uint8_t *) (x)) +/** Size of the first part to copy */ +#define PART1_LEN (offsetof(struct ipx_nf5_rec, ts_first)) + +/** Start position of the second part to copy in the original NetFlow v5 Message */ +#define PART2_NF_POS(x) (((uint8_t *) (x)) + offsetof(struct ipx_nf5_rec, port_src)) +/** Start position of the second part to copy in the new IPFIX Message */ +#define PART2_IPX_POS(x) (((uint8_t *) (x)) + IPX_LAST_OFFSET + sizeof(uint64_t)) +/** Size of the second part to copy */ +#define PART2_LEN (IPX_NF5_MSG_REC_LEN - offsetof(struct ipx_nf5_rec, port_src)) + +/** + * \def CONV_ERROR + * \brief Macro for printing an error message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_ERROR(conv, fmt, ...) \ + if ((conv)->conf.vlevel >= IPX_VERB_ERROR) { \ + ipx_verb_print(IPX_VERB_ERROR, "ERROR: %s: [%s] " fmt "\n", \ + (conv)->conf.ident, (conv)->msg_ctx->session->ident, ## __VA_ARGS__); \ + } + +/** + * \def CONV_WARNING + * \brief Macro for printing a warning message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_WARNING(conv, fmt, ...) \ + if ((conv)->conf.vlevel >= IPX_VERB_WARNING) { \ + ipx_verb_print(IPX_VERB_WARNING, "WARNING: %s: [%s] " fmt "\n", \ + (conv)->conf.ident, (conv)->msg_ctx->session->ident, ## __VA_ARGS__); \ + } + +/** + * \def CONV_INFO + * \brief Macro for printing an info message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_INFO(conv, fmt, ...) \ + if ((conv)->conf.vlevel >= IPX_VERB_INFO) { \ + ipx_verb_print(IPX_VERB_INFO, "INFO: %s: [%s] " fmt "\n", \ + (conv)->conf.ident, (conv)->msg_ctx->session->ident, ## __VA_ARGS__); \ + } + +/** + * \def CONV_DEBUG + * \brief Macro for printing a debug message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_DEBUG(conv, fmt, ...) \ + if ((conv)->conf.vlevel >= IPX_VERB_DEBUG) { \ + ipx_verb_print(IPX_VERB_DEBUG, "DEBUG: %s: [%s] " fmt "\n", \ + (conv)->conf.ident, (conv)->msg_ctx->session->ident, ## __VA_ARGS__); \ + } + +/** Internal converter structure */ +struct ipx_nf5_conv { + /** Message context of a NetFlow message that is converted */ + const struct ipx_msg_ctx *msg_ctx; + + struct { + /** Instance identification (only for log!) */ + char *ident; + /** Verbosity level */ + enum ipx_verb_level vlevel; + /** Template refresh interval (in seconds) */ + uint32_t refresh; + /** Observation Domain ID of IPFIX message */ + uint32_t odid; + } conf; /**< Configuration parameters */ + + struct { + /** Has the template been already sent? */ + bool added; + /** Timestamp of the next template refresh (must be enabled in conf.) + * If the "added" is false, this value is undefined! */ + uint32_t next_refresh; + + /** Template Set data (in "network byte order") */ + uint16_t *tset_data; + /** Template Set size */ + size_t tset_size; + /** Size of converted data record */ + size_t drec_size; + } tmplt; /**< Template information */ +}; + + +ipx_nf5_conv_t * +ipx_nf5_conv_init(const char *ident, enum ipx_verb_level vlevel, uint32_t tmplt_refresh, + uint32_t odid) +{ + struct ipx_nf5_conv *res = calloc(1, sizeof(*res)); + if (!res) { + return NULL; + } + + res->conf.ident = strdup(ident); + if (!res->conf.ident) { + free(res); + return NULL; + } + + // Prepare Template Set and calculate required parameters + size_t tset_size = sizeof(nf5_tmpl_set); + uint16_t *tset_data = malloc(tset_size); + if (!tset_data) { + free(res->conf.ident); + free(res); + return NULL; + } + + + uint16_t drec_size = 0; + uint16_t field_cnt = 0; + + for (size_t i = 0; i < NF5_TSET_ITEMS; ++i) { + // Convert fields from "host byte order" to "network byte order" + tset_data[i] = htons(nf5_tmpl_set[i]); + + if (i >= 4 && i % 2 == 1) { // Ignore Set and Template header i.e. ">= 4" + field_cnt++; + drec_size += nf5_tmpl_set[i]; + } + } + // Update size and field count + struct fds_ipfix_tset *tset_raw = (struct fds_ipfix_tset *) tset_data; + tset_raw->header.length = htons((uint16_t) tset_size); + tset_raw->first_record.count = htons(field_cnt); + + res->tmplt.tset_data = tset_data; + res->tmplt.tset_size = tset_size; + res->tmplt.drec_size = drec_size; + res->tmplt.added = false; + + // Initialize remaining parameters + res->conf.refresh = tmplt_refresh; + res->conf.odid = odid; + res->conf.vlevel = vlevel; + return res; +} + +void +ipx_nf5_conv_destroy(ipx_nf5_conv_t *conv) +{ + free(conv->conf.ident); + free(conv->tmplt.tset_data); + free(conv); +} + +/** + * \brief Calculate required size of the new IPFIX Message + * + * The new IPFIX Message will consists of IPFIX Message header (mandatory), predefined + * IPFIX Template Set (optional) and IPFIX Data Set (optional) + * \param[in] conv Converter internals + * \param[in] rec_cnt Number of data records + * \param[in] tmplt Add IPFIX Template Set with the Template + * \return Total size of the new IPFIX Message (in bytes) + */ +static inline size_t +conv_new_size(const ipx_nf5_conv_t *conv, uint16_t rec_cnt, bool tmplt) +{ + // IPFIX Message header + size_t ret = FDS_IPFIX_MSG_HDR_LEN; + + if (tmplt) { + // Add IPFIX Template Set (with 1 template) + ret += conv->tmplt.tset_size; + } + + if (rec_cnt > 0) { + // Add IPFIX Data Set header + records based on the template + ret += FDS_IPFIX_SET_HDR_LEN; + ret += (rec_cnt * conv->tmplt.drec_size); + } + return ret; +} + +/** + * \brief Compare message timestamps (with timestamp wraparound support) + * \param t1 First timestamp + * \param t2 Second timestamp + * \return The function returns an integer less than, equal to, or greater than zero if the + * first timestamp \p t1 is found, respectively, to be less than, to match, or be greater than + * the second timestamp. + */ +static inline int +conv_time_cmp(uint32_t t1, uint32_t t2) +{ + if (t1 == t2) { + return 0; + } + + if ((t1 - t2) & 0x80000000) { // test the "sign" bit + return (-1); + } else { + return 1; + } +} + +/** + * \brief Add the IPFIX Message header + * + * \param[in] nf_msg Original NetFlow v5 message + * \param[in] ipx_data Pointer to the place where the new IPFIX header will be placed + * \param[in] ipx_size Total size of the new IPFIX Message to be set + * \param[in] ipx_odid Observation Domain ID (ODID) to be set + * \return Pointer to the memory right behind the added header + */ +static inline uint8_t * +conv_add_hdr(const uint8_t *nf_msg, uint8_t *ipx_data, uint16_t ipx_size, uint32_t ipx_odid) +{ + const struct ipx_nf5_hdr *nf5_hdr = (const struct ipx_nf5_hdr *) nf_msg; + struct fds_ipfix_msg_hdr *ipx_hdr = (struct fds_ipfix_msg_hdr *) ipx_data; + + ipx_hdr->version = htons(FDS_IPFIX_VERSION); + ipx_hdr->length = htons(ipx_size); + ipx_hdr->export_time = nf5_hdr->unix_sec; + ipx_hdr->seq_num = nf5_hdr->flow_seq; + ipx_hdr->odid = htonl(ipx_odid); + + return (ipx_data + FDS_IPFIX_MSG_HDR_LEN); +} + +/** + * \brief Add the predefined IPFIX Template Set + * + * \param[in] conv Converter internals + * \param[in] ipx_data Pointer to the place where the new IPFIX Template Set will be placed + * \return Pointer to the memory right behind the added Template Set + */ +static inline uint8_t * +conv_add_tset(const ipx_nf5_conv_t *conv, uint8_t *ipx_data) +{ + memcpy(ipx_data, conv->tmplt.tset_data, conv->tmplt.tset_size); + return (ipx_data + conv->tmplt.tset_size); +} + +/** + * \brief Add the IPFIX Data Set with converted records + * + * The function will add the Data Set header and all NetFlow records are converted to modified + * IPFIX records, where relative timestamps are replaced with absolute ones + * + * \note + * If the NetFlow record doesn't contain any data record, nothing is added. + * \param[in] conv Converter internals + * \param[in] nf_msg Original NetFlow v5 message + * \param[in] ipx_data Pointer to the place where the new IPFIX Data Set will be placed + * \return Pointer to the memory right behind the added Data Set + */ +static inline uint8_t * +conv_add_dset(const ipx_nf5_conv_t *conv, const uint8_t *nf_msg, uint8_t *ipx_data) +{ + const struct ipx_nf5_hdr *nf_hdr = (const struct ipx_nf5_hdr *) nf_msg; + const uint16_t rec_cnt = ntohs(nf_hdr->count); + + if (rec_cnt == 0) { + // Nothing to add + return ipx_data; + } + + // Prepare for timestamps conversion + const uint64_t hdr_time_sec = ntohl(nf_hdr->unix_sec); + const uint64_t hdr_time_nsec = ntohl(nf_hdr->unix_nsec); + const uint64_t hdr_exp_time = (hdr_time_sec * 1000U) + (hdr_time_nsec / 1000000U); + const uint64_t hdr_sys_time = ntohl(nf_hdr->sys_uptime); + + // Add IPFIX Data Set header + struct fds_ipfix_dset *ipx_dset = (struct fds_ipfix_dset *) ipx_data; + const size_t dset_len = FDS_IPFIX_SET_HDR_LEN + (rec_cnt * conv->tmplt.drec_size); + ipx_dset->header.flowset_id = htons(FDS_IPFIX_SET_MIN_DSET); + ipx_dset->header.length = htons((uint16_t) dset_len); + + // Add all data records + uint8_t *ipx_rec = &ipx_dset->records[0]; + for (uint16_t i = 0; i < rec_cnt; ++i, ipx_rec += conv->tmplt.drec_size) { + const struct ipx_nf5_rec *nf_rec = (const struct ipx_nf5_rec *) + (nf_msg + IPX_NF5_MSG_HDR_LEN + (i * IPX_NF5_MSG_REC_LEN)); + // New timestamps (in milliseconds) + uint64_t ts_start = hdr_exp_time - (hdr_sys_time - ntohl(nf_rec->ts_first)); + uint64_t ts_end = hdr_exp_time - (hdr_sys_time - ntohl(nf_rec->ts_last)); + uint64_t *ptr_first = (uint64_t *) (ipx_rec + IPX_FIRST_OFFSET); + uint64_t *ptr_last = (uint64_t *) (ipx_rec + IPX_LAST_OFFSET); + + // Copy and extend the message record + memcpy(PART1_IPX_POS(ipx_rec), PART1_NF_POS(nf_rec), PART1_LEN); + *ptr_first = htobe64(ts_start); + *ptr_last = htobe64(ts_end); + memcpy(PART2_IPX_POS(ipx_rec), PART2_NF_POS(nf_rec), PART2_LEN); + } + + return (ipx_data + dset_len); +} + + +int +ipx_nf5_conv_process(ipx_nf5_conv_t *conv, ipx_msg_ipfix_t *wrapper) +{ + const struct ipx_nf5_hdr *nf5_hdr = (const struct ipx_nf5_hdr *) wrapper->raw_pkt; + const size_t nf5_size = wrapper->raw_size; + conv->msg_ctx = &wrapper->ctx; + + // Check the header and expected message size + if (nf5_size < IPX_NF5_MSG_HDR_LEN) { + CONV_ERROR(conv, "Length of NetFlow v5 Message is smaller than its header size!", '\0'); + return IPX_ERR_FORMAT; + } + + if (ntohs(nf5_hdr->version) != IPX_NF5_VERSION) { + CONV_ERROR(conv, "Invalid version number of NetFlow v5 Message (expected 5)", '\0'); + return IPX_ERR_FORMAT; + } + + uint16_t msg_seq = ntohl(nf5_hdr->flow_seq); + CONV_DEBUG(conv, "Converting a NetFlow Message v5 (seq. num. %" PRIu16" to an IPFIX Message " + "(new seq. num. %" PRIu16 ")", msg_seq, msg_seq); // Yep, both numbers are the same... + + size_t rec_cnt = ntohs(nf5_hdr->count); + if (IPX_NF5_MSG_HDR_LEN + (rec_cnt * IPX_NF5_MSG_REC_LEN) != nf5_size) { + CONV_ERROR(conv, "Length of NetFlow v5 Message doesn't match with the number of records " + "specified in the header.", '\0'); + return IPX_ERR_FORMAT; + } + + const uint16_t nf5_rec_cnt = ntohs(nf5_hdr->count); + const uint32_t nf5_rec_time = ntohl(nf5_hdr->unix_sec); + + // Create a new IPFIX Message + bool add_tset = !conv->tmplt.added + || (conv->conf.refresh != 0 && conv_time_cmp(nf5_rec_time, conv->tmplt.next_refresh) >= 0); + + const size_t ipx_size = conv_new_size(conv, nf5_rec_cnt, add_tset); + if (ipx_size > UINT16_MAX) { + /* Typically NetFlow messages SHOULD not have more that 30 record (due to UDP MTU), however, + * we support as many records per message as possible + */ + CONV_ERROR(conv, "Unable to convert NetFlow v5 to IPFIX. Size of the converted message " + "exceeds the maximum size of an IPFIX Message! (before: %" PRIu16 " B, after: %zu B)", + nf5_size, ipx_size); + return IPX_ERR_FORMAT; + } + + const uint8_t *nf5_msg = wrapper->raw_pkt; + uint8_t *ipx_msg = malloc(ipx_size * sizeof(uint8_t)); + if (!ipx_msg) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + // Fill the IPFIX Message + uint8_t *next_set = ipx_msg; + next_set = conv_add_hdr(nf5_msg, next_set, (uint16_t) ipx_size, conv->conf.odid); + + if (add_tset) { + CONV_DEBUG(conv, "Adding a Template Set into the converted NetFlow Message.", '\0'); + next_set = conv_add_tset(conv, next_set); + conv->tmplt.added = true; + conv->tmplt.next_refresh = nf5_rec_time + conv->conf.refresh; + } + next_set = conv_add_dset(conv, nf5_msg, next_set); + + // Finally, replace the converted NetFlow Message with the new IPFIX Message + assert(next_set == (ipx_msg + ipx_size)); + free(wrapper->raw_pkt); + wrapper->raw_pkt = ipx_msg; + wrapper->raw_size = (uint16_t) ipx_size; + return IPX_OK; +} \ No newline at end of file diff --git a/src/core/netflow2ipfix/netflow9.c b/src/core/netflow2ipfix/netflow9.c new file mode 100644 index 00000000..fd385908 --- /dev/null +++ b/src/core/netflow2ipfix/netflow9.c @@ -0,0 +1,1301 @@ +/** + * \file src/core/netflow2ipfix/netflow9.c + * \author Lukas Hutak + * \brief Converter from NetFlow v9 to IPFIX Message (source code) + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "netflow2ipfix.h" +#include "netflow_structs.h" +#include "netflow_parsers.h" +#include "netflow9_templates.h" +#include "../message_ipfix.h" +#include "../verbose.h" + +/** ID of the first incompatible NetFlow Information Element */ +#define NF_INCOMP_ID_MIN 128 +/** IPFIX Enterprise Number for incompatible NetFlow IEs (128 <= ID <= 32767) */ +#define NF_INCOMP_EN_LOW 4294967294UL +/** IPFIX Enterprise Number for incompatible NetFlow IEs (32767 <= ID <= 65535) */ +#define NF_INCOMP_EN_HIGH 4294967295UL +/** + * Maximum length of any IPFIX Message Set + * \note + * The length represents maximum IPFIX Message size without IPFIX Message header and + * IPFIX Set header size + */ +#define MAX_SET_CONTENT_LEN \ + (UINT16_MAX - FDS_IPFIX_MSG_HDR_LEN - FDS_IPFIX_SET_HDR_LEN) + +// Simple static asserts to prevent unexpected structure modifications! +static_assert(IPX_NF9_MSG_HDR_LEN == 20U, "NetFlow v9 Message header size is not valid!"); +static_assert(IPX_NF9_SET_HDR_LEN == 4U, "NetFlow v9 Set header size is not valid!"); + +/** Auxiliary conversion structure from NetFlow Options Field to IPFIX Inf. Element ID */ +struct nf2ipx_opts { + /** (Source) NetFlow scope field type */ + uint16_t nf_id; + /** (Target) IPFIX Information Element ID */ + uint16_t ipx_id; + /** Maximal size of IPFIX IE */ + uint16_t ipx_max_size; +}; + +/** + * \brief Options conversion table + * + * Each records represents mapping from NetFlow Scope Field Type to IPFIX IE. + * \remark Based on RFC3954, Section 6.1. and available IPFIX IE from IANA + */ +static const struct nf2ipx_opts nf2ipx_opts_table[] = { + // "System" -> iana:exportingProcessId + {1U, 144U, 4U}, + // "Interface" -> iana:ingressInterface + {2U, 10U, 4U}, + // "Line Card" -> iana:lineCardId + {3U, 141U, 4U}, + // "Cache" -> ??? (maybe iana:meteringProcessId) + // {4, xxx, xxx}, + // "Template" -> iana:templateId + {5U, 145U, 2U} +}; + +/** Number of record the the Options conversion table */ +#define NF2IPX_OPTS_TABLE_SIZE (sizeof(nf2ipx_opts_table) / sizeof(nf2ipx_opts_table[0])) + +/** Auxiliary conversion structure from NetFlow Field to IPFIX Inf. Element ID */ +struct nf2ipx_data { + struct { + /** NetFlow Field identification */ + uint16_t id; + /** + * Required size of the field + * \note If the field doesn't have this size, conversion cannot be performed. + */ + uint16_t size; + } netflow; /**< NetFlow Field definition */ + struct { + /** New IPFIX IE ID */ + uint16_t id; + /** New IPFIX IE Enterprise Number */ + uint32_t en; + /** Size of converted field */ + uint16_t size; + } ipfix; /**< IPFIX Field Definition*/ + + /** Conversion instruction */ + struct nf2ipx_instr instr; +}; + +/** + * \brief NetFlow Data conversion table + * + * Each field represents mapping from NetFlow v9 to IPFIX IE and conversion instruction. + * Only incompatible fields that MUST be converted should go here. + */ +static const struct nf2ipx_data nf2ipx_data_table[] = { + // Conversion from relative to absolute TS: "LAST_SWITCHED" -> iana:flowEndMilliseconds + {{21U, 4U}, {153U, 0U, 8U}, {NF2IPX_ITYPE_TS, 8U}}, + // Conversion from relative to absolute TS: "FIRST_SWITCHED" -> iana:flowStartMilliseconds + {{22U, 4U}, {152U, 0U, 8U}, {NF2IPX_ITYPE_TS, 8U}} +}; + +/** Number of record the the Data conversion table */ +#define NF2IPX_DATA_TABLE_SIZE (sizeof(nf2ipx_data_table) / sizeof(nf2ipx_data_table[0])) + +/** Internal converter structure */ +struct ipx_nf9_conv { + /** Instance identification (only for log!) */ + char *ident; + /** Verbosity level */ + enum ipx_verb_level vlevel; + + /** Sequence number of the next expected NetFlow Message */ + uint32_t nf9_seq_next; + /** Have we already processed at least one NetFlow message */ + bool nf9_seq_valid; + /** Sequence number of the next converted IPFIX Message */ + uint32_t ipx_seq_next; + + struct { + /** Pointer to newly generated IPFIX Message */ + uint8_t *ipx_msg; + /** Pointer to the first byte after committed memory */ + uint8_t *write_ptr; + /** Allocated size of the IPFIX Message */ + size_t ipx_size_alloc; + /** Used (i.e. filled) size of the IPFIX Message */ + size_t ipx_size_used; + /** Message context (Source, ODID, Stream) */ + const struct ipx_msg_ctx *msg_ctx; + /** Number of all processed NetFlow records (Data + Templates) during conversion */ + uint16_t recs_processed; + /** Number of converted and added "Data" NetFlow records into the IPFIX Message */ + uint16_t drecs_converted; + } data; /**< Data of currently converted messages */ + + /** Template lookup table - 2-level table (256 x 256) */ + struct tmplts_l1_table l1_table; +}; + +/** + * \def CONV_ERROR + * \brief Macro for printing an error message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_ERROR(conv, fmt, ...) \ + if ((conv)->vlevel >= IPX_VERB_ERROR) { \ + const struct ipx_msg_ctx *msg_ctx = (conv)->data.msg_ctx; \ + ipx_verb_print(IPX_VERB_ERROR, "ERROR: %s: [%s, ODID: %" PRIu32 "] " fmt "\n", \ + (conv)->ident, msg_ctx->session->ident, msg_ctx->odid, ## __VA_ARGS__); \ + } + +/** + * \def CONV_WARNING + * \brief Macro for printing a warning message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_WARNING(conv, fmt, ...) \ + if ((conv)->vlevel >= IPX_VERB_WARNING) { \ + const struct ipx_msg_ctx *msg_ctx = (conv)->data.msg_ctx; \ + ipx_verb_print(IPX_VERB_WARNING, "WARNING: %s: [%s, ODID: %" PRIu32 "] " fmt "\n", \ + (conv)->ident, msg_ctx->session->ident, msg_ctx->odid, ## __VA_ARGS__); \ + } + +/** + * \def CONV_INFO + * \brief Macro for printing an info message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_INFO(conv, fmt, ...) \ + if ((conv)->vlevel >= IPX_VERB_INFO) { \ + const struct ipx_msg_ctx *msg_ctx = (conv)->data.msg_ctx; \ + ipx_verb_print(IPX_VERB_INFO, "INFO: %s: [%s, ODID: %" PRIu32 "] " fmt "\n", \ + (conv)->ident, msg_ctx->session->ident, msg_ctx->odid, ## __VA_ARGS__); \ + } + +/** + * \def CONV_DEBUG + * \brief Macro for printing a debug message of a converter + * \param[in] conv Converter + * \param[in] fmt Format string (see manual page for "printf" family) + * \param[in] ... Variable number of arguments for the format string + */ +#define CONV_DEBUG(conv, fmt, ...) \ + if ((conv)->vlevel >= IPX_VERB_DEBUG) { \ + const struct ipx_msg_ctx *msg_ctx = (conv)->data.msg_ctx; \ + ipx_verb_print(IPX_VERB_DEBUG, "DEBUG: %s: [%s, ODID: %" PRIu32 "] " fmt "\n", \ + (conv)->ident, msg_ctx->session->ident, msg_ctx->odid, ## __VA_ARGS__); \ + } + + +ipx_nf9_conv_t * +ipx_nf9_conv_init(const char *ident, enum ipx_verb_level vlevel) +{ + struct ipx_nf9_conv *res = calloc(1, sizeof(*res)); + if (!res) { + return NULL; + } + + nf9_tmplts_init(&res->l1_table); + res->ident = strdup(ident); + if (!res->ident) { + free(res); + return NULL; + } + + res->vlevel = vlevel; + return res; +} + +void +ipx_nf9_conv_destroy(struct ipx_nf9_conv *conv) +{ + // Destroy templates and lookup tables + nf9_tmplts_destroy(&conv->l1_table); + + // Destroy the main structure + free(conv->ident); + free(conv); +} + +/** + * \brief Initialize an internal memory buffer for a new IPFIX Message + * \param[in] conv Converter internals + */ +static inline void +conv_mem_init(ipx_nf9_conv_t *conv) +{ + conv->data.ipx_msg = NULL; + conv->data.write_ptr = NULL; + conv->data.ipx_size_used = 0; + conv->data.ipx_size_alloc = 0; +} + +/** + * \brief Destroy the internal memory buffer for a new IPFIX Message + * \param conv Converter internals + */ +static inline void +conv_mem_destroy(ipx_nf9_conv_t *conv) +{ + free(conv->data.ipx_msg); + conv->data.ipx_msg = NULL; +} + +/** + * \brief Release IPFIX Message from the + * \warning Do not use any buffer manipulation functions after call, except conv_mem_init() + * \param conv Converter internals + * \return Pointer to the IPFIX Message + */ +static inline uint8_t * +conv_mem_release(ipx_nf9_conv_t *conv) +{ + uint8_t *tmp = conv->data.ipx_msg; + conv->data.ipx_msg = NULL; + return tmp; +} + +/** + * \brief Reserve a memory in the new IPFIX Message + * + * The function makes sure that at least \p size bytes of the memory after the pointer + * (i.e. conv_mem_ptr_now()) will be prepared for user modification. + * + * \warning The buffer can be reallocated and a user MUST consider that all pointers + * to the IPFIX Message are not valid anymore! + * \param[in] conv Converter internals + * \param[in] size Required size (in bytes) + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM in case of memory allocation error + */ +static inline int +conv_mem_reserve(ipx_nf9_conv_t *conv, size_t size) +{ + if (conv->data.ipx_size_used + size <= conv->data.ipx_size_alloc) { + // Nothing to do + return IPX_OK; + } + + // Reallocate // TODO: improve size - multiples of 1024? 1024, 2048, 4096, ... + size_t new_alloc = conv->data.ipx_size_used + size; // TODO: during TEST allocate only to required size and use valgrind + uint8_t *new_msg = realloc(conv->data.ipx_msg, new_alloc * sizeof(uint8_t)); + if (!new_msg) { + return IPX_ERR_NOMEM; + } + + conv->data.ipx_msg = new_msg; + conv->data.write_ptr = new_msg + conv->data.ipx_size_used; + conv->data.ipx_size_alloc = new_alloc; + return IPX_OK; +} + +/** + * \brief Commit a user defined part of the new IPFIX Message + * + * User defined size of the memory after the current position pointer (i.e. conv_mem_ptr_now()) will + * be marked as filled and the pointer will be moved forward. + * \param[in] conv Converter internals + * \param[in] size Memory size to commit + */ +static inline void +conv_mem_commit(ipx_nf9_conv_t *conv, size_t size) +{ + conv->data.ipx_size_used += size; + conv->data.write_ptr = conv->data.ipx_msg + conv->data.ipx_size_used; + assert(conv->data.ipx_size_used <= conv->data.ipx_size_alloc); +} + +/** + * \brief Get a pointer into the new IPFIX Message after the last committed byte + * + * \warning + * Keep on mind, the returned pointer MUST be considered as invalid after calling + * conv_mem_reserve() function. There is a chance that the returned address doesn't point + * to the IPFIX Message anymore! + * \param[in] conv Converter internals + * \return Pointer + */ +void * +conv_mem_ptr_now(ipx_nf9_conv_t *conv) +{ + return (void *) (conv->data.write_ptr); +} + +/** + * \brief Get a pointer into the new IPFIX Message with a given offset + * + * \warning + * Keep on mind, the returned pointer MUST be considered as invalid after calling + * conv_mem_reserve() function. There is a chance that the returned address doesn't point + * to the IPFIX Message anymore! + * \warning + * Accessing offsets after reserved memory is undefined! + * \param[in] conv Converter internals + * \param[in] offset Offset from the start of the Message + * \return Pointer + */ +void * +conv_mem_ptr_offset(ipx_nf9_conv_t *conv, size_t offset) +{ + return (void *) (conv->data.ipx_msg + offset); +} + +/** + * \brief Get offset of the "commit" pointer (represents memory area labeled as filled) + * \param[in] conv Converter internals + * \return Offset from the start of the Message + */ +static inline size_t +conv_mem_pos_get(ipx_nf9_conv_t *conv) +{ + return conv->data.ipx_size_used; +} + +/** + * \brief Move the "commit" pointer back to the specific place in the IPFIX Message + * + * Memory after a user given \p offset from the beginning of IPFIX Message is considered as + * unspecified (not committed). User can move the pointer only back in the memory, to make sure + * that there are no undefined regions. Typical usage is to "remove" already committed IPFIX + * Message parts. + * + * \warning Setting position after reserved memory is undefined operation! + * \param[in] conv Converter internals + * \param[in] offset Offset from the start of the Message + */ +static inline void +conv_mem_pos_set(ipx_nf9_conv_t *conv, size_t offset) +{ + conv->data.ipx_size_used = offset; + conv->data.write_ptr = conv->data.ipx_msg + conv->data.ipx_size_used; + assert(conv->data.ipx_size_used <= conv->data.ipx_size_alloc); +} + +// ----------------------------------------------------------------------------------------- + +/** + * \brief Get a conversion instruction for conversion from NetFlow to IPFIX Data Field + * \param[in] nf_id NetFlow Field ID + * \return Pointer to the instruction or NULL (no conversion required) + */ +static inline const struct nf2ipx_data * +conv_data_map(uint16_t nf_id) +{ + for (size_t i = 0; i < NF2IPX_DATA_TABLE_SIZE; ++i) { + // Is there a mapping to different IPFIX Element? + const struct nf2ipx_data *map_rec = &nf2ipx_data_table[i]; + if (map_rec->netflow.id != nf_id) { + continue; + } + + // Success + return map_rec; + } + + return NULL; +} + +/** + * \brief Get a conversion instruction for conversion from NetFlow to IPFIX Scope Field ID + * \param[in] nf_id NetFlow Scope Field ID + * \return Pointer to the instruction or NULL (unsupported scope field) + */ +static inline const struct nf2ipx_opts * +conv_opts_map(uint16_t nf_id) +{ + for (size_t i = 0; i < NF2IPX_OPTS_TABLE_SIZE; ++i) { + const struct nf2ipx_opts *rec = &nf2ipx_opts_table[i]; + if (rec->nf_id != nf_id) { + continue; + } + // Found! + return rec; + } + + return NULL; // Not found +} + +/** + * \brief Append an IPFIX Template to the new IPFIX Message + * + * Place the template right after the last committed byte and commit the size of the added template. + * \warning The IPFIX Message might be reallocated! + * \param[in] conv Converter internals + * \param[in] trec Template record to append + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM in case of memory allocation error + */ +static inline int +conv_tmplt_append(ipx_nf9_conv_t *conv, const struct nf9_trec *trec) +{ + size_t size2cpy = trec->ipx_size; + if (conv_mem_reserve(conv, size2cpy) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + assert(size2cpy != 0); + void *mem_ptr = conv_mem_ptr_now(conv); + memcpy(mem_ptr, trec->ipx_data, size2cpy); + + // Commit the modifications + conv_mem_commit(conv, size2cpy); + return IPX_OK; +} + +/** + * \brief Add IPFIX (Options) Template from the Template table + * + * Try to find the NetFlow template and compare it with a previously parsed Template with the same + * ID. If the templates are the same, append its IPFIX Template counterpart to the IPFIX Message. + * + * \warning The IPFIX Message might be reallocated! + * \param[in] conv Converter internals + * \param[in] it Template Set iterator with a template to convert + * \param[in] type Template type (::IPX_NF9_SET_TMPLT or ::IPX_NF9_SET_OPTS_TMPLT) + * \return #IPX_OK on success and the IPFIX Template has been added to the Set. + * \return #IPX_ERR_NOTFOUND if the template is unknown or doesn't match previously defined one. + * \return #IPX_ERR_DENIED if the templates match but for format compatibility reasons its + * conversion is disabled (i.e. no template is added to the (Options) Template Set) + * \return #IPX_ERR_NOMEM in case of memory allocation error + */ +static inline int +conv_tmplt_from_table(ipx_nf9_conv_t *conv, const struct ipx_nf9_tset_iter *it, uint16_t type) +{ + uint16_t tid = ntohs(it->ptr.trec->template_id); // TID is always on the same place + struct nf9_trec *trec = nf9_tmplts_find(&conv->l1_table, tid); + if (!trec || trec->type != type) { + // Template not found + return IPX_ERR_NOTFOUND; + } + + if (trec->nf9_size != it->size || memcmp(trec->nf9_data, it->ptr.trec, it->size) != 0) { + // Templates doesn't match + return IPX_ERR_NOTFOUND; + } + + if (trec->action == REC_ACT_DROP) { + // The template and its data records cannot be converter due to format incompatibilities + return IPX_ERR_DENIED; + } + + return conv_tmplt_append(conv, trec); +} + +/** Auxiliary structure for template conversion */ +struct conv_tmplt_aux { + /** New converter template */ + struct nf9_trec *tmplt; + /** Template Set iterator with the current template to process */ + const struct ipx_nf9_tset_iter *it; + + /** Pointer to the start of NetFlow field definitions to process */ + const struct ipx_nf9_tmplt_ie *ie_nf9_ptr; + /** Pointer to the start of IPFIX field definitions to add */ + fds_ipfix_tmplt_ie *ie_ipx_ptr; +}; + +/** + * \brief Convert a NetFlow Template header to an IPFIX Template header (aux. function) + * + * Pointers to the template field definitions to be process by conv_tmplt_process_field() are set + * appropriately. + * \param[in] aux Auxiliary conversion structure + * \param[in] fset_id FlowSet ID (::IPX_NF9_SET_TMPLT or ::IPX_NF9_SET_OPTS_TMPLT) + */ +static inline void +conv_tmplt_process_hdr(struct conv_tmplt_aux *aux, uint16_t fset_id) +{ + if (fset_id == IPX_NF9_SET_TMPLT) { + // Template header + const struct ipx_nf9_trec *nf9_trec = aux->it->ptr.trec; + struct fds_ipfix_trec *ipx_ptr = (struct fds_ipfix_trec *) aux->tmplt->ipx_data; + ipx_ptr->template_id = nf9_trec->template_id; + ipx_ptr->count = nf9_trec->count; + + aux->ie_ipx_ptr = &ipx_ptr->fields[0]; + aux->ie_nf9_ptr = &nf9_trec->fields[0]; + aux->tmplt->ipx_size = (uint8_t *) aux->ie_ipx_ptr - (uint8_t *) ipx_ptr; // Size of added data + return; + } + + // Options Template header + assert(fset_id == IPX_NF9_SET_OPTS_TMPLT); + const struct ipx_nf9_opts_trec *nf9_trec = aux->it->ptr.opts_trec; + struct fds_ipfix_opts_trec *ipx_ptr = (struct fds_ipfix_opts_trec *) aux->tmplt->ipx_data; + ipx_ptr->template_id = nf9_trec->template_id; + ipx_ptr->scope_field_count = htons(aux->it->scope_cnt); + ipx_ptr->count = htons(aux->it->field_cnt); + + aux->ie_ipx_ptr = &ipx_ptr->fields[0]; + aux->ie_nf9_ptr = &nf9_trec->fields[0]; + aux->tmplt->ipx_size = (uint8_t *) aux->ie_ipx_ptr - (uint8_t *) ipx_ptr; // Size of added data +} + +/** + * \brief Convert a NetFlow (Options) Template into an IPFIX (Options) Template (aux. function) + * + * The (Options) Template header is converted to the new format, followed by conversion of + * template field definitions. In case Options Scope fields, a lookup function is used to + * find new IPFIX IE definitions instead of incompatible NetFlow enumeration. + * + * \warning + * The Template conversion structure (aux->tmplt) can be reallocated during conversion! + * Pointers to its records must be considered as invalid after calling this function! + * \param[in] conv Converter internals + * \param[in] aux Auxiliary conversion parameters (the Template, converter internals, etc.) + * \param[in] fset_id FlowSet ID (::IPX_NF9_SET_TMPLT or ::IPX_NF9_SET_OPTS_TMPLT) + * \return #IPX_OK on success and the template is successfully converted to IPFIX + * \return #IPX_ERR_DENIED IPFIX if the Template cannot be converted due to format incompatibility + * \return #IPX_ERR_NOMEM in case of memory allocation error + */ +static inline int +conv_tmplt_process(ipx_nf9_conv_t *conv, struct conv_tmplt_aux *aux, uint16_t fset_id) +{ + // Parse header and prepare field pointers + conv_tmplt_process_hdr(aux, fset_id); + + fds_ipfix_tmplt_ie *ipx_ie_ptr = aux->ie_ipx_ptr; + uint16_t tid = ntohs(aux->it->ptr.trec->template_id); + + uint16_t fields_total = aux->it->field_cnt; + uint16_t fields_scope = aux->it->scope_cnt; + size_t cpy_size = 0; // Size (in bytes) to copy without conversion of data record + size_t nf9_drec_len = 0; + size_t ipx_drec_len = 0; + + // Calculate size of NetFlow Data record (we need to know it even if conversion fails) + for (size_t i = 0; i < fields_total; ++i) { + nf9_drec_len += ntohs(aux->ie_nf9_ptr[i].length); + } + assert(nf9_drec_len < UINT16_MAX); + aux->tmplt->nf9_drec_len = nf9_drec_len; + + if (aux->tmplt->type == IPX_NF9_SET_OPTS_TMPLT && fields_scope == 0) { + // IPFIX prohibits Options Template without Scope fields + CONV_WARNING(conv, "Unable to convert an Options Template (ID: %" PRIu16 ") from NetFlow " + "to IPFIX due to missing Options Fields. Data records of this template will be dropped!", + tid); + return IPX_ERR_DENIED; + } + + for (size_t i = 0; i < fields_total; ++i) { + const struct ipx_nf9_tmplt_ie nf_ie_def = aux->ie_nf9_ptr[i]; + uint16_t ie_id = ntohs(nf_ie_def.id); + uint16_t ie_size = ntohs(nf_ie_def.length); + + if (i < fields_scope) { + // Process Scope field -> try to find mapping of Scope identifiers + const struct nf2ipx_opts *opts_map = conv_opts_map(ie_id); + if (!opts_map || opts_map->ipx_max_size < ie_size) { + CONV_WARNING(conv, "Unable to convert an Options Template (ID %" PRIu16 ") from " + "NetFlow to IPFIX due to unknown Scope Field conversion. Options records of " + "this template will be dropped!", tid); + return IPX_ERR_DENIED; + } + + ipx_ie_ptr->ie.id = htons(opts_map->ipx_id); + ipx_ie_ptr->ie.length = nf_ie_def.length; + ipx_ie_ptr++; + + cpy_size += ie_size; + ipx_drec_len += ie_size; + continue; + } + + // Non-scope field -> check if the field should be remapped to different IPFIX field + const struct nf2ipx_data *data_map = conv_data_map(ie_id); + if (data_map == NULL) { + // Conversion doesn't exist + cpy_size += ie_size; + ipx_drec_len += ie_size; + + if (ie_id < NF_INCOMP_ID_MIN) { + ipx_ie_ptr->ie.id = nf_ie_def.id; + ipx_ie_ptr->ie.length = nf_ie_def.length; + ipx_ie_ptr++; + continue; + } + + // Incompatible fields -> add Enterprise Number + ipx_ie_ptr->ie.id = htons(ie_id | (uint16_t) 0x8000); // The highest bit must be set! + ipx_ie_ptr->ie.length = nf_ie_def.length; + ipx_ie_ptr++; + + uint32_t en_id = ((ie_id & 0x8000) == 0) ? NF_INCOMP_EN_LOW : NF_INCOMP_EN_HIGH; + ipx_ie_ptr->enterprise_number = htonl(en_id); + ipx_ie_ptr++; + continue; + } + + // ---- Conversion exists ---- + if (data_map->netflow.size != ie_size) { + CONV_WARNING(conv, "Conversion from NetFlow (Field ID %" PRIu16 ") to IPFIX (EN: " + "%" PRIu32 ", ID %" PRIu16 ") cannot be performed due to unexpected NetFlow field " + "size. Template ID %" PRIu16 " and its data records will be ignored!", + ie_id, data_map->ipfix.en, data_map->ipfix.id, tid); + return IPX_ERR_DENIED; + } + ipx_drec_len += data_map->ipfix.size; + + if (cpy_size > 0) { + // Add an instruction to copy X bytes of an original data record before conversion + struct nf2ipx_instr instr = {.itype = NF2IPX_ITYPE_CPY, .size = cpy_size}; + if (nf9_trec_instr_add(&aux->tmplt, instr) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + cpy_size = 0; + } + + // Add an instruction to convert data field + if (nf9_trec_instr_add(&aux->tmplt, data_map->instr) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + // Add IPFIX Field ID to the new template + uint16_t new_id = data_map->ipfix.id; + if (data_map->ipfix.en != 0) { + new_id |= (uint16_t) 0x8000; // Set the highest bit + } + + ipx_ie_ptr->ie.id = htons(new_id); + ipx_ie_ptr->ie.length = htons(data_map->ipfix.size); + ipx_ie_ptr++; + + if (data_map->ipfix.en == 0) { + continue; + } + + // Add Enterprise Number + ipx_ie_ptr->enterprise_number = htonl(data_map->ipfix.en); + ipx_ie_ptr++; + continue; + } + + if (cpy_size > 0) { + // Add an instruction to copy the rest of the original message + struct nf2ipx_instr instr = {.itype = NF2IPX_ITYPE_CPY, .size = cpy_size}; + if (nf9_trec_instr_add(&aux->tmplt, instr) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + } + + // Update template parameters + size_t ipfix_fields_size = (uint8_t *) ipx_ie_ptr - (uint8_t *) aux->ie_ipx_ptr; + aux->tmplt->ipx_size += ipfix_fields_size; // Add size of added fields + aux->tmplt->ipx_drec_len = ipx_drec_len; + + if (ipx_drec_len > (size_t) (UINT16_MAX - FDS_IPFIX_MSG_HDR_LEN - FDS_IPFIX_SET_HDR_LEN)) { + CONV_WARNING(conv, "Unable to convert an (Options) Template (ID %" PRIu16 ") from NetFlow " + "to IPFIX. Size of a single Data record exceeds the maximum size of an IPFIX message. " + "Records of this template will be dropped!", tid); + return IPX_ERR_DENIED; + } + + return IPX_OK; +} + +/** + * \brief Parse and add IPFIX (Options) Template from a NetFlow (Options) Template record + * + * The function tries to parse and convert a NetFlow (Options) Template record to IPFIX (Options) + * Template record and append it to the new IPFIX Message. Once the Template is parsed, it is + * also added into the internal Template table of already parsed templates and the next time + * the template must be converted, it is possible to find it using conv_tmplt_from_table(). + * + * \param[in] conv Converter internals + * \param[in] it Template iterator (points to Template to be parsed) + * \param[in] fset_id Template type (::IPX_NF9_SET_TMPLT or ::IPX_NF9_SET_OPTS_TMPLT) + * \return #IPX_OK on success + * \return #IPX_ERR_DENIED if the IPFIX Template cannot be added due to format incompatibility. + * However, a dummy record that signalizes unsupported conversion is added into the Template table. + * \return #IPX_ERR_NOMEM in case of a memory allocation error + */ +static inline int +conv_tmplt_from_data(ipx_nf9_conv_t *conv, const struct ipx_nf9_tset_iter *it, uint16_t fset_id) +{ + assert(fset_id == IPX_NF9_SET_TMPLT || fset_id == IPX_NF9_SET_OPTS_TMPLT); + uint16_t tid = ntohs(it->ptr.trec->template_id); + + // Prepare memory for a new IPFIX Template (the worst possible case) and copy of NetFlow tmplt. + size_t max_size = 6U + (8U * it->field_cnt); // 6 = Opts. Template. header, 8 = IE ID + EN + struct nf9_trec *template = nf9_trec_new(it->size, max_size); + if (!template) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + // Copy the NetFlow Template + memcpy(template->nf9_data, it->ptr.trec, it->size); + template->nf9_size = it->size; + template->action = REC_ACT_CONVERT; + template->type = fset_id; + + // Process the (Options) Template header and fields + struct conv_tmplt_aux aux; + aux.tmplt = template; // Warning: template could be reallocated during conversion! + aux.it = it; + int rc = conv_tmplt_process(conv, &aux, fset_id); + template = aux.tmplt; // Template might have been reallocated during processing! + + if (rc == IPX_ERR_NOMEM) { + // Memory allocation failed + nf9_trec_destroy(template); + return IPX_ERR_NOMEM; + } else if (rc == IPX_ERR_DENIED) { + // Unable to convert the Options Template -> "drop" it and its Data record + free(template->ipx_data); + template->action = REC_ACT_DROP; + template->ipx_data = NULL; + template->ipx_size = 0; + template->ipx_drec_len = 0; + + if (nf9_tmplts_insert(&conv->l1_table, tid, template) != IPX_OK) { + // Failed to insert the "dummy drop" template + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + nf9_trec_destroy(template); + return IPX_ERR_NOMEM; + } + + return IPX_ERR_DENIED; + } + + // Insert the template to the internal table + if (nf9_tmplts_insert(&conv->l1_table, tid, template) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + nf9_trec_destroy(template); + return IPX_ERR_NOMEM; + } + + // Copy the template to the new message + return conv_tmplt_append(conv, template); +} + +/** + * \brief Convert NetFlow (Options) Template FlowSet to IPFIX (Options) Template Set + * + * The function creates and appends a new IPFIX (Options) Template Set with converted + * (Options) Template records to the new IPFIX Message. For each NetFlow (Options) Template + * record a conversion function is called and the new IPFIX Template record is stored for future + * using during parsing and Data record conversions. + * \param[in] conv Converter internals + * \param[in] flowset_hdr NetFlow (Options) Template Set to be converted + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM in case of a memory allocation error + * \return #IPX_ERR_FORMAT if the NetFlow (Options) Template FlowSet is malformed and cannot be + * converted. + */ +int +conv_process_tset(ipx_nf9_conv_t *conv, const struct ipx_nf9_set_hdr *flowset_hdr) +{ + uint16_t nf_fsid = ntohs(flowset_hdr->flowset_id); + assert(nf_fsid == IPX_NF9_SET_TMPLT || nf_fsid == IPX_NF9_SET_OPTS_TMPLT); + + // Add (Options) Template Set header + if (conv_mem_reserve(conv, FDS_IPFIX_SET_HDR_LEN) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + // Get current offset and commit Template Set header (size and ID will be filled later) + size_t hdr_offset = conv_mem_pos_get(conv); + conv_mem_commit(conv, FDS_IPFIX_SET_HDR_LEN); // Size and ID will be filled later + + // Convert all (Options) Templates + uint16_t tmplt_added = 0; + uint16_t tmplt_processed = 0; + + struct ipx_nf9_tset_iter it; + ipx_nf9_tset_iter_init(&it, flowset_hdr); + int rc_conv = IPX_OK; + int rc_iter; + + while (rc_conv == IPX_OK && (rc_iter = ipx_nf9_tset_iter_next(&it)) == IPX_OK) { + tmplt_processed++; + + uint16_t tid = ntohs(it.ptr.trec->template_id); // TID position is the same for both types + CONV_DEBUG(conv, "Processing a definition of %s ID %" PRIu16 "...", + (nf_fsid == IPX_NF9_SET_TMPLT) ? "Template" : "Options Template", tid); + + // Check if the template has been already processed + int rval = conv_tmplt_from_table(conv, &it, nf_fsid); + switch (rval) { + case IPX_OK: // IPFIX Template has been added + tmplt_added++; + CONV_INFO(conv, "A definition of the (Options) Template ID %" PRIu16 " has been " + "converted.", tid); + continue; + case IPX_ERR_DENIED: // IPFIX Template cannot be added due to format incompatibility + CONV_INFO(conv, "A definition of the (Options) Template ID %" PRIu16 " has been " + "dropped due to format incompatibility (see a previous warning for more details).", + tid); + continue; + case IPX_ERR_NOMEM: // Memory allocation error + rc_conv = IPX_ERR_NOMEM; + continue; + case IPX_ERR_NOTFOUND: // Not found -> process the template + break; + default: + assert(false); // Unexpected return code + break; + } + + // Convert the NetFlow template to IPFIX and add it to the template manager + rval = conv_tmplt_from_data(conv, &it, nf_fsid); + switch (rval) { + case IPX_OK: + tmplt_added++; + CONV_INFO(conv, "A definition of the (Options) Template ID %" PRIu16 " has been " + "converted.", tid); + continue; + case IPX_ERR_DENIED: // IPFIX Template cannot be added due to format incompatibility + continue; + case IPX_ERR_NOMEM: // Memory allocation error + rc_conv = IPX_ERR_NOMEM; + continue; + default: + assert(false); // Unexpected return code + break; + } + } + + if (rc_conv != IPX_OK) { + return rc_conv; + } + + if (rc_iter != IPX_EOC) { + // Iterator failed! + CONV_ERROR(conv, "%s", ipx_nf9_tset_iter_err(&it)); + return rc_iter; + } + + // Update number of processed records + conv->data.recs_processed += tmplt_processed; + + if (tmplt_added == 0) { + // No template has been added -> "uncommit" Template Set header + CONV_DEBUG(conv, "Converted (Options) Template Set is empty! Removing its Template Set " + "header.", '\0'); + conv_mem_pos_set(conv, hdr_offset); + return IPX_OK; + } + + // Update IPFIX (Options) Template Set header + size_t set_size = conv_mem_pos_get(conv) - hdr_offset; + if (set_size > MAX_SET_CONTENT_LEN) { + CONV_ERROR(conv, "Unable to convert NetFlow v9 (Options) Template Set (Flow Set ID: " + "%" PRIu16 ") to IPFIX due to exceeding maximum content size.", nf_fsid); + return IPX_ERR_FORMAT; + } + + struct fds_ipfix_tset *hdr_ptr = conv_mem_ptr_offset(conv, hdr_offset); + hdr_ptr->header.flowset_id = (nf_fsid == IPX_NF9_SET_TMPLT) + ? htons(FDS_IPFIX_SET_TMPLT) : htons(FDS_IPFIX_SET_OPTS_TMPLT); + hdr_ptr->header.length = htons((uint16_t) set_size); // Cast is safe, value has been checked + return IPX_OK; +} + +/** + * \brief Convert (NetFlow) relative timestamp to absolute timestamp + * \param[in] hdr NetFlow Message header (required for an exporter timestamps) + * \param[in] nf_ts Relative NetFlow timestamp stored in a Data record (in Network byte order) + * \return Converted absolute timestamp (Unix timestamp in milliseconds) (in Host byte order) + */ +static inline uint64_t +conv_ts_rel2abs(const struct ipx_nf9_msg_hdr *hdr, const uint32_t *nf_ts) +{ + const uint64_t hdr_exp = ntohl(hdr->unix_sec) * 1000U; + const uint64_t hdr_sys = ntohl(hdr->sys_uptime); + const uint64_t rec_sys = ntohl(*nf_ts); + return (hdr_exp - hdr_sys) + rec_sys; +} + +/** + * \brief Convert a NetFlow data record to IPFIX Data record + * + * The function executes instructions described in the internal Template record to perform Data + * record conversion. After conversion the IPFIX Data record is appended to the new IPFIX Message. + * \param[in] conv Converter internals + * \param[in] nf9_msg NetFlow Message header (necessary for timestamp conversion) + * \param[in] nf9_rec NetFlow Data record to convert + * \param[in] tmplt Internal template record with conversion instructions + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM in case of a memory allocation error + */ +static inline int +conv_process_drec(ipx_nf9_conv_t *conv, const struct ipx_nf9_msg_hdr *nf9_msg, + const uint8_t *nf9_rec, const struct nf9_trec *tmplt) +{ + assert(tmplt->action == REC_ACT_CONVERT); + assert(tmplt->instr_size > 0); + + // Reserve enough memory for converted IPFIX record + if (conv_mem_reserve(conv, tmplt->ipx_drec_len) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + // Execute conversion instructions + const uint8_t *nf9_pos = nf9_rec; + uint8_t *ipx_pos = conv_mem_ptr_now(conv); + size_t offset_start = conv_mem_pos_get(conv); + + for (size_t i = 0; i < tmplt->instr_size; ++i) { + const struct nf2ipx_instr *instr = &tmplt->instr_data[i]; + switch (instr->itype) { + case NF2IPX_ITYPE_CPY: + // Just copy memory + memcpy(ipx_pos, nf9_pos, instr->size); + nf9_pos += instr->size; + ipx_pos += instr->size; + break; + case NF2IPX_ITYPE_TS: + // Convert relative timestamp to absolute timestamp + *((uint64_t *) ipx_pos) = conv_ts_rel2abs(nf9_msg, (uint32_t *) nf9_pos); + nf9_pos += 4U; // NetFlow TS (FIRST_SWITCHED/LAST_SWITCHED) + ipx_pos += 8U; // IPFIX TS (iana:flowStartMilliseconds/flowEndMilliseconds) + break; + default: + CONV_ERROR(conv, "(internal) Invalid NetFlow-to-IPFIX conversion instruction", '\0'); + return IPX_ERR_NOMEM; // This will start component termination + } + } + + // Commit written memory + assert(tmplt->ipx_drec_len == (conv_mem_pos_get(conv) - offset_start)); + conv_mem_commit(conv, tmplt->ipx_drec_len); + return IPX_OK; +} + +/** + * \brief Convert NetFlow Data FlowSet to IPFIX Data Set + * + * The function creates and appends a new IPFIX Data Set (with converted Data records) to the new + * IPFIX Message. For each NetFlow Data record a conversion function is called. + * \param[in] conv Converter internals + * \param[in] flowset_hdr NetFlow Data FlowSet to be converted + * \param[in] nf9_hdr NetFlow Message header (necessary for timestamp conversions) + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM in case of a memory allocation error + * \return #IPX_ERR_FORMAT if the NetFlow Data FlowSet is malformed and cannot be converted. + */ +int +conv_process_dset(ipx_nf9_conv_t *conv, const struct ipx_nf9_set_hdr *flowset_hdr, + const struct ipx_nf9_msg_hdr *nf9_hdr) +{ + uint16_t tid = ntohs(flowset_hdr->flowset_id); + assert(tid >= IPX_NF9_SET_MIN_DSET); + + // Try to find a template in the manager + const struct nf9_trec *tmplt = nf9_tmplts_find(&conv->l1_table, tid); + if (!tmplt) { + // Template NOT found! + CONV_WARNING(conv, "Unable to convert NetFlow v9 Data Set (FlowSet ID: " PRIu16 ") to " + "IPFIX due to missing NetFlow template. The Data FlowSet and its records will be " + "dropped!", tid); + return IPX_OK; + } + + if (tmplt->action == REC_ACT_DROP) { + // Data Set will be ignored! + CONV_DEBUG(conv, "Unable to convert NetFlow v9 Data Set (FlowSet ID: %" PRIu16 ") to " + "IPFIX. Its template wasn't properly converted from NetFlow to IPFIX.", tid); + // However, we still have to calculate number of records in the Set + uint16_t data_size = ntohs(flowset_hdr->length) - IPX_NF9_SET_HDR_LEN; + uint16_t rec_cnt = data_size / tmplt->nf9_drec_len; + conv->data.recs_processed += rec_cnt; + return IPX_OK; + } + + // Add Data Set header (parameters will be filled later) + if (conv_mem_reserve(conv, FDS_IPFIX_SET_HDR_LEN) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + + size_t hdr_offset = conv_mem_pos_get(conv); + conv_mem_commit(conv, FDS_IPFIX_SET_HDR_LEN); + + // Convert all records in the Template Set + uint16_t rec_processed = 0; + struct ipx_nf9_dset_iter it; + ipx_nf9_dset_iter_init(&it, flowset_hdr, tmplt->nf9_drec_len); + int rc_conv = IPX_OK; + int rc_iter; + + while (rc_conv == IPX_OK && (rc_iter = ipx_nf9_dset_iter_next(&it)) == IPX_OK) { + rec_processed++; + rc_conv = conv_process_drec(conv, nf9_hdr, it.rec, tmplt); + } + + if (rc_conv != IPX_OK) { + // Converter failed (a proper error message has been already printed) + return rc_conv; + } + + if (rc_iter != IPX_EOC) { + CONV_ERROR(conv, "%s", ipx_nf9_dset_iter_err(&it)); + return rc_iter; + } + + // Update number of processed records + conv->data.recs_processed += rec_processed; + conv->data.drecs_converted += rec_processed; + + // Update IPFIX Data Set header + size_t set_size = conv_mem_pos_get(conv) - hdr_offset; + if (set_size > MAX_SET_CONTENT_LEN) { + CONV_ERROR(conv, "Unable to convert NetFlow v9 Data Set (FlowSet ID: %" PRIu16 ") to " + "IPFIX due to exceeding maximum content size.", tid); + return IPX_ERR_FORMAT; + } + + struct fds_ipfix_dset *hdr_ptr = conv_mem_ptr_offset(conv, hdr_offset); + hdr_ptr->header.flowset_id = flowset_hdr->flowset_id; + hdr_ptr->header.length = htons((uint16_t) set_size); // Case is safe, value has been checked + return IPX_OK; +} + +/** + * \brief Convert a NetFlow Message to an IPFIX Message + * + * For each FlowSet in the NetFlow Message is called a particular converter to equivalent IPFIX Set. + * The converted message is stored to the internal IPFIX Message buffer, however, the header of + * the IPFIX Message header is not filled. + * + * \param[in] conv Converter internals + * \param[in] nf9_msg NetFlow v9 Message data + * \param[in] nf9_size NetFlow v9 Message size (in bytes) + * \return #IPX_OK on success + * \return #IPX_ERR_FORMAT if the NetFlow Message is malformed + * \return #IPX_ERR_NOMEM in case of a memory allocation error + */ +int +conv_process_msg(ipx_nf9_conv_t *conv, const struct ipx_nf9_msg_hdr *nf9_msg, uint16_t nf9_size) +{ + /* Prepare a new IPFIX message + * Typical NetFlow messages consist of data records and we usually modify only timestamps + * (conversion from relative to absolute value), therefore, we should just add (2 x 4B) per + * record. However, keep in mind that templates (with non-compatible field, i.e. ID > 127) can + * be also part of the message and we need to change their template definition (i.e. add + * Enterprise Numbers). + */ + size_t new_size = (size_t) nf9_size + (8U * ntohs(nf9_msg->count)); // TODO: try only header size during tests... + if (conv_mem_reserve(conv, new_size) != IPX_OK) { + CONV_ERROR(conv, "A memory allocation failed (%s:%d).", __FILE__, __LINE__); + return IPX_ERR_NOMEM; + } + // Add/commit IPFIX Message header (it will be filled later when we know all parameters) + conv_mem_commit(conv, FDS_IPFIX_MSG_HDR_LEN); + + // Iterate over all NetFlow FlowSets and convert them to IPFIX Sets + struct ipx_nf9_sets_iter it; + ipx_nf9_sets_iter_init(&it, nf9_msg, nf9_size); + + int rc_conv = IPX_OK; + int rc_iter; + while (rc_conv == IPX_OK && (rc_iter = ipx_nf9_sets_iter_next(&it)) == IPX_OK) { + uint16_t flowset_id = ntohs(it.set->flowset_id); + + // Try to convert the FlowSet + if (flowset_id >= IPX_NF9_SET_MIN_DSET) { + // Data FlowSet + rc_conv = conv_process_dset(conv, it.set, nf9_msg); + } else if (flowset_id == IPX_NF9_SET_TMPLT || flowset_id == IPX_NF9_SET_OPTS_TMPLT) { + // (Options) Template FlowSet + rc_conv = conv_process_tset(conv, it.set); + } else { + // Unknown FlowSet ID + CONV_ERROR(conv, "Unknown FlowSet ID %" PRIu16, flowset_id); + rc_conv = IPX_ERR_FORMAT; + } + } + + if (rc_conv != IPX_OK) { + // Converter failed (a proper error message has been already printed) + return rc_conv; + } + + if (rc_iter != IPX_EOC) { + // Iterator failed! + CONV_ERROR(conv, "%s", ipx_nf9_sets_iter_err(&it)); + return rc_iter; + } + + + return IPX_OK; +} + +/** + * \brief Compare sequence numbers (with wraparound support) + * \param t1 First number + * \param t2 Second number + * \return The function returns an integer less than, equal to, or greater than zero if the + * first number \p t1 is found, respectively, to be less than, to match, or be greater than + * the second number. + */ +static inline int +conv_seq_num_cmp(uint32_t t1, uint32_t t2) +{ + if (t1 == t2) { + return 0; + } + + if ((t1 - t2) & 0x80000000) { // test the "sign" bit + return (-1); + } else { + return 1; + } +} + +int +ipx_nf9_conv_process(ipx_nf9_conv_t *conv, ipx_msg_ipfix_t *wrapper) +{ + int rc; + const struct ipx_nf9_msg_hdr *nf9_hdr = (const struct ipx_nf9_msg_hdr *) wrapper->raw_pkt; + const uint16_t nf9_size = wrapper->raw_size; + + // Initialize conversion internals + conv_mem_init(conv); + conv->data.msg_ctx = &wrapper->ctx; + conv->data.recs_processed = 0; // Will be updated by conversion functions + conv->data.drecs_converted = 0; + + // Check the Message header + if (nf9_size < IPX_NF9_MSG_HDR_LEN) { + CONV_ERROR(conv, "Length of a NetFlow (v9) Message is smaller than its header size.", '\0'); + conv_mem_destroy(conv); + return IPX_ERR_FORMAT; + } + + if (ntohs(nf9_hdr->version) != IPX_NF9_VERSION) { + CONV_ERROR(conv, "Invalid version number of a NetFlow Message (expected 9)", '\0'); + conv_mem_destroy(conv); + return IPX_ERR_FORMAT; + } + + // Check Sequence number + uint32_t msg_seq = ntohl(nf9_hdr->seq_number); + CONV_DEBUG(conv, "Converting a NetFlow Message v9 (seq. num. %" PRIu32 ") to an IPFIX Message " + "(new seq. num. %" PRIu32 ")", msg_seq, conv->ipx_seq_next); + + bool old_oos = false; // Old (out of sequence message) + if (conv->nf9_seq_next != msg_seq) { + if (!conv->nf9_seq_valid) { + // The first message to convert + conv->nf9_seq_valid = true; + conv->nf9_seq_next = msg_seq + 1; + } else { + // Out of sequence message + CONV_WARNING(conv, "Unexpected Sequence number (expected: %" PRIu32 ", got: %" PRId32 ")", + conv->nf9_seq_next, msg_seq); + if (conv_seq_num_cmp(msg_seq, conv->nf9_seq_next) > 0) { + conv->nf9_seq_next = msg_seq + 1; // Newer than expected + } else { + old_oos = true; // Older than expected + } + } + } + conv->nf9_seq_valid = true; + + // Update expected Sequence number of the next NetFlow message + if (!old_oos) { + ++conv->nf9_seq_next; + } + + // Convert the message + rc = conv_process_msg(conv, nf9_hdr, nf9_size); + if (rc != IPX_OK) { + // Something went wrong! + conv_mem_destroy(conv); + return rc; + } + + // Check number of record in the message + if (conv->data.recs_processed != ntohs(nf9_hdr->count)) { // TODO: improve in case of ignored templates + CONV_WARNING(conv, "Number of records in NetFlow v9 Message header doesn't match number of " + "records found in the Message (expected: %" PRIu16 ", found: %" PRIu16 ")", + ntohs(nf9_hdr->count), conv->data.recs_processed); + } + + // Fill the new IPFIX Message header + size_t ipx_size = conv_mem_pos_get(conv); + if (ipx_size > UINT16_MAX) { + CONV_ERROR(conv, "Unable to convert NetFlow v9 to IPFIX. Size of the converted message " + "exceeds the maximum size of an IPFIX Message! (before: %" PRIu16 " B, after: %zu B)", + nf9_size, ipx_size); + conv_mem_destroy(conv); + return IPX_ERR_FORMAT; + } + + struct fds_ipfix_msg_hdr *ipx_ptr = conv_mem_ptr_offset(conv, 0); + ipx_ptr->version = htons(FDS_IPFIX_VERSION); + ipx_ptr->length = htons((uint16_t) ipx_size); // Cast is safe, size has been checked! + ipx_ptr->export_time = nf9_hdr->unix_sec; + ipx_ptr->seq_num = htonl(conv->ipx_seq_next); + ipx_ptr->odid = nf9_hdr->source_id; + // Update sequence number of the new IPFIX Message + conv->ipx_seq_next += conv->data.drecs_converted; + + // Finally, replace the converted NetFlow Message with the new IPFIX Message + free(wrapper->raw_pkt); + wrapper->raw_pkt = conv_mem_release(conv); + wrapper->raw_size = (uint16_t) ipx_size; + return IPX_OK; +} diff --git a/src/core/netflow2ipfix/netflow9_templates.c b/src/core/netflow2ipfix/netflow9_templates.c new file mode 100644 index 00000000..e463d29f --- /dev/null +++ b/src/core/netflow2ipfix/netflow9_templates.c @@ -0,0 +1,187 @@ +/** + * \file src/core/netflow2ipfix/netflow9_templates.c + * \author Lukas Hutak + * \brief Auxiliary template management for NetFlow v9 to IPFIX conversion (source file) + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include // offsetof +#include + +#include +#include "netflow9_templates.h" + +/** Default number of pre-allocated instructions */ +#define NF9_TREC_DEF_INSTR 8U + + +void +nf9_tmplts_init(struct tmplts_l1_table *tbl) +{ + // Set all pointers to NULL + memset(tbl, 0, sizeof(*tbl)); +} + +void +nf9_tmplts_destroy(struct tmplts_l1_table *tbl) +{ + // Destroy templates and lookup tables + for (size_t l1_idx = 0; l1_idx < TMPLTS_TABLE_SIZE; ++l1_idx) { + struct tmplts_l2_table *l2_table = tbl->l2_tables[l1_idx]; + if (!l2_table) { + continue; + } + + for (size_t l2_idx = 0; l2_idx < TMPLTS_TABLE_SIZE; ++l2_idx) { + struct nf9_trec *rec = l2_table->recs[l2_idx]; + if (!rec) { + continue; + } + + // Destroy the record + nf9_trec_destroy(rec); + } + + // Destroy the L2 table + free(l2_table); + } +} + +struct nf9_trec * +nf9_tmplts_find(struct tmplts_l1_table *tbl, uint16_t id) +{ + const uint16_t l1_idx = id / TMPLTS_TABLE_SIZE; + struct tmplts_l2_table *l2_table = tbl->l2_tables[l1_idx]; + if (!l2_table) { + return NULL; + } + + const uint16_t l2_idx = id % TMPLTS_TABLE_SIZE; + struct nf9_trec *rec = l2_table->recs[l2_idx]; + return rec; +} + +int +nf9_tmplts_insert(struct tmplts_l1_table *tbl, uint16_t id, struct nf9_trec *rec) +{ + const uint16_t l1_idx = id / TMPLTS_TABLE_SIZE; + struct tmplts_l2_table *l2_table = tbl->l2_tables[l1_idx]; + if (!l2_table) { + // Create a new L2 table + l2_table = calloc(1, sizeof(*l2_table)); + if (!l2_table) { + return IPX_ERR_NOMEM; + } + + tbl->l2_tables[l1_idx] = l2_table; + } + + // Remove the old record, if exists + const uint16_t l2_idx = id % TMPLTS_TABLE_SIZE; + struct nf9_trec *old_rec = l2_table->recs[l2_idx]; + if (old_rec != NULL && old_rec != rec) { + nf9_trec_destroy(old_rec); + } + + l2_table->recs[l2_idx] = rec; + return IPX_OK; +} + +/** + * \brief Calculate size of Template structure with user defined number of instructions + * \param[in] instr_cnt Instruction count + * \return Size of the Template structure + */ +static inline size_t +nf9_trec_size(size_t instr_cnt) +{ + return offsetof(struct nf9_trec, instr_data) + (instr_cnt * sizeof(struct nf2ipx_instr)); +} + +struct nf9_trec * +nf9_trec_new(size_t nf_size, size_t ipx_size) +{ + struct nf9_trec *res = calloc(1, nf9_trec_size(NF9_TREC_DEF_INSTR)); + if (!res) { + // Memory allocation error + return NULL; + } + + res->nf9_data = malloc(nf_size * sizeof(uint8_t)); + res->ipx_data = malloc(ipx_size * sizeof(uint8_t)); + if (res->nf9_data == NULL || res->ipx_data == NULL) { + // Memory allocation error + free(res->ipx_data); + free(res->nf9_data); + free(res); + return NULL; + } + + res->instr_alloc = NF9_TREC_DEF_INSTR; + res->instr_size = 0; + return res; +} + +void +nf9_trec_destroy(struct nf9_trec *rec) +{ + free(rec->nf9_data); + free(rec->ipx_data); + free(rec); +} + +int +nf9_trec_instr_add(struct nf9_trec **ptr, struct nf2ipx_instr instr) +{ + struct nf9_trec *rec = *ptr; + if (rec->instr_size == rec->instr_alloc) { + // Reallocate the structure + size_t new_cnt = 2 * rec->instr_size; + struct nf9_trec *new_rec = realloc(rec, nf9_trec_size(new_cnt)); + if (!new_rec) { + return IPX_ERR_NOMEM; + } + + new_rec->instr_alloc = new_cnt; + rec = new_rec; + *ptr = new_rec; + } + + rec->instr_data[rec->instr_size++] = instr; + return IPX_OK; +} \ No newline at end of file diff --git a/src/core/netflow2ipfix/netflow9_templates.h b/src/core/netflow2ipfix/netflow9_templates.h new file mode 100644 index 00000000..6ec1c280 --- /dev/null +++ b/src/core/netflow2ipfix/netflow9_templates.h @@ -0,0 +1,228 @@ +/** + * \file src/core/netflow2ipfix/netflow9_templates.h + * \author Lukas Hutak + * \brief Auxiliary template management for NetFlow v9 to IPFIX conversion (header file) + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef IPFIXCOL2_NETFLOW9_TEMPLATES_H +#define IPFIXCOL2_NETFLOW9_TEMPLATES_H + +#include +#include + +/** Data conversion instructions */ +enum NF2IPX_ITYPE { + /** + * Copy memory + * \note Copy specified \p size from the original NetFlow record to the new IPFIX record. + */ + NF2IPX_ITYPE_CPY, + /** + * Convert relative timestamp to absolute timestamp + * \note Value of \p size is ignored, the converted timestamp is always 8 bytes long. + */ + NF2IPX_ITYPE_TS +}; + +/** Data conversion instructions with parameters */ +struct nf2ipx_instr { + /** Instruction type */ + enum NF2IPX_ITYPE itype; + /** Size of used memory after conversion (in bytes) */ + size_t size; +}; + +/** + * \brief Template record action + */ +enum REC_ACTION { + /** + * \brief Perform conversion of the record + */ + REC_ACT_CONVERT, + /** + * \brief Drop the Data record + * + * Typical usage is in case of Options Template records, where the Scope is not valid + * (for example, failed to convert Scope ID to IPFIX IE, missing scope, etc.) + */ + REC_ACT_DROP +}; + +/** + * \brief Template record + * + * This structure represents NetFlow-to-IPFIX template conversion. It consists of the original + * NetFlow Template, a new converted IPFIX (Options) Template and Data Record conversion + * instructions. + */ +struct nf9_trec { + /** Conversion action */ + enum REC_ACTION action; + /** Template type (::IPX_NF9_SET_TMPLT or ::IPX_NF9_SET_OPTS_TMPLT) */ + uint16_t type; + + /** Copy of the original NetFlow (Options) Template */ + uint8_t *nf9_data; + /** Size of the original NetFlow (Options) Template */ + uint16_t nf9_size; + /** Data record length of an original (a.k.a NetFlow v9) record described by the NF Template */ + uint16_t nf9_drec_len; + + /** + * Copy of the new IPFIX (Options) Template + * \note If the action is ::REC_ACT_DROP, the pointer is always NULL! + */ + uint8_t *ipx_data; + /** + * Size of the new IPFIX (Options) Template + * \note If the action is ::REC_ACT_DROP, the size is always 0! + */ + uint16_t ipx_size; + /** + * Data record length of a converter (a.k.a. IPFIX) record described by the IPFIX Template + * \note If the action is ::REC_ACT_DROP, the size is always 0! + */ + uint16_t ipx_drec_len; + + // --- Note: following fields are filled automatically --- + /** Number of pre-allocated instructions */ + size_t instr_alloc; + /** Number of valid instructions */ + size_t instr_size; + /** + * Conversion instruction + * \warning This field MUST be the last in the structure! + */ + struct nf2ipx_instr instr_data[1]; +}; + +/** Size of L1 and L2 template lookup table */ +#define TMPLTS_TABLE_SIZE 256 + +/** L1 template table */ +struct tmplts_l1_table { + /** + * Array of L2 tables (sparse array) + * Keep in mind that if there are no templates in a L2 table, the L2 table doesn't exist + * (i.e. its value is NULL) + */ + struct tmplts_l2_table *l2_tables[TMPLTS_TABLE_SIZE]; +}; + +/** L2 template table */ +struct tmplts_l2_table { + /** + * Array of template records (sparse array) + * Keep in mind that if a template is missing in the array, the pointer is NULL. + */ + struct nf9_trec *recs[TMPLTS_TABLE_SIZE]; +}; + + +/** + * \brief Initialize template lookup structure + * \param[in] tbl L1 template table + */ +void +nf9_tmplts_init(struct tmplts_l1_table *tbl); + +/** + * \brief Destroy template lookup structure and destroy all templates + * \param[in] tbl L1 template table + */ +void +nf9_tmplts_destroy(struct tmplts_l1_table *tbl); + +/** + * \brief Find a template definition in the template lookup table + * \param[in] tbl L1 template table + * \param[in] id Template ID + * \return Pointer to the template or NULL (not found) + */ +struct nf9_trec * +nf9_tmplts_find(struct tmplts_l1_table *tbl, uint16_t id); + +/** + * \brief Insert a template record into Template lookup table + * + * \note If the template already exists, it will be automatically freed + * \param[in] tbl L1 template table + * \param[in] id Template ID + * \param[in] rec Template record to add + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM in case of memory allocation error + */ +int +nf9_tmplts_insert(struct tmplts_l1_table *tbl, uint16_t id, struct nf9_trec *rec); + + +/** + * \brief Create a new Template structure + * + * All fields in the structure are initialized by calloc() and the structure is prepared for + * up to ::NF9_TREC_DEF_INSTR instructions. + * \param[in] nf_size Memory allocated for raw NetFlow (Options) Template (in bytes) + * \param[in] ipx_size Memory allocated for raw IPFIX (Options) Template (in bytes) + * \return Pointer to the structure or NULL (memory allocation error) + */ +struct nf9_trec * +nf9_trec_new(size_t nf_size, size_t ipx_size); + +/** + * \brief Destroy a Template structure + * \note NetFlow and IPFIX Templates will be destroyed, if defined. + * \param[in] rec Template structure + */ +void +nf9_trec_destroy(struct nf9_trec *rec); + +/** + * \brief Add a conversion instruction to a Template record + * + * \warning Due to reallocation, pointer to the template can be changed! + * \param[in] ptr Pointer to a pointer to the Template record + * \param[in] instr Instruction to add + * \return #IPX_OK on success + * \return #IPX_ERR_NOMEM in case of memory allocation error + */ +int +nf9_trec_instr_add(struct nf9_trec **ptr, struct nf2ipx_instr instr); + +#endif //IPFIXCOL2_NETFLOW9_TEMPLATES_H diff --git a/src/core/netflow2ipfix/netflow_parsers.c b/src/core/netflow2ipfix/netflow_parsers.c new file mode 100644 index 00000000..0a42e5be --- /dev/null +++ b/src/core/netflow2ipfix/netflow_parsers.c @@ -0,0 +1,340 @@ +/** + * \file src/core/netflow2ipfix/netflow_parsers.c + * \author Lukas Hutak + * \brief NetFlow v9 parsers (source file) + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include "netflow_parsers.h" + +/** Internal error codes */ +enum error_codes { + // No error + ERR_OK, + // FlowSets iterator + ERR_SETS_UEND, + ERR_SETS_SHORT, + ERR_SETS_LONG, + // Data FlowSet iterator + ERR_DSET_EMPTY, + // (Options) Template FlowSet iterator + ERR_TSET_EMPTY, + ERR_TSET_SOPTS, + ERR_TSET_SFIELD, + ERR_TSET_TID, + ERR_TSET_CNT, + ERR_TSET_END, + ERR_TSET_ZERO, + ERR_TSET_DATA +}; + +/** Error messages */ +static const char *err_msg[] = { + [ERR_OK] = "No error.", + // FlowSets iterator + [ERR_SETS_UEND] = "The NetFlow v9 Message size is invalid (unexpected end of the message).", + [ERR_SETS_SHORT] = "Total length of the FlowSet is shorter than a length of a NetFlow v9 " + "FlowSet header.", + [ERR_SETS_LONG] = "Total length of the FlowSet is longer than its enclosing NetFlow v9 " + "Message.", + // Data FlowSet iterator + [ERR_DSET_EMPTY] = "A DataFlow Set must not be empty. At least one record must be present.", + // (Options) Template FlowSet iterator + [ERR_TSET_EMPTY] = "An (Options) Template FlowSet must not be empty. At least one record must " + "be present.", + [ERR_TSET_SOPTS] = "Invalid Options Scope length in an Options Template definition.", + [ERR_TSET_SFIELD] = "Invalid Option Length in an Options Template definition.", + [ERR_TSET_TID] = "Template ID of an (Options) Template is invalid (< 256).", + [ERR_TSET_CNT] = "An (Options) Template without field definitions is not valid.", + [ERR_TSET_END] = "Invalid definition of an (Options) Template (unexpected end of the " + "(Options) Template FlowSet).", + [ERR_TSET_ZERO] = "An (Options) Template defines a prohibited zero length Data Record.", + [ERR_TSET_DATA] = "An (Options) Template defines a Data Record which length exceeds " + "the maximum length of a Data Record that fits into a NetFlow v9 Message.", +}; + + +void +ipx_nf9_sets_iter_init(struct ipx_nf9_sets_iter *it, const struct ipx_nf9_msg_hdr *nf9_msg, + uint16_t nf9_size) +{ + it->_private.set_next = ((const uint8_t *) nf9_msg) + IPX_NF9_MSG_HDR_LEN; + it->_private.msg_end = ((const uint8_t *) nf9_msg) + nf9_size; + assert(it->_private.set_next <= it->_private.msg_end); + it->_private.err_msg = err_msg[ERR_OK]; +} + +int +ipx_nf9_sets_iter_next(struct ipx_nf9_sets_iter *it) +{ + if (it->_private.set_next == it->_private.msg_end) { + return IPX_EOC; + } + + // Make sure that the iterator is not beyond the end of the message + assert(it->_private.set_next < it->_private.msg_end); + + // Check the remaining length of the NetFlow message + if (it->_private.set_next + IPX_NF9_SET_HDR_LEN > it->_private.msg_end) { + // Unexpected end of the NetFlow message + it->_private.err_msg = err_msg[ERR_SETS_UEND]; + return IPX_ERR_FORMAT; + } + + const struct ipx_nf9_set_hdr *new_set = (const struct ipx_nf9_set_hdr *) it->_private.set_next; + uint16_t set_len = ntohs(new_set->length); + + // Check the length of the FlowSet + if (set_len < IPX_NF9_SET_HDR_LEN) { + // Length of a FlowSet is shorted that an NetFlow FlowSet header + it->_private.err_msg = err_msg[ERR_SETS_SHORT]; + return IPX_ERR_FORMAT; + } + + if (it->_private.set_next + set_len > it->_private.msg_end) { + // Length of the FlowSet is longer than its enclosing NetFlow Message + it->_private.err_msg = err_msg[ERR_SETS_LONG]; + return IPX_ERR_FORMAT; + } + + it->_private.set_next += set_len; + it->set = new_set; + return IPX_OK; +} + +const char * +ipx_nf9_sets_iter_err(const struct ipx_nf9_sets_iter *it) +{ + return it->_private.err_msg; +} + +// ------------------------------------------------------------------------------------------------- + +/** \brief Internal iterator flags */ +enum ipx_nf9_dset_iter_flags { + /** Initialization fail and an error message is set */ + IPX_NF9_DSET_ITER_FAILED = (1 << 0) +}; + +void +ipx_nf9_dset_iter_init(struct ipx_nf9_dset_iter *it, const struct ipx_nf9_set_hdr *set, + uint16_t rec_size) +{ + const uint16_t set_id = ntohs(set->flowset_id); + const uint16_t set_len = ntohs(set->length); + assert(set_id >= IPX_NF9_SET_MIN_DSET); + assert(set_len >= IPX_NF9_SET_HDR_LEN); + + it->_private.flags = 0; + it->_private.rec_size = rec_size; + it->_private.rec_next = ((const uint8_t *) set) + IPX_NF9_SET_HDR_LEN; + it->_private.set_end = ((const uint8_t *) set) + set_len; + it->_private.err_msg = err_msg[ERR_OK]; + + if (it->_private.rec_next + rec_size > it->_private.set_end) { + // Empty set is not valid (see RFC 3954, Section 2, FlowSet) + it->_private.flags |= IPX_NF9_DSET_ITER_FAILED; + it->_private.err_msg = err_msg[ERR_DSET_EMPTY]; + } +} + +int +ipx_nf9_dset_iter_next(struct ipx_nf9_dset_iter *it) +{ + if ((it->_private.flags & IPX_NF9_DSET_ITER_FAILED) != 0) { + // Initialization failed, error code is properly set + return IPX_ERR_FORMAT; + } + + if (it->_private.rec_next + it->_private.rec_size > it->_private.set_end) { + // The iterator reached the end of the Data Set or the rest is padding + return IPX_EOC; + } + + it->rec = it->_private.rec_next; + it->_private.rec_next += it->_private.rec_size; + return IPX_OK; +} + +const char * +ipx_nf9_dset_iter_err(const struct ipx_nf9_dset_iter *it) +{ + return it->_private.err_msg; +} + +// ------------------------------------------------------------------------------------------------- + +/** \brief Internal iterator flags */ +enum ipx_nf9_tset_iter_flags { + /** Initialization fail and an error message is set */ + IPX_NF9_TSET_ITER_FAILED = (1 << 0) +}; + +void +ipx_nf9_tset_iter_init(struct ipx_nf9_tset_iter *it, const struct ipx_nf9_set_hdr *set) +{ + const uint16_t set_id = ntohs(set->flowset_id); + const uint16_t set_len = ntohs(set->length); + + assert(set_id == IPX_NF9_SET_TMPLT || set_id == IPX_NF9_SET_OPTS_TMPLT); + assert(set_len >= IPX_NF9_SET_HDR_LEN); + + it->_private.type = set_id; + it->_private.flags = 0; + it->_private.rec_next = ((const uint8_t *) set) + IPX_NF9_SET_HDR_LEN; + it->_private.set_end = ((const uint8_t *) set) + set_len; + it->_private.err_msg = err_msg[ERR_OK]; + + // Minimal size of a definition is a template with just one field (4B + 4B, resp. 6B + 4B) + const uint16_t min_size = (set_id == IPX_NF9_SET_TMPLT) ? 8U : 10U; + if (it->_private.rec_next + min_size > it->_private.set_end) { + // Empty (Options) Template FlowSet is not valid (see RFC 3954, Section 2, Template FlowSet + it->_private.flags |= IPX_NF9_TSET_ITER_FAILED; + it->_private.err_msg = err_msg[ERR_TSET_EMPTY]; + return; + } +} + +int +ipx_nf9_tset_iter_next(struct ipx_nf9_tset_iter *it) +{ + if ((it->_private.flags & IPX_NF9_TSET_ITER_FAILED) != 0) { + // Initialization failed, error code is properly set + return IPX_ERR_FORMAT; + } + + // Minimal size of a definition is a template with just one field (4B + 4B, resp. 6B + 4B) + const uint16_t min_size = (it->_private.type == IPX_NF9_SET_TMPLT) ? 8U : 10U; + if (it->_private.rec_next + min_size > it->_private.set_end) { + // End of the FlowSet or padding + return IPX_EOC; + } + + uint16_t tmplt_id; + uint16_t field_cnt; + uint16_t scope_cnt = 0; + uint32_t data_size = 0; + const struct ipx_nf9_tmplt_ie *field_ptr; + + if (it->_private.type == IPX_NF9_SET_TMPLT) { + // Template Record + const struct ipx_nf9_trec *rec = (const struct ipx_nf9_trec *) it->_private.rec_next; + tmplt_id = ntohs(rec->template_id); + field_cnt = ntohs(rec->count); + field_ptr = &rec->fields[0]; + } else { + // Options Template Record + const struct ipx_nf9_opts_trec *rec = (struct ipx_nf9_opts_trec *) it->_private.rec_next; + tmplt_id = ntohs(rec->template_id); + uint16_t size_opts = ntohs(rec->scope_length); + uint16_t size_fields = ntohs(rec->option_length); + + if (size_opts % IPX_NF9_TMPLT_IE_LEN != 0) { + it->_private.err_msg = err_msg[ERR_TSET_SOPTS]; + return IPX_ERR_FORMAT; + } + + if (size_fields % IPX_NF9_TMPLT_IE_LEN != 0) { + it->_private.err_msg = err_msg[ERR_TSET_SFIELD]; + return IPX_ERR_FORMAT; + } + + scope_cnt = size_opts / IPX_NF9_TMPLT_IE_LEN; + field_cnt = scope_cnt + (size_fields / IPX_NF9_TMPLT_IE_LEN); + field_ptr = &rec->fields[0]; + } + + if (tmplt_id < IPX_NF9_SET_MIN_DSET) { + // Invalid Template ID + it->_private.err_msg = err_msg[ERR_TSET_TID]; + return IPX_ERR_FORMAT; + } + + if (field_cnt == 0) { + // The template is empty! + it->_private.err_msg = err_msg[ERR_TSET_CNT]; + return IPX_ERR_FORMAT; + } + + const uint8_t *tmptl_end = (const uint8_t *) (field_ptr + field_cnt); + if (tmptl_end > it->_private.set_end) { + // Unexpected end of the Set + it->_private.err_msg = err_msg[ERR_TSET_END]; + return IPX_ERR_FORMAT; + } + + // Calculate size of the corresponding Data record + for (size_t i = 0; i < field_cnt; ++i, ++field_ptr) { + data_size += ntohs(field_ptr->length); + } + + if (data_size == 0) { + // Template definition that describes zero-length records doesn't make sense + it->_private.err_msg = err_msg[ERR_TSET_ZERO]; + return IPX_ERR_FORMAT; + } + + // Maximum size of a data record (Max. Msg. length - NetFlow Msg. header - NetFlow Set header) + const size_t data_max = UINT16_MAX - IPX_NF9_MSG_HDR_LEN - IPX_NF9_SET_HDR_LEN; + if (data_size > data_max) { + it->_private.err_msg = err_msg[ERR_TSET_DATA]; + return IPX_ERR_FORMAT; + } + + // Everything seems ok... + if (it->_private.type == IPX_NF9_SET_TMPLT) { + it->ptr.trec = (const struct ipx_nf9_trec *) it->_private.rec_next; + } else { + it->ptr.opts_trec = (const struct ipx_nf9_opts_trec *) it->_private.rec_next; + } + + const uint16_t tmplt_size = (uint16_t) (tmptl_end - it->_private.rec_next); + it->size = tmplt_size; + it->field_cnt = field_cnt; + it->scope_cnt = scope_cnt; + it->_private.rec_next += tmplt_size; + return IPX_OK; +} + +const char * +ipx_nf9_tset_iter_err(const struct ipx_nf9_tset_iter *it) +{ + return it->_private.err_msg; +} diff --git a/src/core/netflow2ipfix/netflow_parsers.h b/src/core/netflow2ipfix/netflow_parsers.h new file mode 100644 index 00000000..889ec859 --- /dev/null +++ b/src/core/netflow2ipfix/netflow_parsers.h @@ -0,0 +1,308 @@ +/** + * \file src/core/netflow2ipfix/netflow_parsers.h + * \author Lukas Hutak + * \brief NetFlow v9 parsers (header file) + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef IPFIXCOL2_NETFLOW_PARSERS_H +#define IPFIXCOL2_NETFLOW_PARSERS_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include "netflow_structs.h" + +/** Iterator over NetFlow v9 Sets in an NetFlow v9 Message */ +struct ipx_nf9_sets_iter { + /** Pointer to the current FlowSet header */ + const struct ipx_nf9_set_hdr *set; + + struct { + /** Pointer to the start of the next FlowSet */ + const uint8_t *set_next; + /** First byte after the end of the message */ + const uint8_t *msg_end; + /** Error buffer */ + const char *err_msg; + } _private; /**< Internal structure (dO NOT use directly!) */ +}; + +/** + * \brief Initialize NetFlow v9 FlowSet iterator + * \warning + * After initialization the iterator has initialized only internal structures but public part + * is still undefined i.e. doesn't point to the first FlowSet in the Message. To get the first + * field see ipx_nf9_sets_iter_next(). + * \warning + * The NetFlow Message header is not checked. Make sure that its length is at + * least >= ::IPX_NF9_MSG_HDR_LEN. + * \param[in] it Uninitialized iterator structure + * \param[in] nf9_msg NetFlow v9 Message to iterate through + * \param[in] nf9_size NetFlow v9 Message size + */ +void +ipx_nf9_sets_iter_init(struct ipx_nf9_sets_iter *it, const struct ipx_nf9_msg_hdr *nf9_msg, + uint16_t nf9_size); + +/** + * \brief Get the next FlowSet in the Message + * + * Move the iterator to the next FlowSet in the order, If this function was not previously called + * after initialization by ipx_nf9_sets_iter_init(), then the iterator will point to the first + * FlowSet in the Message. + * + * \code{.c} + * struct ipx_nf9_msg_hdr *nf9_msg = ...; + * struct ipx_nf9_sets_iter it; + * ipx_nf9_sets_iter_init(&it, nf9_msg); + * + * int rc; + * while ((rc = ipx_nf9_sets_iter_next(&it)) == IPX_OK) { + * // Add your code here... + * } + * + * if (rc != IPX_EOC) { + * fprintf(stderr, "Error: %s\n", ipx_nf9_sets_iter_err(&it)); + * } + * \endcode + * \param[in] it Pointer to the iterator + * \return #IPX_OK on success an the next Set is ready to use. + * \return #IPX_EOC if no more Sets are available. + * \return #IPX_ERR_FORMAT if the format of the message is invalid (an appropriate error message + * is set - see ipx_nf9_sets_iter_err()). + */ +int +ipx_nf9_sets_iter_next(struct ipx_nf9_sets_iter *it); + +/** + * \brief Get the last error message + * \note The message is statically allocated string that can be passed to other function even + * when the iterator doesn't exist anymore. + * \param[in] it Iterator + * \return The error message. + */ +const char * +ipx_nf9_sets_iter_err(const struct ipx_nf9_sets_iter *it); + +// ------------------------------------------------------------------------------------------------ + +/** Iterator over Data Records in an Netflow v9 Data FlowSet */ +struct ipx_nf9_dset_iter { + /** Start of the data record */ + const uint8_t *rec; + + struct { + /** Iterator flags */ + uint16_t flags; + /** Size of the data record (in bytes) */ + uint16_t rec_size; + /** Pointer to the start of the next record */ + const uint8_t *rec_next; + /** First byte after end of the data set */ + const uint8_t *set_end; + /** Error buffer */ + const char *err_msg; + } _private; /**< Internal structure (dO NOT use directly!) */ +}; + +/** + * \brief Initialize NetFlow v9 Data records iterator + * + * The function takes the size of a data record that corresponds to the particular (Options) + * Template. It only up to user to manager Templates and track size of data records. + * + * \warning + * After initialization the iterator has initialized only internal structures but public part + * is still undefined i.e. doesn't point to the first Record in the Data Set. To get the first + * field see ipx_nf9_dset_iter_next(). + * \warning + * Make sure that the length of allocated memory of the Set is at least the same as the length + * from the Set header before using the iterator. Not required if you got the FlowSet from + * ipx_nf9_sets_iter_next(). + * \param[in] it Uninitialized structure + * \param[in] set Data FlowSet header to iterate through + * \param[in] rec_size Size of data record + */ +void +ipx_nf9_dset_iter_init(struct ipx_nf9_dset_iter *it, const struct ipx_nf9_set_hdr *set, + uint16_t rec_size); + +/** + * \brief Get the next Data Record in the Data Set + * + * Move the iterator to the next Record in the order, If this function was not previously called + * after initialization by ipx_nf9_dset_iter_init(), then the iterator will point to the first + * Record in the Data Set. + * + * \code{.c} + * struct ipx_nf9_set_hdr *data_set = ...; + * uint16_t rec_size = ... + * struct ipx_nf9_dset_iter it; + * ipx_nf9_dset_iter_init(&it, data_set, rec_size); + * + * int rc; + * while ((rc = ipx_nf9_dset_iter_next(&it)) == IPX_OK) { + * // Add your code here... + * } + * + * if (rc != IPX_EOC) { + * fprintf(stderr, "Error: %s\n", ipx_nf9_dset_iter_err(&it)); + * } + * \endcode + * \param[in] it Pointer to the iterator + * \return #IPX_OK on success and the next Record is ready to use. + * \return #IPX_EOC if no more Records are available (the end of the Set has been reached). + * \return #IPX_ERR_FORMAT if the format of the Data FlowSet is invalid (an appropriate error + * message is set - see ipx_nf9_dset_iter_err()). + */ +int +ipx_nf9_dset_iter_next(struct ipx_nf9_dset_iter *it); + +/** + * \brief Get the last error message + * \note The message is statically allocated string that can be passed to other function even + * when the iterator doesn't exist anymore. + * \param[in] it Iterator + * \return The error message + */ +const char * +ipx_nf9_dset_iter_err(const struct ipx_nf9_dset_iter *it); + +// ------------------------------------------------------------------------------------------------ + +/** Iterator over template records in an NetFlow v9 (Options) Template Set */ +struct ipx_nf9_tset_iter { + union { + /** Start of the Template Record (scope_cnt == 0) */ + const struct ipx_nf9_trec *trec; + /** Start of the Options Template Record (scope_cnt >= 0) */ + const struct ipx_nf9_opts_trec *opts_trec; + } ptr; /**< Pointer to the start of the template record */ + /** Size of the template record (in bytes) */ + uint16_t size; + /** Total field count (scope + non-scope field) */ + uint16_t field_cnt; + /** + * Scope field count + * \warning Unlike IPFIX, the value can be zero even if it is Options Template. + */ + uint16_t scope_cnt; + + struct { + /** Type of templates */ + uint16_t type; + /** Iterator flags */ + uint16_t flags; + /** Pointer to the start of the next record */ + const uint8_t *rec_next; + /** First byte after end of the data set */ + const uint8_t *set_end; + /** Error buffer */ + const char *err_msg; + } _private; /**< Internal structure (do NOT use directly!) */ +}; + +/** + * \brief Initialize NetFlow v9 (Options) Template records iterator + * + * \note + * FlowSet ID of the \p set MUST be 0 (::IPX_NF9_SET_TMPLT) or 1 (::IPX_NF9_SET_OPTS_TMPLT). + * Otherwise behavior of the parser is undefined. + * \warning + * After initialization the iterator has initialized only internal structures but public part + * is still undefined i.e. doesn't point to the first Record in the FlowSet. To get the + * first field see ipx_nf9_tset_iter_next(). + * \warning + * Make sure that the length of allocated memory of the Set is at least the same as the length + * from the Set header before using the iterator. Not required if you got the FlowSet from + * ipx_nf9_sets_iter_next(). + * \param[in] it Uninitialized structure + * \param[in] set (Options) Template FlowSet header to iterate through + */ +void +ipx_nf9_tset_iter_init(struct ipx_nf9_tset_iter *it, const struct ipx_nf9_set_hdr *set); + +/** + * \brief Get the next (Options) Template Record in the Data FlowSet + * + * Move the iterator to the next Record in the order, If this function was not previously called + * after initialization by ipx_nf9_dset_iter_init(), then the iterator will point to the first + * Record in the (Options) Template FlowSet. + * + * \code{.c} + * struct ipx_nf9_set_hdr *tmplt_set = ...; + * struct ipx_nf9_tset_iter it; + * ipx_nf9_tset_iter_init(&it, tmplt_set); + * + * int rc; + * while ((rc = ipx_nf9_tset_iter_next(&it)) == IPX_OK) { + * // Add your code here... + * } + * + * if (rc != IPX_EOC) { + * fprintf(stderr, "Error: %s\n", ipx_nf9_tset_iter_err(&it)); + * } + * \endcode + * \param[in] it Pointer to the iterator + * \return #IPX_OK on success and the Template Record is ready to use. + * \return #IPX_EOC if no more Template Records are available (the end of the Set has been reached). + * \return #IPX_ERR_FORMAT if the format of the Template FlowSet is invalid (an appropriate error + * message is set - see ipx_nf9_tset_iter_err()). + */ +int +ipx_nf9_tset_iter_next(struct ipx_nf9_tset_iter *it); + +/** + * \brief Get the last error message + * \note The message is statically allocated string that can be passed to other function even + * when the iterator doesn't exist anymore. + * \param[in] it Iterator + * \return The error message + */ +const char * +ipx_nf9_tset_iter_err(const struct ipx_nf9_tset_iter *it); + +#ifdef __cplusplus +} +#endif + +#endif // IPFIXCOL2_NETFLOW_PARSERS_H diff --git a/src/core/netflow2ipfix/netflow_structs.h b/src/core/netflow2ipfix/netflow_structs.h new file mode 100644 index 00000000..105eacf5 --- /dev/null +++ b/src/core/netflow2ipfix/netflow_structs.h @@ -0,0 +1,348 @@ +/** + * \file src/core/netflow2ipfix/netflow_structs.h + * \author Lukas Hutak + * \brief NetFlow v5/v9 structures + * \date 2018 + */ + +/* + * Copyright (C) 2018 CESNET, z.s.p.o. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided ``as is``, and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef IPFIXCOL2_NETFLOW_H +#define IPFIXCOL2_NETFLOW_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/** + * \brief NetFlow v5 Packet Header structure + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf5_hdr { + /** NetFlow export format version number */ + uint16_t version; + /** Number of flows exported in this packet (1 - 30) */ + uint16_t count; + /** Current time in milliseconds since the export device booted */ + uint32_t sys_uptime; + /** Current count of seconds since 0000 UTC 1970 */ + uint32_t unix_sec; + /** Residual nanoseconds since 0000 UTC 1970 */ + uint32_t unix_nsec; + /** Sequence counter of total flows seen */ + uint32_t flow_seq; + /** Type of flow-switching engine */ + uint8_t engine_type; + /** Slot number of the flow-switching engine */ + uint8_t engine_id; + /** First two bits hold the sampling mode. + * Remaining 14 bits hold value of sampling interval */ + uint16_t sampling_interval; +}; + +/** + * \brief NetFlow v5 Record structure + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf5_rec { + /** Source IPv4 address */ + uint32_t addr_src; + /** Destination IPv4 address */ + uint32_t addr_dst; + /** IPv4 address of next hop router */ + uint32_t nexthop; + /** SNMP index of input interface */ + uint16_t snmp_input; + /** SNMP index of output interface */ + uint16_t snmp_output; + /** Packets in the flow */ + uint32_t delta_pkts; + /** Total number of Layer 3 bytes in the packets of the flow */ + uint32_t delta_octets; + /** SysUptime at start of flow */ + uint32_t ts_first; + /** SysUptime at the time the last packet of the flow was received */ + uint32_t ts_last; + /** TCP/UDP source port number or equivalent */ + uint16_t port_src; + /** TCP/UDP destination port number or equivalent */ + uint16_t port_dst; + /** Unused (zero) bytes */ + uint8_t _pad1; + /** Cumulative OR of TCP flags */ + uint8_t tcp_flags; + /** IP protocol type (for example, TCP = 6; UDP = 17) */ + uint8_t proto; + /** IP type of service (ToS) */ + uint8_t tos; + /** Autonomous system number of the source, either origin or peer */ + uint16_t as_src; + /** Autonomous system number of the destination, either origin or peer */ + uint16_t as_dst; + /** Source address prefix mask bits */ + uint8_t mask_src; + /** Destination address prefix mask bits */ + uint8_t mask_dst; + /** Unused (zero) bytes */ + uint16_t _pad2; +}; + +/** NetFlow v5 version number */ +#define IPX_NF5_VERSION 0x5 +/** Size of NetFlow v5 Packet Header */ +#define IPX_NF5_MSG_HDR_LEN sizeof(struct ipx_nf5_hdr) +/** Size of NetFlow v5 Packet Record */ +#define IPX_NF5_MSG_REC_LEN sizeof(struct ipx_nf5_rec) + +// ------------------------------------------------------------------------------------------------ + +/** + * \struct ipx_nf9_msg_hdr + * \brief NetFlow v9 Packet Header structure + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf9_msg_hdr { + /** Version of Flow Record format exported in this packet */ + uint16_t version; + /** The total number of records in the Export Packet, which is the sum of Options FlowSet + * records, Template FlowSet records, and Data FlowSet records. + */ + uint16_t count; + /** Time in milliseconds since this device was first booted. */ + uint32_t sys_uptime; + /** Time in seconds since 0000 UTC 1970, at which the Export Packet leaves the Exporter. */ + uint32_t unix_sec; + /** + * Incremental sequence counter of all Export Packets sent from the current Observation + * Domain by the Exporter. This value MUST be cumulative, and SHOULD be used by the Collector + * to identify whether any Export Packets have been missed. + */ + uint32_t seq_number; + /** + * A 32-bit value that identifies the Exporter Observation Domain. NetFlow Collectors SHOULD + * use the combination of the source IP address and the Source ID field to separate different + * export streams originating from the same Exporter. + */ + uint32_t source_id; +}; + +/** NetFlow v9 version number */ +#define IPX_NF9_VERSION 0x9 +/** Size of NetFlow v5 Packet Header */ +#define IPX_NF9_MSG_HDR_LEN sizeof(struct ipx_nf9_msg_hdr) + +/** Flowset type identifiers */ +enum ipx_nf9_set_id { + IPX_NF9_SET_TMPLT = 0, /**< Template FlowSet ID */ + IPX_NF9_SET_OPTS_TMPLT = 1, /**< Options Template FlowSet ID */ + IPX_NF9_SET_MIN_DSET = 256 /**< Minimum FlowSet ID for any Data FlowSet */ +}; + +/** + * \struct ipx_nf9_set_hdr + * \brief NetFlow v9 Set Header structure + * \remark Based on RFC 3954, Section 5.1. + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf9_set_hdr { + /** FlowSet ID */ + uint16_t flowset_id; + /** Total length of this FlowSet. + * + * Because an individual FlowSet MAY contain multiple Records, the Length value MUST be used to + * determine the position of the next FlowSet record, which could be any type of FlowSet. + * Length is the sum of the lengths of the FlowSet ID, the Length itself, and all Records + * within this FlowSet. + */ + uint16_t length; +}; + +/** Length of NetFlow FlowSet header (in bytes) */ +#define IPX_NF9_SET_HDR_LEN sizeof(struct ipx_nf9_set_hdr) + +/** + * \struct ipx_nf9_tmplt_ie + * \brief NetFlow v9 Field definition structure + * \remark Based on RFC 3954, Section 5.2. + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf9_tmplt_ie { + /** A numeric value that represents the type of the field. */ + uint16_t id; + /** The length of the corresponding Field Type, in bytes. */ + uint16_t length; +}; + +/** Length of NetFlow v9 Field definition structure */ +#define IPX_NF9_TMPLT_IE_LEN sizeof(struct ipx_nf9_tmplt_ie) + +/** + * \struct ipx_nf9_trec + * \brief NetFlow v9 Template record structure + * \remark Based on RFC 3954, Section 5.2. + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf9_trec { + /** + * \brief Template ID of this Template. + * + * Each of the newly generated Template Records is given a unique Template ID. + * This uniqueness is local to the Observation Domain that generated the Template ID. + * Template IDs 0-255 are reserved for Template FlowSets, Options FlowSets, and other + * reserved FlowSets yet to be created. + * + * Template IDs of Data FlowSets are numbered from 256 (::IPX_NF9_SET_MIN_DSET) to 65535. + */ + uint16_t template_id; + /** + * Number of fields in this Template Record. Because a Template FlowSet usually contains + * multiple Template Records, this field allows the Collector to determine the end of the + * current Template Record and the start of the next. + */ + uint16_t count; + /** + * Field Specifier(s) + */ + struct ipx_nf9_tmplt_ie fields[1]; +}; + +/** + * \struct ipx_nf9_tset + * \brief NetFlow Template Set structure + * + * Consists of the common Set header and the first Template record. + * \remark Based on RFC 3954, Section 5.2. + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf9_tset { + /** + * Common NetFlow v9 FlowSet header. + * Identification of the FlowSet MUST be 0 (::IPX_NF9_SET_TMPLT) + */ + struct ipx_nf9_set_hdr header; + + /** + * The first of template records in this Template FlowSet. Real size of the record is unknown + * here due to a variable count of fields in each record. + */ + struct ipx_nf9_trec first_record; +}; + +/** + * \struct ipx_nf9_opts_trec + * \brief NetFlow v9 Options Template record structure + * \remark Based on RFC 3954, Section 6.1. + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf9_opts_trec { + /** + * \brief Template ID of this Options Template. + * + * Each of the newly generated Template Records is given a unique Template ID. + * This uniqueness is local to the Observation Domain that generated the Template ID. + * Template IDs 0-255 are reserved for Template FlowSets, Options FlowSets, and other + * reserved FlowSets yet to be created. + * + * Template IDs of Data FlowSets are numbered from 256 (::IPX_NF9_SET_MIN_DSET) to 65535. + */ + uint16_t template_id; + /** + * The length in bytes of any Scope field definition contained in the Options Template + * Record (The use of "Scope" is described below). + */ + uint16_t scope_length; + /** + * The length (in bytes) of any options field definitions contained in this Options Template + * Record. + */ + uint16_t option_length; + + /** + * Field Specifier(s) + * \note The Scope fields always precede the Option fields. + */ + struct ipx_nf9_tmplt_ie fields[1]; +}; + +/** + * \struct ipx_nf9_opts_tset + * \brief NetFlow Template Set structure + * + * Consists of the common Set header and the first Options Template record. + * \remark Based on RFC 3954, Section 6.1. + * \warning All values are stored in Network Byte Order! + */ +struct __attribute__((__packed__)) ipx_nf9_opts_tset { + /** + * Common NetFlow v9 FlowSet header. + * Identification of the FlowSet MUST be 1 (::IPX_NF9_SET_OPTS_TMPLT) + */ + struct ipx_nf9_set_hdr header; + + /** + * The first of template records in this Options Template FlowSet. Real size of the record is + * unknown here due to a variable count of fields in each record. + */ + struct ipx_nf9_opts_trec first_record; +}; + +/** + * \struct ipx_nf9_dset + * \brief NetFlow v9 Data FlowSet structure + * + * The Data Records are sent in Data Sets. It consists only of one or more Field Values. The + * Template ID to which the Field Values belong is encoded in the Set Header field "Set ID", i.e., + * "Set ID" = "Template ID". + */ +struct __attribute__((__packed__)) ipx_nf9_dset { + /** + * Common NetFlow v9 FlowSet header. + * Identification of the FlowSet MUST be at least 256 (::IPX_NF9_SET_MIN_DSET) and at + * most 65535. + */ + struct ipx_nf9_set_hdr header; + + /** Start of the first Data record */ + uint8_t record[1]; +}; + +#ifdef __cplusplus +} +#endif +#endif // IPFIXCOL2_NETFLOW_H