diff --git a/extra_plugins/output/unirec/CMakeLists.txt b/extra_plugins/output/unirec/CMakeLists.txt index 5908e7ce..fcd9626a 100644 --- a/extra_plugins/output/unirec/CMakeLists.txt +++ b/extra_plugins/output/unirec/CMakeLists.txt @@ -7,7 +7,7 @@ set(UNIREC_DESCRIPTION ) set(UNIREC_VERSION_MAJOR 2) -set(UNIREC_VERSION_MINOR 0) +set(UNIREC_VERSION_MINOR 2) set(UNIREC_VERSION_PATCH 0) set(UNIREC_VERSION ${UNIREC_VERSION_MAJOR}.${UNIREC_VERSION_MINOR}.${UNIREC_VERSION_PATCH}) @@ -19,9 +19,9 @@ include(CheckCXXCompilerFlag) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") # Find IPFIXcol and libnf -find_package(IPFIXcol2 2.0.0 REQUIRED) -find_package(LibTrap REQUIRED) -find_package(Unirec REQUIRED) +find_package(IPFIXcol2 2.1.0 REQUIRED) # support for basicList is required +find_package(LibTrap 1.13.1 REQUIRED) +find_package(LibUnirec 2.8.0 REQUIRED) # Set default build type if not specified by user if (NOT CMAKE_BUILD_TYPE) @@ -49,7 +49,7 @@ configure_file( include_directories( "${IPFIXCOL2_INCLUDE_DIRS}" # IPFIXcol2 header files "${LIBTRAP_INCLUDE_DIRS}" # libtrap header files - "${UNIREC_INCLUDE_DIRS}" # unirec header files + "${LIBUNIREC_INCLUDE_DIRS}" # unirec header files ) # Create a linkable module @@ -67,7 +67,7 @@ add_library(unirec-output MODULE target_link_libraries(unirec-output ${LIBTRAP_LIBRARIES} # libtrap - ${UNIREC_LIBRARIES} # unirec + ${LIBUNIREC_LIBRARIES} # unirec m # standard math library ) diff --git a/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake b/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake index 607561f4..979aecbb 100644 --- a/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake +++ b/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake @@ -5,30 +5,53 @@ # use pkg-config to get the directories and then use these values # in the find_path() and find_library() calls -find_package(PkgConfig) -pkg_check_modules(PC_LIBTRAP QUIET LibTrap) +find_package(PkgConfig QUIET) +if (PKG_CONFIG_FOUND) + pkg_check_modules(PC_LIBTRAP QUIET "libtrap") +endif() set(LIBTRAP_DEFINITIONS ${PC_LIBTRAP_CFLAGS_OTHER}) find_path( - LIBTRAP_INCLUDE_DIR trap.h + LIBTRAP_INCLUDE_DIR libtrap/trap.h HINTS ${PC_LIBTRAP_INCLUDEDIR} ${PC_LIBTRAP_INCLUDE_DIRS} - PATH_SUFFIXES include/libtrap + PATH_SUFFIX include ) find_library( - LIBTRAP_LIBRARY NAMES trap + LIBTRAP_LIBRARY NAMES trap libtrap HINTS ${PC_LIBTRAP_LIBDIR} ${PC_LIBTRAP_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# handle the QUIETLY and REQUIRED arguments and set LIBLIBTRAP_FOUND to TRUE +if (PC_LIBTRAP_VERSION) + # Version extracted from pkg-config + set(LIBTRAP_VERSION_STRING ${PC_LIBTRAP_VERSION}) +elseif (LIBTRAP_INCLUDE_DIR AND LIBTRAP_LIBRARY) + # Try to get the version of the installed library + try_run( + TRAP_RES_RUN TRAP_RES_COMP + ${CMAKE_CURRENT_BINARY_DIR}/try_run/trap_version_test/ + ${PROJECT_SOURCE_DIR}/CMakeModules/try_run/trap_version.c + CMAKE_FLAGS + -DLINK_LIBRARIES=${LIBTRAP_LIBRARY} + -DINCLUDE_DIRECTORIES=${LIBTRAP_INCLUDE_DIR} + RUN_OUTPUT_VARIABLE LIBTRAP_VERSION_VAR + ) + + if (TRAP_RES_COMP AND TRAP_RES_RUN EQUAL 0) + # Successfully compiled and executed with return code 0 + set(LIBTRAP_VERSION_STRING ${LIBTRAP_VERSION_VAR}) + endif() +endif() + +# handle the QUIETLY and REQUIRED arguments and set LIBTRAP_FOUND to TRUE # if all listed variables are TRUE include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(libtrap +find_package_handle_standard_args(LibTrap REQUIRED_VARS LIBTRAP_LIBRARY LIBTRAP_INCLUDE_DIR VERSION_VAR LIBTRAP_VERSION_STRING ) set(LIBTRAP_LIBRARIES ${LIBTRAP_LIBRARY}) set(LIBTRAP_INCLUDE_DIRS ${LIBTRAP_INCLUDE_DIR}) -mark_as_advanced(LIBTRAP_INCLUDE_DIR LIBTRAP_LIBRARIES) +mark_as_advanced(LIBTRAP_INCLUDE_DIR LIBTRAP_LIBRARY) diff --git a/extra_plugins/output/unirec/CMakeModules/FindLibUnirec.cmake b/extra_plugins/output/unirec/CMakeModules/FindLibUnirec.cmake new file mode 100644 index 00000000..658cde97 --- /dev/null +++ b/extra_plugins/output/unirec/CMakeModules/FindLibUnirec.cmake @@ -0,0 +1,42 @@ +# LIBUNIREC_FOUND - System has libfds +# LIBUNIREC_INCLUDE_DIRS - The libfds include directories +# LIBUNIREC_LIBRARIES - The libraries needed to use libfds +# LIBUNIREC_DEFINITIONS - Compiler switches required for using libfds + +# use pkg-config to get the directories and then use these values +# in the find_path() and find_library() calls +find_package(PkgConfig QUIET) +if (PKG_CONFIG_FOUND) + pkg_check_modules(PC_UNIREC QUIET "unirec") +endif() +set(LIBUNIREC_DEFINITIONS ${PC_UNIREC_CFLAGS_OTHER}) + +find_path( + UNIREC_INCLUDE_DIR unirec/unirec.h + HINTS ${PC_UNIREC_INCLUDEDIR} ${PC_UNIREC_INCLUDE_DIRS} + PATH_SUFFIXES include +) + +find_library( + UNIREC_LIBRARY NAMES unirec libunirec + HINTS ${PC_UNIREC_LIBDIR} ${PC_UNIREC_LIBRARY_DIRS} + PATH_SUFFIXES lib lib64 +) + +if (PC_UNIREC_VERSION) + # Version extracted from pkg-config + set(UNIREC_VERSION_STRING ${PC_UNIREC_VERSION}) +endif() + + +# handle the QUIETLY and REQUIRED arguments and set LIBUNIREC_FOUND to TRUE +# if all listed variables are TRUE +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LibUnirec + REQUIRED_VARS UNIREC_LIBRARY UNIREC_INCLUDE_DIR + VERSION_VAR UNIREC_VERSION_STRING +) + +set(LIBUNIREC_LIBRARIES ${UNIREC_LIBRARY}) +set(LIBUNIREC_INCLUDE_DIRS ${UNIREC_INCLUDE_DIR}) +mark_as_advanced(UNIREC_INCLUDE_DIR UNIREC_LIBRARY) diff --git a/extra_plugins/output/unirec/CMakeModules/FindUnirec.cmake b/extra_plugins/output/unirec/CMakeModules/FindUnirec.cmake deleted file mode 100644 index 10d86a86..00000000 --- a/extra_plugins/output/unirec/CMakeModules/FindUnirec.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# UNIREC_FOUND - System has libfds -# UNIREC_INCLUDE_DIRS - The libfds include directories -# UNIREC_LIBRARIES - The libraries needed to use libfds -# UNIREC_DEFINITIONS - Compiler switches required for using libfds - -# use pkg-config to get the directories and then use these values -# in the find_path() and find_library() calls -find_package(PkgConfig) -pkg_check_modules(PC_UNIREC QUIET LibTrap) -set(UNIREC_DEFINITIONS ${PC_UNIREC_CFLAGS_OTHER}) - -find_path( - UNIREC_INCLUDE_DIR unirec.h - HINTS ${PC_UNIREC_INCLUDEDIR} ${PC_UNIREC_INCLUDE_DIRS} - PATH_SUFFIXES include/unirec -) - -find_library( - UNIREC_LIBRARY NAMES unirec - HINTS ${PC_UNIREC_LIBDIR} ${PC_UNIREC_LIBRARY_DIRS} - PATH_SUFFIXES lib lib64 -) - -# handle the QUIETLY and REQUIRED arguments and set LIBUNIREC_FOUND to TRUE -# if all listed variables are TRUE -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(libtrap - REQUIRED_VARS UNIREC_LIBRARY UNIREC_INCLUDE_DIR - VERSION_VAR UNIREC_VERSION_STRING -) - -set(UNIREC_LIBRARIES ${UNIREC_LIBRARY}) -set(UNIREC_INCLUDE_DIRS ${UNIREC_INCLUDE_DIR}) -mark_as_advanced(UNIREC_INCLUDE_DIR UNIREC_LIBRARIES) diff --git a/extra_plugins/output/unirec/CMakeModules/try_run/trap_version.c b/extra_plugins/output/unirec/CMakeModules/try_run/trap_version.c new file mode 100644 index 00000000..d77cc471 --- /dev/null +++ b/extra_plugins/output/unirec/CMakeModules/try_run/trap_version.c @@ -0,0 +1,10 @@ +#include +#include +#include + +int +main(int argc, char *argv[]) +{ + printf("%s", trap_version); + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 2fb2e643..ac42b050 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -4,8 +4,12 @@ # fields. You can change setting by editing this file. Each entry consists # of the following parameters: # - UniRec field name -# - UniRec data type (int{8,16,32,64}, uint{8,16,32,64}, float, double, time, -# ipaddr, macaddr, char, string, bytes) +# - UniRec data type - one of the following: +# int{8,16,32,64}, uint{8,16,32,64}, +# float, double, time, ipaddr, macaddr, char, string, bytes +# int{8,16,32,64}*, uint{8,16,32,64}*, // "array of" types +# float*, double*, time*, ipaddr*, macaddr* // "array of" types +# string_trimmed // trimmed string (i.e. no tailing '\0') # - Comma separated list of IPFIX Information Elements identifiers # ("eXXidYY" where XX is Private Enterprise Number and YY is field ID) # @@ -31,6 +35,13 @@ TIME_FIRST time e0id150,e0id152,e0id154,e0id156 # T TIME_LAST time e0id151,e0id153,e0id155,e0id157 # Time of the last packet of a flow DIR_BIT_FIELD uint8 _internal_dbf_ # Bit field used for determining incoming/outgoing flow (1 => Incoming, 0 => Outgoing) LINK_BIT_FIELD uint64 _internal_lbf_ # Bit field of links on which was flow seen +SRC_MAC macaddr e0id56 +DST_MAC macaddr e0id80 + +# --- Additional biflow fields --- +BYTES_REV uint64 e29305id1 +PACKETS_REV uint32 e29305id2 +TCP_FLAGS_REV uint8 e29305id6 # --- DNS specific fields --- DNS_ID uint16 e39499id110 # DNS transaction id @@ -123,32 +134,40 @@ IPV6_TUN_TYPE uint8 e16982id405 # IPv6 tunnel type APP_ID bytes e0id95 # Application ID from libprotoident / NBAR2 / Flowmon's NBAR plugin # --- Flowmon TLS fields -TLS_CONTENT_TYPE uint8 flowmon:tlsContentType # tlsContentType -TLS_HANDSHAKE_TYPE uint32 flowmon:tlsHandshakeType # https://tools.ietf.org/html/rfc5246#appendix-A.4 -TLS_SETUP_TIME uint64 flowmon:tlsSetupTime # tlsSetupTime -TLS_SERVER_VERSION uint16 flowmon:tlsServerVersion # 8b major and 8b minor, 0x0303 ~ TLS1.2 -TLS_SERVER_RANDOM bytes flowmon:tlsServerRandom # tlsServerRandom -TLS_SERVER_SESSIONID bytes flowmon:tlsServerSessionId # tlsServerSessionId -TLS_CIPHER_SUITE uint16 flowmon:tlsCipherSuite # https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 -TLS_ALPN string flowmon:tlsAlpn # TLS Application-Layer Protocol Negotiation https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids -TLS_SNI string flowmon:tlsSni # Server Name Indication https://en.wikipedia.org/wiki/Server_Name_Indication -TLS_SNI_LENGTH uint16 flowmon:tlsSniLength # Length of TLS_SNI field -TLS_CLIENT_VERSION uint16 flowmon:tlsClientVersion # tlsClientVersion -TLS_CIPHER_SUITES bytes flowmon:tlsCipherSuites # List of 2B ciphers, beware of network byte order. See https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 -TLS_CLIENT_RANDOM bytes flowmon:tlsClientRandom # tlsClientRandom -TLS_CLIENT_SESSIONID bytes flowmon:tlsClientSessionId # tlsClientSessionId -TLS_EXTENSION_TYPES bytes flowmon:tlsExtensionTypes # tlsExtensionTypes -TLS_EXTENSION_LENGTHS bytes flowmon:tlsExtensionLengths # tlsExtensionLengths -TLS_ELLIPTIC_CURVES bytes flowmon:tlsEllipticCurves # tlsEllipticCurves -TLS_EC_POINTFORMATS bytes flowmon:tlsEcPointFormats # tlsEcPointFormats -TLS_CLIENT_KEYLENGTH int32 flowmon:tlsClientKeyLength # Length of client's key -TLS_ISSUER_CN string flowmon:tlsIssuerCn # Common name of certificate issuer -TLS_SUBJECT_CN string flowmon:tlsSubjectCn # Certificate Common Name -TLS_SUBJECT_ON string flowmon:tlsSubjectOn # Certificate Organization Name -TLS_VALIDITY_NOTBEFORE int64 flowmon:tlsValidityNotBefore # UNIX timestamp of certificate creation -TLS_VALIDITY_NOTAFTER int64 flowmon:tlsValidityNotAfter # UNIX timestamp of certificate expiration -TLS_SIGNATURE_ALG uint16 flowmon:tlsSignatureAlg # tlsSignatureAlg -TLS_PUBLIC_KEYALG uint16 flowmon:tlsPublicKeyAlg # tlsPublicKeyAlg -TLS_PUBLIC_KEYLENGTH int32 flowmon:tlsPublicKeyLength # tlsPublicKeyLength -TLS_JA_3FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint +TLS_CONTENT_TYPE uint8 flowmon:tlsContentType # tlsContentType +TLS_HANDSHAKE_TYPE uint32 flowmon:tlsHandshakeType # https://tools.ietf.org/html/rfc5246#appendix-A.4 +TLS_SETUP_TIME uint64 flowmon:tlsSetupTime # tlsSetupTime +TLS_SERVER_VERSION uint16 flowmon:tlsServerVersion # 8b major and 8b minor, 0x0303 ~ TLS1.2 +TLS_SERVER_RANDOM bytes flowmon:tlsServerRandom # tlsServerRandom +TLS_SERVER_SESSIONID bytes flowmon:tlsServerSessionId # tlsServerSessionId +TLS_CIPHER_SUITE uint16 flowmon:tlsCipherSuite # https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 +TLS_ALPN string flowmon:tlsAlpn # TLS Application-Layer Protocol Negotiation https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids +TLS_SNI string flowmon:tlsSni # Server Name Indication https://en.wikipedia.org/wiki/Server_Name_Indication +TLS_SNI_LENGTH uint16 flowmon:tlsSniLength # Length of TLS_SNI field +TLS_CLIENT_VERSION uint16 flowmon:tlsClientVersion # tlsClientVersion +TLS_CIPHER_SUITES bytes flowmon:tlsCipherSuites # List of 2B ciphers, beware of network byte order. See https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 +TLS_CLIENT_RANDOM bytes flowmon:tlsClientRandom # tlsClientRandom +TLS_CLIENT_SESSIONID bytes flowmon:tlsClientSessionId # tlsClientSessionId +TLS_EXTENSION_TYPES bytes flowmon:tlsExtensionTypes # tlsExtensionTypes +TLS_EXTENSION_LENGTHS bytes flowmon:tlsExtensionLengths # tlsExtensionLengths +TLS_ELLIPTIC_CURVES bytes flowmon:tlsEllipticCurves # tlsEllipticCurves +TLS_EC_POINTFORMATS bytes flowmon:tlsEcPointFormats # tlsEcPointFormats +TLS_CLIENT_KEYLENGTH int32 flowmon:tlsClientKeyLength # Length of client's key +TLS_ISSUER_CN string flowmon:tlsIssuerCn # Common name of certificate issuer +TLS_SUBJECT_CN string flowmon:tlsSubjectCn # Certificate Common Name +TLS_SUBJECT_ON string flowmon:tlsSubjectOn # Certificate Organization Name +TLS_VALIDITY_NOTBEFORE int64 flowmon:tlsValidityNotBefore # UNIX timestamp of certificate creation +TLS_VALIDITY_NOTAFTER int64 flowmon:tlsValidityNotAfter # UNIX timestamp of certificate expiration +TLS_SIGNATURE_ALG uint16 flowmon:tlsSignatureAlg # tlsSignatureAlg +TLS_PUBLIC_KEYALG uint16 flowmon:tlsPublicKeyAlg # tlsPublicKeyAlg +TLS_PUBLIC_KEYLENGTH int32 flowmon:tlsPublicKeyLength # tlsPublicKeyLength +TLS_JA_3FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint +# --- Per-Packet Information elements --- +PPI_TLS_REC_LENGTHS int16* e0id291/e8057id1010 # basicList of TLS record lengths +PPI_TLS_REC_TIMES uint16* e0id291/e8057id1011 # basicList of TLS record timestamps +PPI_TLS_CONTENT_TYPES uint8* e0id291/e8057id1012 # basicList of TLS record content types +PPI_PKT_LENGTHS int16* e0id291/e8057id1013 # basicList of packet lengths +PPI_PKT_TIMES time* e0id291/e8057id1014 # basicList of packet timestamps +PPI_PKT_FLAGS int8* e0id291/e8057id1015 # basicList of packet TCP flags +PPI_PKT_DIRECTIONS int8* e0id291/e8057id1016 # basicList of packet directions diff --git a/extra_plugins/output/unirec/src/map.c b/extra_plugins/output/unirec/src/map.c index d5b96768..415facda 100644 --- a/extra_plugins/output/unirec/src/map.c +++ b/extra_plugins/output/unirec/src/map.c @@ -101,6 +101,13 @@ map_clear(map_t *map) { for (size_t i = 0; i < map->rec_size; ++i) { struct map_rec *rec = map->rec_array[i]; + struct map_ipfix_s *ipfix = rec->ipfix.next; + while (ipfix) { + struct map_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; + } + free(rec->unirec.name); free(rec->unirec.type_str); free(rec); @@ -280,17 +287,53 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, } // Parse IPFIX specifier - const struct fds_iemgr_elem *elem_def = map_elem_get_ipfix(map->iemgr, subtoken); + const struct fds_iemgr_elem *elem_def = NULL; + char *subsave_ptr_list = NULL; + struct map_ipfix_s *ipfix = &rec.ipfix; + rec.ipfix.next = NULL; + + for (char *list = subtoken; ;list = NULL) { + // Test if the specifier describes basicList + char *list_token = strtok_r(list, "/", &subsave_ptr_list); + if (!list_token) { + break; + } + + elem_def = map_elem_get_ipfix(map->iemgr, list_token); + if (elem_def != NULL) { + if (list == NULL) { + ipfix->next = malloc(sizeof(struct map_ipfix_s)); + if (!ipfix->next) { + rc = IPX_ERR_NOMEM; + elem_def = NULL; + break; + } + ipfix = ipfix->next; + } + + // Store the "IPFIX element" record + ipfix->source = MAP_SRC_IPFIX; + ipfix->def = elem_def; + ipfix->id = elem_def->id; + ipfix->en = elem_def->scope->pen; + ipfix->next = NULL; + continue; + } + break; + } + if (elem_def != NULL) { - // Store the "IPFIX element" record - rec.ipfix.source = MAP_SRC_IPFIX; - rec.ipfix.def = elem_def; - rec.ipfix.id = elem_def->id; - rec.ipfix.en = elem_def->scope->pen; rc = map_rec_add(map, &rec); continue; } + ipfix = rec.ipfix.next; + while (ipfix) { + struct map_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; + } + enum MAP_SRC fn_id = map_elem_get_internal(subtoken); if (fn_id != MAP_SRC_INVALID) { // Store the "Internal function" record @@ -298,6 +341,7 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, rec.ipfix.def = NULL; rec.ipfix.id = 0; rec.ipfix.en = 0; + rec.ipfix.next = NULL; rc = map_rec_add(map, &rec); continue; } @@ -414,21 +458,32 @@ map_load_line(map_t *map, const char *line, size_t line_id) static int map_sort_fn(const void *p1, const void *p2) { - struct map_rec *rec1 = *(struct map_rec **) p1; - struct map_rec *rec2 = *(struct map_rec **) p2; + const struct map_ipfix_s *ipfix1 = &(*(struct map_rec **) p1)->ipfix; + const struct map_ipfix_s *ipfix2 = &(*(struct map_rec **) p2)->ipfix; - if (rec1->ipfix.source != rec2->ipfix.source) { - return (rec1->ipfix.source < rec2->ipfix.source) ? (-1) : 1; - } + while (ipfix1 && ipfix2) { + if (ipfix1->source != ipfix2->source) { + return (ipfix1->source < ipfix2->source) ? (-1) : 1; + } + + // Primary sort by PEN + if (ipfix1->en != ipfix2->en) { + return (ipfix1->en < ipfix2->en) ? (-1) : 1; + } - // Primary sort by PEN - if (rec1->ipfix.en != rec2->ipfix.en) { - return (rec1->ipfix.en < rec2->ipfix.en) ? (-1) : 1; + // Secondary sort by ID + if (ipfix1->id != ipfix2->id) { + return (ipfix1->id < ipfix2->id) ? (-1) : 1; + } + + ipfix1 = ipfix1->next; + ipfix2 = ipfix2->next; } - // Secondary sort by ID - if (rec1->ipfix.id != rec2->ipfix.id) { - return (rec1->ipfix.id < rec2->ipfix.id) ? (-1) : 1; + if (ipfix1->next) { + return -1; + } else if (ipfix2->next) { + return 1; } return 0; @@ -507,8 +562,24 @@ map_load(map_t *map, const char *file) continue; } - if (rec_prev->ipfix.en != rec_now->ipfix.en || rec_prev->ipfix.id != rec_now->ipfix.id) { - rec_prev = rec_now; + bool collision = false; + const struct map_ipfix_s *ipfix_prev = &rec_prev->ipfix; + const struct map_ipfix_s *ipfix_now = &rec_now->ipfix; + while (1) { + if (!ipfix_prev || !ipfix_now || ipfix_prev->en != ipfix_now->en + || ipfix_prev->id != ipfix_now->id) { + rec_prev = rec_now; + break; + } + + ipfix_prev = ipfix_prev->next; + ipfix_now = ipfix_now->next; + if (!ipfix_prev && !ipfix_now) { + collision = true; + break; + } + } + if (!collision) { continue; } diff --git a/extra_plugins/output/unirec/src/map.h b/extra_plugins/output/unirec/src/map.h index 35c4cad1..04c2caee 100644 --- a/extra_plugins/output/unirec/src/map.h +++ b/extra_plugins/output/unirec/src/map.h @@ -68,7 +68,7 @@ enum MAP_FLAGS { /** IPFIX-to-UniRec mapping record */ struct map_rec { - struct { + struct map_ipfix_s { /** * \brief Data source * \note If the field is not ::MAP_SRC_IPFIX, parameters en, id and def are NOT defined! @@ -81,6 +81,8 @@ struct map_rec { uint16_t id; /** Definition of the IE (MUST not be NULL) */ const struct fds_iemgr_elem *def; + /** Points to an element of list, NULL if field is not list. */ + struct map_ipfix_s *next; } ipfix; /**< IPFIX specific parameters */ struct { diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index 899bf9f6..36ea75d0 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -2,10 +2,11 @@ * \file translator.c * \author Lukas Hutak * \author Tomas Cejka + * \author Jiri Havranek * \brief Conversion of IPFIX to UniRec format (source file) */ -/* Copyright (C) 2015 - 2017 CESNET, z.s.p.o. +/* Copyright (C) 2015 - 2020 CESNET, z.s.p.o. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -68,7 +69,7 @@ typedef int (*translator_func)(translator_t *trans, const struct translator_rec /** Translator record */ struct translator_rec { - struct { + struct tr_ipfix_s { /** Private Enterprise number */ uint32_t pen; /** Information Element ID */ @@ -77,6 +78,8 @@ struct translator_rec { enum fds_iemgr_element_type type; /** Field semantic */ enum fds_iemgr_element_semantic sem; + /** Pointer to element of IPFIX list */ + struct tr_ipfix_s *next; } ipfix; /** IPFIX field identification */ struct { @@ -170,27 +173,11 @@ struct translator_s { *((dst_type *) (dst_ptr)) = (dst_type) (src_val); \ } -/** - * \brief Convert IPFIX unsigned integer to UniRec (un)signed integer - * \param[in,out] trans Translator instance (UniRec record will be modified) - * \param[in] rec Translator record (description of source IPFIX and destination UniRec fields) - * \param[in] field IPFIX field data - * \return On success returns 0. Otherwise returns a non-zero value. - */ -static int -translate_uint(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) +static inline int +translator_store_uint(ur_field_type_t ur_type, void *field_ptr, uint64_t value, + const enum fds_iemgr_element_semantic ipx_sem) { - uint64_t value; - if (fds_get_uint_be(field->data, field->size, &value) != FDS_OK) { - return 1; // Conversion failed - } - - ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; - - switch (rec->unirec.type) { + switch (ur_type) { case UR_TYPE_UINT64: *((uint64_t *) field_ptr) = value; break; @@ -219,6 +206,32 @@ translate_uint(translator_t *trans, const struct translator_rec *rec, default: return 1; // Unsupported data type } + return 0; +} + +/** + * \brief Convert IPFIX unsigned integer to UniRec (un)signed integer + * \param[in,out] trans Translator instance (UniRec record will be modified) + * \param[in] rec Translator record (description of source IPFIX and destination UniRec fields) + * \param[in] field IPFIX field data + * \return On success returns 0. Otherwise returns a non-zero value. + */ +static int +translate_uint(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + uint64_t value; + if (fds_get_uint_be(field->data, field->size, &value) != FDS_OK) { + return 1; // Conversion failed + } + + ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; + + if (translator_store_uint(rec->unirec.type, field_ptr, value, ipx_sem)) { + return 1; + } return 0; } @@ -253,24 +266,11 @@ translate_uint(translator_t *trans, const struct translator_rec *rec, *((dst_type *) (dst_ptr)) = (dst_type) (src_val); \ } -/** - * \brief Convert IPFIX signed integer to UniRec (un)signed integer - * \copydetails translate_uint() - */ -static int -translate_int(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) +static inline int +translator_store_int(ur_field_type_t ur_type, void *field_ptr, int64_t value, + const enum fds_iemgr_element_semantic ipx_sem) { - int64_t value; - if (fds_get_int_be(field->data, field->size, &value) != FDS_OK) { - return 1; // Conversion failed - } - - ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; - - switch (rec->unirec.type) { + switch (ur_type) { case UR_TYPE_INT64: *((int64_t *) field_ptr) = value; break; @@ -299,6 +299,29 @@ translate_int(translator_t *trans, const struct translator_rec *rec, default: return 1; // Unsupported data type } + return 0; +} + +/** + * \brief Convert IPFIX signed integer to UniRec (un)signed integer + * \copydetails translate_uint() + */ +static int +translate_int(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + int64_t value; + if (fds_get_int_be(field->data, field->size, &value) != FDS_OK) { + return 1; // Conversion failed + } + + ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; + + if (translator_store_int(rec->unirec.type, field_ptr, value, ipx_sem)) { + return 1; + } return 0; } @@ -332,24 +355,11 @@ translator_string_trim(translator_t *trans, const struct translator_rec *rec, return 0; } -/** - * \brief Convert IPFIX boolean to UniRec char/(un)signed integer - * \copydetails translate_uint() - */ -static int -translate_bool(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) +static inline int +translator_store_bool(int ur_size, void *field_ptr, bool value) { - bool value; - if (fds_get_bool(field->data, field->size, &value) != FDS_OK) { - return 1; - } - - const ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - const uint8_t res = value ? 1U : 0U; - switch (rec->unirec.size) { + switch (ur_size) { case 1: *((uint8_t *) field_ptr) = res; break; @@ -366,27 +376,37 @@ translate_bool(translator_t *trans, const struct translator_rec *rec, // Invalid size of the field return 1; } - return 0; } + /** - * \brief Convert IPFIX float to UniRec float + * \brief Convert IPFIX boolean to UniRec char/(un)signed integer * \copydetails translate_uint() */ static int -translate_float(translator_t *trans, const struct translator_rec *rec, +translate_bool(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - double value; - if (fds_get_float_be(field->data, field->size, &value) != FDS_OK) { - return 1; // Conversion failed + bool value; + if (fds_get_bool(field->data, field->size, &value) != FDS_OK) { + return 1; } const ur_field_id_t ur_id = rec->unirec.id; void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - switch (rec->unirec.type) { + if (translator_store_bool(rec->unirec.size, field_ptr, value)) { + return 1; + } + + return 0; +} + +static inline int +translator_store_float(ur_field_type_t ur_type, void *field_ptr, double value) +{ + switch (ur_type) { case UR_TYPE_FLOAT: if (value < -FLT_MAX && isnormal(value)) { *((float *) field_ptr) = -FLT_MAX; @@ -408,22 +428,37 @@ translate_float(translator_t *trans, const struct translator_rec *rec, } /** - * \brief Convert IPFIX IPv4/IPv6 address to UniRec IPv4/IPv6 address + * \brief Convert IPFIX float to UniRec float * \copydetails translate_uint() */ static int -translate_ip(translator_t *trans, const struct translator_rec *rec, +translate_float(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { + double value; + if (fds_get_float_be(field->data, field->size, &value) != FDS_OK) { + return 1; // Conversion failed + } + const ur_field_id_t ur_id = rec->unirec.id; void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - switch (field->size) { + if (translator_store_float(rec->unirec.type, field_ptr, value)) { + return 1; + } + + return 0; +} + +static inline int +translator_store_ip(const uint8_t *ip_bytes, uint16_t data_size, void *field_ptr) +{ + switch (data_size) { case 4: // IPv4 - *((ip_addr_t *) field_ptr) = ip_from_4_bytes_be((char *) field->data); + *((ip_addr_t *) field_ptr) = ip_from_4_bytes_be((char *) ip_bytes); break; case 16: // IPv6 - *((ip_addr_t *) field_ptr) = ip_from_16_bytes_be((char *) field->data); + *((ip_addr_t *) field_ptr) = ip_from_16_bytes_be((char *) ip_bytes); break; default: // Invalid size of the field @@ -434,40 +469,49 @@ translate_ip(translator_t *trans, const struct translator_rec *rec, } /** - * \brief Convert IPFIX MAC address to UniRec MAC address + * \brief Convert IPFIX IPv4/IPv6 address to UniRec IPv4/IPv6 address * \copydetails translate_uint() */ static int -translate_mac(translator_t *trans, const struct translator_rec *rec, +translate_ip(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - if (field->size != 6U) { - return 1; + const ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (translator_store_ip(field->data, field->size, field_ptr)) { + return 1; } - const ur_field_id_t ur_id = rec->unirec.id; - ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - memcpy(field_ptr, field->data, 6U); return 0; } /** - * \brief Convert IPFIX timestamp to UniRec timestamp + * \brief Convert IPFIX MAC address to UniRec MAC address * \copydetails translate_uint() */ static int -translate_time(translator_t *trans, const struct translator_rec *rec, +translate_mac(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - // Get the value - const enum fds_iemgr_element_type type_ipx = rec->ipfix.type; + if (field->size != 6U) { + return 1; + } + const ur_field_id_t ur_id = rec->unirec.id; ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + memcpy(field_ptr, field->data, 6U); + return 0; +} +static inline int +translator_store_time(const enum fds_iemgr_element_type type_ipx, const uint8_t *time_bytes, + uint16_t time_size, ur_time_t *field_ptr) +{ if (type_ipx == FDS_ET_DATE_TIME_MILLISECONDS || type_ipx == FDS_ET_DATE_TIME_SECONDS) { // Low precision timestamp uint64_t ts; - if (fds_get_datetime_lp_be(field->data, field->size, type_ipx, &ts) != FDS_OK) { + if (fds_get_datetime_lp_be(time_bytes, time_size, type_ipx, &ts) != FDS_OK) { return 1; } @@ -476,7 +520,7 @@ translate_time(translator_t *trans, const struct translator_rec *rec, } else if (type_ipx == FDS_ET_DATE_TIME_MICROSECONDS || type_ipx == FDS_ET_DATE_TIME_NANOSECONDS) { // High precision timestamp struct timespec ts; - if (fds_get_datetime_hp_be(field->data, field->size, type_ipx, &ts) != FDS_OK) { + if (fds_get_datetime_hp_be(time_bytes, time_size, type_ipx, &ts) != FDS_OK) { return 1; } @@ -487,6 +531,228 @@ translate_time(translator_t *trans, const struct translator_rec *rec, return 1; } +/** + * \brief Convert IPFIX timestamp to UniRec timestamp + * \copydetails translate_uint() + */ +static int +translate_time(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + // Get the value + const enum fds_iemgr_element_type type_ipx = rec->ipfix.type; + const ur_field_id_t ur_id = rec->unirec.id; + ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (translator_store_time(type_ipx, field->data, field->size, field_ptr)) { + return 1; + } + + return 0; +} + +/** + * \brief Convert IPFIX list of uint elements to UniRec uint array + * \copydetails translate_uint() + */ +static int +translate_array_uint(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + uint64_t value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_uint_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } + + if (translator_store_uint(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } + + return 0; +} + +/** + * \brief Convert IPFIX list of int elements to UniRec int array + * \copydetails translate_uint() + */ +static int +translate_array_int(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + int64_t value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_int_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } + + if (translator_store_int(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } + + return 0; +} + +/** + * \brief Convert IPFIX list of bool elements to UniRec bool array + * \copydetails translate_uint() + */ +static int +translate_array_bool(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + bool value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_bool(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } + + if (translator_store_bool(ur_array_get_elem_size(ur_id), field_ptr, value)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } + + return 0; +} + +/** + * \brief Convert IPFIX list of float elements to UniRec float array + * \copydetails translate_uint() + */ +static int +translate_array_float(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + double value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_float_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } + + if (translator_store_float(ur_array_get_elem_type(ur_id), field_ptr, value)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } + + return 0; +} + +/** + * \brief Convert IPFIX list of IPv4/IPv6 elements to UniRec IP array + * \copydetails translate_uint() + */ +static int +translate_array_ip(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (field_ptr == NULL || translator_store_ip(list_it.field.data, list_it.field.size, field_ptr)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } + + return 0; +} + +/** + * \brief Convert IPFIX list of MAC elements to UniRec MAC array + * \copydetails translate_uint() + */ +static int +translate_array_mac(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + if (list_it.field.size != 6U) { + return 1; + } + + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + + memcpy(field_ptr, list_it.field.data, 6U); + } + + return 0; +} + +/** + * \brief Convert IPFIX list of timestamp elements to UniRec timestamp array + * \copydetails translate_uint() + */ +static int +translate_array_time(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_type type_ipx = rec->ipfix.next->type; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || translator_store_time(type_ipx, list_it.field.data, list_it.field.size, field_ptr)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } + + return 0; +} + /** * \brief Convert "ingressInterface" (EN: 0, ID: 10) to dir_bif_field * \copydetails translate_uint() @@ -579,17 +845,27 @@ translator_size_ur_int(ur_field_type_t type) case UR_TYPE_CHAR: case UR_TYPE_UINT8: case UR_TYPE_INT8: + case UR_TYPE_A_UINT8: + case UR_TYPE_A_INT8: return 1U; case UR_TYPE_UINT16: case UR_TYPE_INT16: + case UR_TYPE_A_UINT16: + case UR_TYPE_A_INT16: return 2U; case UR_TYPE_UINT32: case UR_TYPE_INT32: case UR_TYPE_FLOAT: + case UR_TYPE_A_UINT32: + case UR_TYPE_A_INT32: + case UR_TYPE_A_FLOAT: return 4U; case UR_TYPE_UINT64: case UR_TYPE_INT64: case UR_TYPE_DOUBLE: + case UR_TYPE_A_UINT64: + case UR_TYPE_A_INT64: + case UR_TYPE_A_DOUBLE: return 8U; default: return 0U; @@ -632,11 +908,18 @@ translator_size_ipx_int(enum fds_iemgr_element_type type) * \return Pointer to the function or NULL (i.e. conversion not supported) */ static translator_func -translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) +translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec, bool is_array) { ur_field_type_t type_ur = rec->unirec.type; enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; + if (is_array) { + if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { + return NULL; + } + type_ipx = rec->ipfix.next->def->data_type; + } + uint16_t size_ur = translator_size_ur_int(type_ur); uint16_t size_ipx = translator_size_ipx_int(type_ipx); if (size_ur == 0 || size_ipx == 0) { @@ -647,13 +930,21 @@ translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) bool warn_msg; if (fds_iemgr_is_type_unsigned(type_ipx)) { - fn = translate_uint; + if (is_array) { + fn = translate_array_uint; + } else { + fn = translate_uint; + } // Check possible conversion errors bool is_signed = (type_ur == UR_TYPE_INT8 || type_ur == UR_TYPE_INT16 || type_ur == UR_TYPE_INT32 || type_ur == UR_TYPE_INT64); warn_msg = ((is_signed && size_ur <= size_ipx) || (!is_signed && size_ur < size_ipx)); } else if (fds_iemgr_is_type_signed(type_ipx)) { - fn = translate_int; + if (is_array) { + fn = translate_array_int; + } else { + fn = translate_int; + } // Check possible conversion errors bool is_unsigned = (type_ur == UR_TYPE_UINT8 || type_ur == UR_TYPE_UINT16 || type_ur == UR_TYPE_UINT32 || type_ur == UR_TYPE_UINT64); @@ -664,6 +955,9 @@ translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) if (warn_msg) { const struct fds_iemgr_elem *el = rec->ipfix.def; + if (is_array) { + el = rec->ipfix.next->def; + } IPX_CTX_WARNING(ctx, "Conversion from IPFIX IE '%s:%s' (%s) to UniRec '%s' (%s) may alter " "its value!", el->scope->name, el->name, fds_iemgr_type2str(type_ipx), rec->unirec.name, rec->unirec.type_str); @@ -679,11 +973,18 @@ translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) * \return Pointer to the function or NULL (i.e. conversion not supported) */ static translator_func -translator_get_float_func(ipx_ctx_t *ctx, const struct map_rec *rec) +translator_get_float_func(ipx_ctx_t *ctx, const struct map_rec *rec, bool is_array) { ur_field_type_t type_ur = rec->unirec.type; enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; + if (is_array) { + if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { + return NULL; + } + type_ipx = rec->ipfix.next->def->data_type; + } + uint16_t size_ur = translator_size_ur_int(type_ur); uint16_t size_ipx = translator_size_ipx_int(type_ipx); if (size_ur == 0 || size_ipx == 0) { @@ -692,12 +993,15 @@ translator_get_float_func(ipx_ctx_t *ctx, const struct map_rec *rec) if (size_ur < size_ipx) { const struct fds_iemgr_elem *el = rec->ipfix.def; + if (is_array) { + el = rec->ipfix.next->def; + } IPX_CTX_WARNING(ctx, "Conversion from IPFIX IE '%s:%s' (%s) to UniRec '%s' (%s) may alter " "its value!", el->scope->name, el->name, fds_iemgr_type2str(type_ipx), rec->unirec.name, rec->unirec.type_str); } - return translate_float; + return (is_array ? translate_array_float : translate_float); } /** @@ -713,6 +1017,10 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) ur_field_type_t type_ur = rec->unirec.type; enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; + if (rec->ipfix.next) { + type_ipx = rec->ipfix.next->def->data_type; + } + switch (type_ur) { case UR_TYPE_STRING: // String array @@ -737,7 +1045,7 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) case UR_TYPE_UINT64: // Char and (un)signed integer if (fds_iemgr_is_type_unsigned(type_ipx) || fds_iemgr_is_type_signed(type_ipx)) { - return translator_get_numeric_func(ctx, rec); + return translator_get_numeric_func(ctx, rec, false); } else if (type_ipx == FDS_ET_BOOLEAN) { return translate_bool; } @@ -746,7 +1054,7 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) case UR_TYPE_DOUBLE: // Floating-point number if (fds_iemgr_is_type_float(type_ipx)) { - return translator_get_float_func(ctx, rec); + return translator_get_float_func(ctx, rec, false); } break; case UR_TYPE_IP: @@ -767,6 +1075,46 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) return translate_time; } break; + case UR_TYPE_A_INT8: + case UR_TYPE_A_INT16: + case UR_TYPE_A_INT32: + case UR_TYPE_A_INT64: + case UR_TYPE_A_UINT8: + case UR_TYPE_A_UINT16: + case UR_TYPE_A_UINT32: + case UR_TYPE_A_UINT64: + // Char and (un)signed integer + if (fds_iemgr_is_type_unsigned(type_ipx) || fds_iemgr_is_type_signed(type_ipx)) { + return translator_get_numeric_func(ctx, rec, true); + } else if (type_ipx == FDS_ET_BOOLEAN) { + return translate_array_bool; + } + break; + case UR_TYPE_A_FLOAT: + case UR_TYPE_A_DOUBLE: + // Floating-point number + if (fds_iemgr_is_type_float(type_ipx)) { + return translator_get_float_func(ctx, rec, true); + } + break; + case UR_TYPE_A_IP: + // IP addresses + if (fds_iemgr_is_type_ip(type_ipx)) { + return translate_array_ip; + } + break; + case UR_TYPE_A_MAC: + // MAC address + if (type_ipx == FDS_ET_MAC_ADDRESS) { + return translate_array_mac; + } + break; + case UR_TYPE_A_TIME: + // Timestamp + if (fds_iemgr_is_type_time(type_ipx)) { + return translate_array_time; + } + break; default: break; } @@ -937,18 +1285,29 @@ translator_destroy_record(translator_t *trans) static int translator_cmp(const void *p1, const void *p2) { - const struct translator_rec *elem1, *elem2; - elem1 = (const struct translator_rec *) p1; - elem2 = (const struct translator_rec *) p2; + const struct tr_ipfix_s *ipfix1 = &((const struct translator_rec *) p1)->ipfix; + const struct tr_ipfix_s *ipfix2 = &((const struct translator_rec *) p2)->ipfix; - uint64_t elem1_val = ((uint64_t) elem1->ipfix.pen) << 16 | elem1->ipfix.id; - uint64_t elem2_val = ((uint64_t) elem2->ipfix.pen) << 16 | elem2->ipfix.id; + while (ipfix1 && ipfix2) { + uint64_t elem1_val = ((uint64_t) ipfix1->pen) << 16 | ipfix1->id; + uint64_t elem2_val = ((uint64_t) ipfix2->pen) << 16 | ipfix2->id; - if (elem1_val == elem2_val) { - return 0; - } else { - return (elem1_val < elem2_val) ? (-1) : 1; + if (elem1_val < elem2_val) { + return -1; + } else if (elem1_val > elem2_val) { + return 1; + } + + ipfix1 = ipfix1->next; + ipfix2 = ipfix2->next; + } + + if (ipfix1) { + return -1; + } else if (ipfix2) { + return 1; } + return 0; } /** @@ -988,10 +1347,22 @@ translator_table_fill_rec(translator_t *trans, const struct map_rec *map_rec, trans_rec->unirec.size = ur_get_size(ur_id); trans_rec->unirec.type = ur_get_type(ur_id); trans_rec->unirec.req_idx = field_idx; - trans_rec->ipfix.pen = map_rec->ipfix.en; - trans_rec->ipfix.id = map_rec->ipfix.id; - trans_rec->ipfix.type = map_rec->ipfix.def->data_type; - trans_rec->ipfix.sem = map_rec->ipfix.def->data_semantic; + + const struct map_ipfix_s *ipfix = &map_rec->ipfix; + struct tr_ipfix_s *tr_ipfix = &trans_rec->ipfix; + while (ipfix) { + tr_ipfix->pen = ipfix->en; + tr_ipfix->id = ipfix->id; + tr_ipfix->type = ipfix->def->data_type; + tr_ipfix->sem = ipfix->def->data_semantic; + if (ipfix->next) { + tr_ipfix->next = malloc(sizeof(struct tr_ipfix_s)); + tr_ipfix = tr_ipfix->next; + } else { + tr_ipfix->next = NULL; + } + ipfix = ipfix->next; + } IPX_CTX_DEBUG(trans->ctx, "Added conversion from IPFIX IE '%s:%s' to UniRec '%s'", map_rec->ipfix.def->scope->name, map_rec->ipfix.def->name, map_rec->unirec.name); @@ -1073,6 +1444,7 @@ translator_table_fill_internal(translator_t *trans, const struct map_rec *map_re trans_rec->ipfix.id = 10; // iana:ingressInterface trans_rec->ipfix.type = FDS_ET_UNSIGNED_32; trans_rec->ipfix.sem = FDS_ES_IDENTIFIER; + trans_rec->ipfix.next = NULL; trans_rec->func = translate_internal_dbf; IPX_CTX_DEBUG(trans->ctx, "Added conversion from internal 'dir_bit_field' to UniRec '%s'", map_rec->unirec.name); @@ -1083,6 +1455,21 @@ translator_table_fill_internal(translator_t *trans, const struct map_rec *map_re return IPX_ERR_DENIED; } +static inline void +translator_table_free_recs(struct translator_rec *recs, size_t recs_cnt) +{ + for (size_t i = 0; i < recs_cnt; ++i) { + struct tr_ipfix_s *ipfix = recs[i].ipfix.next; + while (ipfix) { + struct tr_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; + } + } + + free(recs); +} + /** * \brief Initialize a conversion table * @@ -1157,7 +1544,7 @@ translator_init_table(translator_t *trans, const map_t *map, const ur_template_t } if (ret_val != IPX_OK) { // Failed - free(table); + translator_table_free_recs(table, rec_cnt); return ret_val; } @@ -1181,7 +1568,7 @@ translator_init_table(translator_t *trans, const map_t *map, const ur_template_t static void translator_destroy_table(translator_t *trans) { - free(trans->table.recs); + translator_table_free_recs(trans->table.recs, trans->table.size); } /** @@ -1276,12 +1663,29 @@ translator_translate(translator_t *trans, struct fds_drec *ipfix_rec, uint16_t f // First, call special internal conversion functions, if enabled int converted_fields = translator_call_internals(trans); + struct tr_ipfix_s ipx_list_elem; // Try to convert all IPFIX fields while (fds_drec_iter_next(&it) != FDS_EOC) { // Find the conversion function const struct fds_tfield *info = it.field.info; key.ipfix.id = info->id; key.ipfix.pen = info->en; + key.ipfix.next = NULL; + + if (info->def && info->def->data_type == FDS_ET_BASIC_LIST) { + struct fds_blist_iter list_it; + + fds_blist_iter_init(&list_it, &it.field, NULL); + if (fds_blist_iter_next(&list_it) == FDS_ERR_FORMAT) { + continue; + } + const struct fds_tfield *tmp = list_it.field.info; + ipx_list_elem.id = tmp->id; + ipx_list_elem.pen = tmp->en; + ipx_list_elem.next = NULL; + + key.ipfix.next = &ipx_list_elem; + } def = bsearch(&key, trans->table.recs, table_rec_cnt, table_rec_size, translator_cmp); if (!def) { @@ -1306,7 +1710,7 @@ translator_translate(translator_t *trans, struct fds_drec *ipfix_rec, uint16_t f return NULL; } - // Check if conversion filled are required fields + // Check if conversion filled all required fields size_t idx; for (idx = 0; idx < trans->progress.size && trans->progress.req_fields[idx] == 0; ++idx); if (idx < trans->progress.size) {