From fc9ee6837aeb382f2e31b2c924f7a37d26b8c899 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 29 Jan 2020 15:58:28 +0100 Subject: [PATCH 01/16] Unirec output: improved detection of libtrap and libunirec (added version extraction) --- extra_plugins/output/unirec/CMakeLists.txt | 6 +-- .../unirec/CMakeModules/FindLibTrap.cmake | 39 +++++++++++++---- .../unirec/CMakeModules/FindLibUnirec.cmake | 42 +++++++++++++++++++ .../unirec/CMakeModules/FindUnirec.cmake | 34 --------------- .../CMakeModules/try_run/trap_version.c | 10 +++++ 5 files changed, 86 insertions(+), 45 deletions(-) create mode 100644 extra_plugins/output/unirec/CMakeModules/FindLibUnirec.cmake delete mode 100644 extra_plugins/output/unirec/CMakeModules/FindUnirec.cmake create mode 100644 extra_plugins/output/unirec/CMakeModules/try_run/trap_version.c diff --git a/extra_plugins/output/unirec/CMakeLists.txt b/extra_plugins/output/unirec/CMakeLists.txt index 5908e7ce..794ebf19 100644 --- a/extra_plugins/output/unirec/CMakeLists.txt +++ b/extra_plugins/output/unirec/CMakeLists.txt @@ -21,7 +21,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") # Find IPFIXcol and libnf find_package(IPFIXcol2 2.0.0 REQUIRED) find_package(LibTrap REQUIRED) -find_package(Unirec REQUIRED) +find_package(LibUnirec REQUIRED) # Set default build type if not specified by user if (NOT CMAKE_BUILD_TYPE) @@ -49,7 +49,7 @@ configure_file( include_directories( "${IPFIXCOL2_INCLUDE_DIRS}" # IPFIXcol2 header files "${LIBTRAP_INCLUDE_DIRS}" # libtrap header files - "${UNIREC_INCLUDE_DIRS}" # unirec header files + "${LIBUNIREC_INCLUDE_DIRS}" # unirec header files ) # Create a linkable module @@ -67,7 +67,7 @@ add_library(unirec-output MODULE target_link_libraries(unirec-output ${LIBTRAP_LIBRARIES} # libtrap - ${UNIREC_LIBRARIES} # unirec + ${LIBUNIREC_LIBRARIES} # unirec m # standard math library ) diff --git a/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake b/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake index 607561f4..979aecbb 100644 --- a/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake +++ b/extra_plugins/output/unirec/CMakeModules/FindLibTrap.cmake @@ -5,30 +5,53 @@ # use pkg-config to get the directories and then use these values # in the find_path() and find_library() calls -find_package(PkgConfig) -pkg_check_modules(PC_LIBTRAP QUIET LibTrap) +find_package(PkgConfig QUIET) +if (PKG_CONFIG_FOUND) + pkg_check_modules(PC_LIBTRAP QUIET "libtrap") +endif() set(LIBTRAP_DEFINITIONS ${PC_LIBTRAP_CFLAGS_OTHER}) find_path( - LIBTRAP_INCLUDE_DIR trap.h + LIBTRAP_INCLUDE_DIR libtrap/trap.h HINTS ${PC_LIBTRAP_INCLUDEDIR} ${PC_LIBTRAP_INCLUDE_DIRS} - PATH_SUFFIXES include/libtrap + PATH_SUFFIX include ) find_library( - LIBTRAP_LIBRARY NAMES trap + LIBTRAP_LIBRARY NAMES trap libtrap HINTS ${PC_LIBTRAP_LIBDIR} ${PC_LIBTRAP_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# handle the QUIETLY and REQUIRED arguments and set LIBLIBTRAP_FOUND to TRUE +if (PC_LIBTRAP_VERSION) + # Version extracted from pkg-config + set(LIBTRAP_VERSION_STRING ${PC_LIBTRAP_VERSION}) +elseif (LIBTRAP_INCLUDE_DIR AND LIBTRAP_LIBRARY) + # Try to get the version of the installed library + try_run( + TRAP_RES_RUN TRAP_RES_COMP + ${CMAKE_CURRENT_BINARY_DIR}/try_run/trap_version_test/ + ${PROJECT_SOURCE_DIR}/CMakeModules/try_run/trap_version.c + CMAKE_FLAGS + -DLINK_LIBRARIES=${LIBTRAP_LIBRARY} + -DINCLUDE_DIRECTORIES=${LIBTRAP_INCLUDE_DIR} + RUN_OUTPUT_VARIABLE LIBTRAP_VERSION_VAR + ) + + if (TRAP_RES_COMP AND TRAP_RES_RUN EQUAL 0) + # Successfully compiled and executed with return code 0 + set(LIBTRAP_VERSION_STRING ${LIBTRAP_VERSION_VAR}) + endif() +endif() + +# handle the QUIETLY and REQUIRED arguments and set LIBTRAP_FOUND to TRUE # if all listed variables are TRUE include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(libtrap +find_package_handle_standard_args(LibTrap REQUIRED_VARS LIBTRAP_LIBRARY LIBTRAP_INCLUDE_DIR VERSION_VAR LIBTRAP_VERSION_STRING ) set(LIBTRAP_LIBRARIES ${LIBTRAP_LIBRARY}) set(LIBTRAP_INCLUDE_DIRS ${LIBTRAP_INCLUDE_DIR}) -mark_as_advanced(LIBTRAP_INCLUDE_DIR LIBTRAP_LIBRARIES) +mark_as_advanced(LIBTRAP_INCLUDE_DIR LIBTRAP_LIBRARY) diff --git a/extra_plugins/output/unirec/CMakeModules/FindLibUnirec.cmake b/extra_plugins/output/unirec/CMakeModules/FindLibUnirec.cmake new file mode 100644 index 00000000..658cde97 --- /dev/null +++ b/extra_plugins/output/unirec/CMakeModules/FindLibUnirec.cmake @@ -0,0 +1,42 @@ +# LIBUNIREC_FOUND - System has libfds +# LIBUNIREC_INCLUDE_DIRS - The libfds include directories +# LIBUNIREC_LIBRARIES - The libraries needed to use libfds +# LIBUNIREC_DEFINITIONS - Compiler switches required for using libfds + +# use pkg-config to get the directories and then use these values +# in the find_path() and find_library() calls +find_package(PkgConfig QUIET) +if (PKG_CONFIG_FOUND) + pkg_check_modules(PC_UNIREC QUIET "unirec") +endif() +set(LIBUNIREC_DEFINITIONS ${PC_UNIREC_CFLAGS_OTHER}) + +find_path( + UNIREC_INCLUDE_DIR unirec/unirec.h + HINTS ${PC_UNIREC_INCLUDEDIR} ${PC_UNIREC_INCLUDE_DIRS} + PATH_SUFFIXES include +) + +find_library( + UNIREC_LIBRARY NAMES unirec libunirec + HINTS ${PC_UNIREC_LIBDIR} ${PC_UNIREC_LIBRARY_DIRS} + PATH_SUFFIXES lib lib64 +) + +if (PC_UNIREC_VERSION) + # Version extracted from pkg-config + set(UNIREC_VERSION_STRING ${PC_UNIREC_VERSION}) +endif() + + +# handle the QUIETLY and REQUIRED arguments and set LIBUNIREC_FOUND to TRUE +# if all listed variables are TRUE +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LibUnirec + REQUIRED_VARS UNIREC_LIBRARY UNIREC_INCLUDE_DIR + VERSION_VAR UNIREC_VERSION_STRING +) + +set(LIBUNIREC_LIBRARIES ${UNIREC_LIBRARY}) +set(LIBUNIREC_INCLUDE_DIRS ${UNIREC_INCLUDE_DIR}) +mark_as_advanced(UNIREC_INCLUDE_DIR UNIREC_LIBRARY) diff --git a/extra_plugins/output/unirec/CMakeModules/FindUnirec.cmake b/extra_plugins/output/unirec/CMakeModules/FindUnirec.cmake deleted file mode 100644 index 10d86a86..00000000 --- a/extra_plugins/output/unirec/CMakeModules/FindUnirec.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# UNIREC_FOUND - System has libfds -# UNIREC_INCLUDE_DIRS - The libfds include directories -# UNIREC_LIBRARIES - The libraries needed to use libfds -# UNIREC_DEFINITIONS - Compiler switches required for using libfds - -# use pkg-config to get the directories and then use these values -# in the find_path() and find_library() calls -find_package(PkgConfig) -pkg_check_modules(PC_UNIREC QUIET LibTrap) -set(UNIREC_DEFINITIONS ${PC_UNIREC_CFLAGS_OTHER}) - -find_path( - UNIREC_INCLUDE_DIR unirec.h - HINTS ${PC_UNIREC_INCLUDEDIR} ${PC_UNIREC_INCLUDE_DIRS} - PATH_SUFFIXES include/unirec -) - -find_library( - UNIREC_LIBRARY NAMES unirec - HINTS ${PC_UNIREC_LIBDIR} ${PC_UNIREC_LIBRARY_DIRS} - PATH_SUFFIXES lib lib64 -) - -# handle the QUIETLY and REQUIRED arguments and set LIBUNIREC_FOUND to TRUE -# if all listed variables are TRUE -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(libtrap - REQUIRED_VARS UNIREC_LIBRARY UNIREC_INCLUDE_DIR - VERSION_VAR UNIREC_VERSION_STRING -) - -set(UNIREC_LIBRARIES ${UNIREC_LIBRARY}) -set(UNIREC_INCLUDE_DIRS ${UNIREC_INCLUDE_DIR}) -mark_as_advanced(UNIREC_INCLUDE_DIR UNIREC_LIBRARIES) diff --git a/extra_plugins/output/unirec/CMakeModules/try_run/trap_version.c b/extra_plugins/output/unirec/CMakeModules/try_run/trap_version.c new file mode 100644 index 00000000..d77cc471 --- /dev/null +++ b/extra_plugins/output/unirec/CMakeModules/try_run/trap_version.c @@ -0,0 +1,10 @@ +#include +#include +#include + +int +main(int argc, char *argv[]) +{ + printf("%s", trap_version); + return EXIT_SUCCESS; +} \ No newline at end of file From 87d11c5d72ec9998ba99cdb214dceaa1fe2c6545 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 11 Feb 2020 10:33:22 +0100 Subject: [PATCH 02/16] UniRec output: add dependency on the newest UniRec and TRAP library The newest UniRec library is required due to new function ur_array_append_get_ptr() --- extra_plugins/output/unirec/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extra_plugins/output/unirec/CMakeLists.txt b/extra_plugins/output/unirec/CMakeLists.txt index 794ebf19..fcd9626a 100644 --- a/extra_plugins/output/unirec/CMakeLists.txt +++ b/extra_plugins/output/unirec/CMakeLists.txt @@ -7,7 +7,7 @@ set(UNIREC_DESCRIPTION ) set(UNIREC_VERSION_MAJOR 2) -set(UNIREC_VERSION_MINOR 0) +set(UNIREC_VERSION_MINOR 2) set(UNIREC_VERSION_PATCH 0) set(UNIREC_VERSION ${UNIREC_VERSION_MAJOR}.${UNIREC_VERSION_MINOR}.${UNIREC_VERSION_PATCH}) @@ -19,9 +19,9 @@ include(CheckCXXCompilerFlag) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules") # Find IPFIXcol and libnf -find_package(IPFIXcol2 2.0.0 REQUIRED) -find_package(LibTrap REQUIRED) -find_package(LibUnirec REQUIRED) +find_package(IPFIXcol2 2.1.0 REQUIRED) # support for basicList is required +find_package(LibTrap 1.13.1 REQUIRED) +find_package(LibUnirec 2.8.0 REQUIRED) # Set default build type if not specified by user if (NOT CMAKE_BUILD_TYPE) From 06f0fe5e7a7f469d516af3781d0240968bf0c646 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Tue, 11 Feb 2020 10:37:10 +0100 Subject: [PATCH 03/16] UniRec output: config file: add structured field examples, align TLS fields --- .../output/unirec/config/unirec-elements.txt | 68 +++++++++++-------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index fb5559c5..445d2c4f 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -4,8 +4,12 @@ # fields. You can change setting by editing this file. Each entry consists # of the following parameters: # - UniRec field name -# - UniRec data type (int{8,16,32,64}, uint{8,16,32,64}, float, double, time, -# ipaddr, macaddr, char, string, bytes) +# - UniRec data type - one of the following: +# int{8,16,32,64}, uint{8,16,32,64}, +# float, double, time, ipaddr, macaddr, char, string, bytes +# int{8,16,32,64}*, uint{8,16,32,64}*, // "array of" types +# float*, double*, time*, ipaddr*, macaddr* // "array of" types +# string_trimmed // trimmed string (i.e. no tailing '\0') # - Comma separated list of IPFIX Information Elements identifiers # ("eXXidYY" where XX is Private Enterprise Number and YY is field ID) # @@ -118,32 +122,36 @@ IPV6_TUN_TYPE uint8 e16982id405 # IPv6 tunnel type APP_ID bytes e0id95 # Application ID from libprotoident / NBAR2 / Flowmon's NBAR plugin # --- Flowmon TLS fields -TLS_CONTENT_TYPE uint8 flowmon:tlsContentType # tlsContentType -TLS_HANDSHAKE_TYPE uint32 flowmon:tlsHandshakeType # https://tools.ietf.org/html/rfc5246#appendix-A.4 -TLS_SETUP_TIME uint64 flowmon:tlsSetupTime # tlsSetupTime -TLS_SERVER_VERSION uint16 flowmon:tlsServerVersion # 8b major and 8b minor, 0x0303 ~ TLS1.2 -TLS_SERVER_RANDOM bytes flowmon:tlsServerRandom # tlsServerRandom -TLS_SERVER_SESSIONID bytes flowmon:tlsServerSessionId # tlsServerSessionId -TLS_CIPHER_SUITE uint16 flowmon:tlsCipherSuite # https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 -TLS_ALPN string flowmon:tlsAlpn # TLS Application-Layer Protocol Negotiation https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids -TLS_SNI string flowmon:tlsSni # Server Name Indication https://en.wikipedia.org/wiki/Server_Name_Indication -TLS_SNI_LENGTH uint16 flowmon:tlsSniLength # Length of TLS_SNI field -TLS_CLIENT_VERSION uint16 flowmon:tlsClientVersion # tlsClientVersion -TLS_CIPHER_SUITES bytes flowmon:tlsCipherSuites # List of 2B ciphers, beware of network byte order. See https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 -TLS_CLIENT_RANDOM bytes flowmon:tlsClientRandom # tlsClientRandom -TLS_CLIENT_SESSIONID bytes flowmon:tlsClientSessionId # tlsClientSessionId -TLS_EXTENSION_TYPES bytes flowmon:tlsExtensionTypes # tlsExtensionTypes -TLS_EXTENSION_LENGTHS bytes flowmon:tlsExtensionLengths # tlsExtensionLengths -TLS_ELLIPTIC_CURVES bytes flowmon:tlsEllipticCurves # tlsEllipticCurves -TLS_EC_POINTFORMATS bytes flowmon:tlsEcPointFormats # tlsEcPointFormats -TLS_CLIENT_KEYLENGTH int32 flowmon:tlsClientKeyLength # Length of client's key -TLS_ISSUER_CN string flowmon:tlsIssuerCn # Common name of certificate issuer -TLS_SUBJECT_CN string flowmon:tlsSubjectCn # Certificate Common Name -TLS_SUBJECT_ON string flowmon:tlsSubjectOn # Certificate Organization Name -TLS_VALIDITY_NOTBEFORE int64 flowmon:tlsValidityNotBefore # UNIX timestamp of certificate creation -TLS_VALIDITY_NOTAFTER int64 flowmon:tlsValidityNotAfter # UNIX timestamp of certificate expiration -TLS_SIGNATURE_ALG uint16 flowmon:tlsSignatureAlg # tlsSignatureAlg -TLS_PUBLIC_KEYALG uint16 flowmon:tlsPublicKeyAlg # tlsPublicKeyAlg -TLS_PUBLIC_KEYLENGTH int32 flowmon:tlsPublicKeyLength # tlsPublicKeyLength -TLS_JA_3FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint +TLS_CONTENT_TYPE uint8 flowmon:tlsContentType # tlsContentType +TLS_HANDSHAKE_TYPE uint32 flowmon:tlsHandshakeType # https://tools.ietf.org/html/rfc5246#appendix-A.4 +TLS_SETUP_TIME uint64 flowmon:tlsSetupTime # tlsSetupTime +TLS_SERVER_VERSION uint16 flowmon:tlsServerVersion # 8b major and 8b minor, 0x0303 ~ TLS1.2 +TLS_SERVER_RANDOM bytes flowmon:tlsServerRandom # tlsServerRandom +TLS_SERVER_SESSIONID bytes flowmon:tlsServerSessionId # tlsServerSessionId +TLS_CIPHER_SUITE uint16 flowmon:tlsCipherSuite # https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 +TLS_ALPN string flowmon:tlsAlpn # TLS Application-Layer Protocol Negotiation https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml#alpn-protocol-ids +TLS_SNI string flowmon:tlsSni # Server Name Indication https://en.wikipedia.org/wiki/Server_Name_Indication +TLS_SNI_LENGTH uint16 flowmon:tlsSniLength # Length of TLS_SNI field +TLS_CLIENT_VERSION uint16 flowmon:tlsClientVersion # tlsClientVersion +TLS_CIPHER_SUITES bytes flowmon:tlsCipherSuites # List of 2B ciphers, beware of network byte order. See https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml#tls-parameters-4 +TLS_CLIENT_RANDOM bytes flowmon:tlsClientRandom # tlsClientRandom +TLS_CLIENT_SESSIONID bytes flowmon:tlsClientSessionId # tlsClientSessionId +TLS_EXTENSION_TYPES bytes flowmon:tlsExtensionTypes # tlsExtensionTypes +TLS_EXTENSION_LENGTHS bytes flowmon:tlsExtensionLengths # tlsExtensionLengths +TLS_ELLIPTIC_CURVES bytes flowmon:tlsEllipticCurves # tlsEllipticCurves +TLS_EC_POINTFORMATS bytes flowmon:tlsEcPointFormats # tlsEcPointFormats +TLS_CLIENT_KEYLENGTH int32 flowmon:tlsClientKeyLength # Length of client's key +TLS_ISSUER_CN string flowmon:tlsIssuerCn # Common name of certificate issuer +TLS_SUBJECT_CN string flowmon:tlsSubjectCn # Certificate Common Name +TLS_SUBJECT_ON string flowmon:tlsSubjectOn # Certificate Organization Name +TLS_VALIDITY_NOTBEFORE int64 flowmon:tlsValidityNotBefore # UNIX timestamp of certificate creation +TLS_VALIDITY_NOTAFTER int64 flowmon:tlsValidityNotAfter # UNIX timestamp of certificate expiration +TLS_SIGNATURE_ALG uint16 flowmon:tlsSignatureAlg # tlsSignatureAlg +TLS_PUBLIC_KEYALG uint16 flowmon:tlsPublicKeyAlg # tlsPublicKeyAlg +TLS_PUBLIC_KEYLENGTH int32 flowmon:tlsPublicKeyLength # tlsPublicKeyLength +TLS_JA_3FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint +# --- Cisco Joy exporter --- +JOY_TLS_RECORD_LENGTHS int16* e0id291/e9id12188 # basicList of packet lengths +JOY_TLS_RECORD_TIMES uint16* e0id291/e9id12189 # basicList of packet timestamps +JOY_TLS_CONTENT_TYPES uint8* e0id291/e9id12190 # basicList of packet types \ No newline at end of file From aa8fe7f4be4bac3c5c0cdbbfe885970c1f76ee6f Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Sun, 15 Mar 2020 22:04:52 +0100 Subject: [PATCH 04/16] UniRec output: added support for UniRec arrays --- extra_plugins/output/unirec/src/map.c | 96 +++- extra_plugins/output/unirec/src/map.h | 3 +- extra_plugins/output/unirec/src/translator.c | 439 ++++++++++++++++++- 3 files changed, 508 insertions(+), 30 deletions(-) diff --git a/extra_plugins/output/unirec/src/map.c b/extra_plugins/output/unirec/src/map.c index d5b96768..bf0620a5 100644 --- a/extra_plugins/output/unirec/src/map.c +++ b/extra_plugins/output/unirec/src/map.c @@ -101,6 +101,13 @@ map_clear(map_t *map) { for (size_t i = 0; i < map->rec_size; ++i) { struct map_rec *rec = map->rec_array[i]; + struct map_ipfix_s *ipfix = rec->ipfix.next; + while (ipfix) { + struct map_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; + } + free(rec->unirec.name); free(rec->unirec.type_str); free(rec); @@ -280,15 +287,48 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, } // Parse IPFIX specifier - const struct fds_iemgr_elem *elem_def = map_elem_get_ipfix(map->iemgr, subtoken); + const struct fds_iemgr_elem *elem_def = NULL; + char *subsave_ptr_list = NULL; + struct map_ipfix_s *ipfix = &rec.ipfix; + rec.ipfix.next = NULL; + for (char *list = subtoken; ;list = NULL) { + char *list_token = strtok_r(list, "/", &subsave_ptr_list); + if (!list_token) { + break; + } + + elem_def = map_elem_get_ipfix(map->iemgr, list_token); + if (elem_def != NULL) { + if (list == NULL) { + ipfix->next = malloc(sizeof(struct map_ipfix_s)); + if (!ipfix->next) { + rc = IPX_ERR_NOMEM; + elem_def = NULL; + break; + } + ipfix = ipfix->next; + } + // Store the "IPFIX element" record + ipfix->source = MAP_SRC_IPFIX; + ipfix->def = elem_def; + ipfix->id = elem_def->id; + ipfix->en = elem_def->scope->pen; + ipfix->next = NULL; + + continue; + } + break; + } if (elem_def != NULL) { - // Store the "IPFIX element" record - rec.ipfix.source = MAP_SRC_IPFIX; - rec.ipfix.def = elem_def; - rec.ipfix.id = elem_def->id; - rec.ipfix.en = elem_def->scope->pen; - rc = map_rec_add(map, &rec); - continue; + rc = map_rec_add(map, &rec); + continue; + } + + ipfix = rec.ipfix.next; + while (ipfix) { + struct map_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; } enum MAP_SRC fn_id = map_elem_get_internal(subtoken); @@ -298,6 +338,7 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, rec.ipfix.def = NULL; rec.ipfix.id = 0; rec.ipfix.en = 0; + rec.ipfix.next = NULL; rc = map_rec_add(map, &rec); continue; } @@ -431,6 +472,24 @@ map_sort_fn(const void *p1, const void *p2) return (rec1->ipfix.id < rec2->ipfix.id) ? (-1) : 1; } + // TODO: better and more readable code + // TODO: check all elements in linked list + if (rec1->ipfix.next != NULL && rec2->ipfix.next != NULL) { + // Primary sort by PEN + if (rec1->ipfix.next->en != rec2->ipfix.next->en) { + return (rec1->ipfix.next->en < rec2->ipfix.next->en) ? (-1) : 1; + } + + // Secondary sort by ID + if (rec1->ipfix.next->id != rec2->ipfix.next->id) { + return (rec1->ipfix.next->id < rec2->ipfix.next->id) ? (-1) : 1; + } + } else if (rec1->ipfix.next != NULL) { + return -1; + } else if (rec2->ipfix.next != NULL) { + return 1; + } + return 0; } @@ -507,9 +566,24 @@ map_load(map_t *map, const char *file) continue; } - if (rec_prev->ipfix.en != rec_now->ipfix.en || rec_prev->ipfix.id != rec_now->ipfix.id) { - rec_prev = rec_now; - continue; + bool collision = false; + const struct map_ipfix_s *ipfix_prev = &rec_prev->ipfix; + const struct map_ipfix_s *ipfix_now = &rec_now->ipfix; + while (1) { + if (ipfix_prev->en != ipfix_now->en || ipfix_prev->id != ipfix_now->id || !ipfix_prev || !ipfix_now) { + rec_prev = rec_now; + break; + } + + ipfix_prev = ipfix_prev->next; + ipfix_now = ipfix_now->next; + if (!ipfix_prev && !ipfix_now) { + collision = true; + break; + } + } + if (!collision) { + continue; } // Collision detected! diff --git a/extra_plugins/output/unirec/src/map.h b/extra_plugins/output/unirec/src/map.h index 35c4cad1..1bc8eda5 100644 --- a/extra_plugins/output/unirec/src/map.h +++ b/extra_plugins/output/unirec/src/map.h @@ -68,7 +68,7 @@ enum MAP_FLAGS { /** IPFIX-to-UniRec mapping record */ struct map_rec { - struct { + struct map_ipfix_s { /** * \brief Data source * \note If the field is not ::MAP_SRC_IPFIX, parameters en, id and def are NOT defined! @@ -81,6 +81,7 @@ struct map_rec { uint16_t id; /** Definition of the IE (MUST not be NULL) */ const struct fds_iemgr_elem *def; + struct map_ipfix_s *next; } ipfix; /**< IPFIX specific parameters */ struct { diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index 899bf9f6..6d37a2ea 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -68,7 +68,7 @@ typedef int (*translator_func)(translator_t *trans, const struct translator_rec /** Translator record */ struct translator_rec { - struct { + struct tr_ipfix_s { /** Private Enterprise number */ uint32_t pen; /** Information Element ID */ @@ -77,6 +77,7 @@ struct translator_rec { enum fds_iemgr_element_type type; /** Field semantic */ enum fds_iemgr_element_semantic sem; + struct tr_ipfix_s *next; } ipfix; /** IPFIX field identification */ struct { @@ -303,6 +304,291 @@ translate_int(translator_t *trans, const struct translator_rec *rec, return 0; } +static int +translate_array_uint(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + uint64_t value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (fds_get_uint_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + return 1; // Conversion failed + } + + switch (rec->unirec.type) { + case UR_TYPE_A_UINT64: + *((uint64_t *) field_ptr) = value; + break; + case UR_TYPE_A_UINT32: + UINT_CONV(uint32_t, field_ptr, value, UINT32_MAX, ipx_sem); + break; + case UR_TYPE_A_UINT16: + UINT_CONV(uint16_t, field_ptr, value, UINT16_MAX, ipx_sem); + break; + case UR_TYPE_A_UINT8: + UINT_CONV(uint8_t, field_ptr, value, UINT8_MAX, ipx_sem); + break; + case UR_TYPE_A_INT64: + UINT_CONV(int64_t, field_ptr, value, INT64_MAX, ipx_sem); + break; + case UR_TYPE_A_INT32: + UINT_CONV(int32_t, field_ptr, value, INT32_MAX, ipx_sem); + break; + case UR_TYPE_A_INT16: + UINT_CONV(int16_t, field_ptr, value, INT16_MAX, ipx_sem); + break; + case UR_TYPE_A_INT8: + UINT_CONV(int8_t, field_ptr, value, INT8_MAX, ipx_sem); + break; + default: + return 1; // Unsupported data type + } + } + + return rc != FDS_OK; +} + +static int +translate_array_int(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + int64_t value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (fds_get_int_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + return 1; // Conversion failed + } + + switch (rec->unirec.type) { + case UR_TYPE_A_INT64: + *((int64_t *) field_ptr) = value; + break; + case UR_TYPE_A_INT32: + INT_CONV(int32_t, field_ptr, value, INT32_MIN, INT32_MAX, ipx_sem); + break; + case UR_TYPE_A_INT16: + INT_CONV(int16_t, field_ptr, value, INT16_MIN, INT16_MAX, ipx_sem); + break; + case UR_TYPE_A_INT8: + INT_CONV(int8_t, field_ptr, value, INT8_MIN, INT8_MAX, ipx_sem); + break; + case UR_TYPE_A_UINT64: + INT_CONV(uint64_t, field_ptr, value, 0, INT64_MAX, ipx_sem); // Must be int64! + break; + case UR_TYPE_A_UINT32: + INT_CONV(uint32_t, field_ptr, value, 0, UINT32_MAX, ipx_sem); + break; + case UR_TYPE_A_UINT16: + INT_CONV(uint16_t, field_ptr, value, 0, UINT16_MAX, ipx_sem); + break; + case UR_TYPE_A_UINT8: + INT_CONV(uint8_t, field_ptr, value, 0, UINT8_MAX, ipx_sem); + break; + default: + return 1; // Unsupported data type + } + } + + return rc != FDS_OK; +} + +/** + * \brief Convert IPFIX boolean to UniRec char/(un)signed integer + * \copydetails translate_uint() + */ +static int +translate_array_bool(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + bool value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (fds_get_bool(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + return 1; // Conversion failed + } + + const uint8_t res = value ? 1U : 0U; + switch (ur_array_get_elem_size(ur_id)) { + case 1: + *((uint8_t *) field_ptr) = res; + break; + case 2: + *((uint16_t *) field_ptr) = res; + break; + case 4: + *((uint32_t *) field_ptr) = res; + break; + case 8: + *((uint64_t *) field_ptr) = res; + break; + default: + // Invalid size of the field + return 1; + } + } + + return rc != FDS_OK; +} + +/** + * \brief Convert IPFIX float to UniRec float + * \copydetails translate_uint() + */ +static int +translate_array_float(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + double value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (fds_get_float_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + return 1; // Conversion failed + } + + switch (rec->unirec.type) { + case UR_TYPE_A_FLOAT: + if (value < -FLT_MAX && isnormal(value)) { + *((float *) field_ptr) = -FLT_MAX; + } else if (value > FLT_MAX && isnormal(value)) { + *((float *) field_ptr) = FLT_MAX; + } else { + *((float *) field_ptr) = (float) value; + } + break; + case UR_TYPE_A_DOUBLE: + *((double *) field_ptr) = value; + break; + default: + // Invalid type of the field + return 1; + } + } + + return rc != FDS_OK; +} + +/** + * \brief Convert IPFIX IPv4/IPv6 address to UniRec IPv4/IPv6 address + * \copydetails translate_uint() + */ +static int +translate_array_ip(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + + switch (list_it.field.size) { + case 4: // IPv4 + *((ip_addr_t *) field_ptr) = ip_from_4_bytes_be((char *) list_it.field.data); + break; + case 16: // IPv6 + *((ip_addr_t *) field_ptr) = ip_from_16_bytes_be((char *) list_it.field.data); + break; + default: + // Invalid size of the field + return 1; + } + } + + return rc != FDS_OK; +} + +/** + * \brief Convert IPFIX MAC address to UniRec MAC address + * \copydetails translate_uint() + */ +static int +translate_array_mac(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + if (list_it.field.size != 6U) { + return 1; + } + + ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + memcpy(field_ptr, list_it.field.data, 6U); + } + + return rc != FDS_OK; +} + +/** + * \brief Convert IPFIX timestamp to UniRec timestamp + * \copydetails translate_uint() + */ +static int +translate_array_time(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_type type_ipx = rec->ipfix.next->type; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + // Get the value + ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (type_ipx == FDS_ET_DATE_TIME_MILLISECONDS || type_ipx == FDS_ET_DATE_TIME_SECONDS) { + // Low precision timestamp + uint64_t ts; + if (fds_get_datetime_lp_be(list_it.field.data, list_it.field.size, type_ipx, &ts) != FDS_OK) { + return 1; + } + + *field_ptr = ur_time_from_sec_msec(ts / 1000, ts % 1000); + continue; + } else if (type_ipx == FDS_ET_DATE_TIME_MICROSECONDS || type_ipx == FDS_ET_DATE_TIME_NANOSECONDS) { + // High precision timestamp + struct timespec ts; + if (fds_get_datetime_hp_be(list_it.field.data, list_it.field.size, type_ipx, &ts) != FDS_OK) { + return 1; + } + + *field_ptr = ur_time_from_sec_msec(ts.tv_sec, ts.tv_nsec / 1000000); + continue; + } + return 1; + } + + return rc != FDS_OK; +} + /** * \brief Convert IPFIX bytes/string to UniRec bytes/string * \copydetails translate_uint() @@ -579,17 +865,27 @@ translator_size_ur_int(ur_field_type_t type) case UR_TYPE_CHAR: case UR_TYPE_UINT8: case UR_TYPE_INT8: + case UR_TYPE_A_UINT8: + case UR_TYPE_A_INT8: return 1U; case UR_TYPE_UINT16: case UR_TYPE_INT16: + case UR_TYPE_A_UINT16: + case UR_TYPE_A_INT16: return 2U; case UR_TYPE_UINT32: case UR_TYPE_INT32: case UR_TYPE_FLOAT: + case UR_TYPE_A_UINT32: + case UR_TYPE_A_INT32: + case UR_TYPE_A_FLOAT: return 4U; case UR_TYPE_UINT64: case UR_TYPE_INT64: case UR_TYPE_DOUBLE: + case UR_TYPE_A_UINT64: + case UR_TYPE_A_INT64: + case UR_TYPE_A_DOUBLE: return 8U; default: return 0U; @@ -632,11 +928,18 @@ translator_size_ipx_int(enum fds_iemgr_element_type type) * \return Pointer to the function or NULL (i.e. conversion not supported) */ static translator_func -translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) +translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec, bool is_array) { ur_field_type_t type_ur = rec->unirec.type; enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; + if (is_array) { + if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { + return NULL; + } + type_ipx = rec->ipfix.next->def->data_type; + } + uint16_t size_ur = translator_size_ur_int(type_ur); uint16_t size_ipx = translator_size_ipx_int(type_ipx); if (size_ur == 0 || size_ipx == 0) { @@ -647,13 +950,21 @@ translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) bool warn_msg; if (fds_iemgr_is_type_unsigned(type_ipx)) { - fn = translate_uint; + if (is_array) { + fn = translate_array_uint; + } else { + fn = translate_uint; + } // Check possible conversion errors bool is_signed = (type_ur == UR_TYPE_INT8 || type_ur == UR_TYPE_INT16 || type_ur == UR_TYPE_INT32 || type_ur == UR_TYPE_INT64); warn_msg = ((is_signed && size_ur <= size_ipx) || (!is_signed && size_ur < size_ipx)); } else if (fds_iemgr_is_type_signed(type_ipx)) { - fn = translate_int; + if (is_array) { + fn = translate_array_int; + } else { + fn = translate_int; + } // Check possible conversion errors bool is_unsigned = (type_ur == UR_TYPE_UINT8 || type_ur == UR_TYPE_UINT16 || type_ur == UR_TYPE_UINT32 || type_ur == UR_TYPE_UINT64); @@ -664,6 +975,9 @@ translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) if (warn_msg) { const struct fds_iemgr_elem *el = rec->ipfix.def; + if (is_array) { + el = rec->ipfix.next->def; + } IPX_CTX_WARNING(ctx, "Conversion from IPFIX IE '%s:%s' (%s) to UniRec '%s' (%s) may alter " "its value!", el->scope->name, el->name, fds_iemgr_type2str(type_ipx), rec->unirec.name, rec->unirec.type_str); @@ -679,11 +993,18 @@ translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec) * \return Pointer to the function or NULL (i.e. conversion not supported) */ static translator_func -translator_get_float_func(ipx_ctx_t *ctx, const struct map_rec *rec) +translator_get_float_func(ipx_ctx_t *ctx, const struct map_rec *rec, bool is_array) { ur_field_type_t type_ur = rec->unirec.type; enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; + if (is_array) { + if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { + return NULL; + } + type_ipx = rec->ipfix.next->def->data_type; + } + uint16_t size_ur = translator_size_ur_int(type_ur); uint16_t size_ipx = translator_size_ipx_int(type_ipx); if (size_ur == 0 || size_ipx == 0) { @@ -692,12 +1013,15 @@ translator_get_float_func(ipx_ctx_t *ctx, const struct map_rec *rec) if (size_ur < size_ipx) { const struct fds_iemgr_elem *el = rec->ipfix.def; + if (is_array) { + el = rec->ipfix.next->def; + } IPX_CTX_WARNING(ctx, "Conversion from IPFIX IE '%s:%s' (%s) to UniRec '%s' (%s) may alter " "its value!", el->scope->name, el->name, fds_iemgr_type2str(type_ipx), rec->unirec.name, rec->unirec.type_str); } - return translate_float; + return (is_array ? translate_array_float : translate_float); } /** @@ -713,6 +1037,10 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) ur_field_type_t type_ur = rec->unirec.type; enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; + if (rec->ipfix.next) { + type_ipx = rec->ipfix.next->def->data_type; + } + switch (type_ur) { case UR_TYPE_STRING: // String array @@ -737,7 +1065,7 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) case UR_TYPE_UINT64: // Char and (un)signed integer if (fds_iemgr_is_type_unsigned(type_ipx) || fds_iemgr_is_type_signed(type_ipx)) { - return translator_get_numeric_func(ctx, rec); + return translator_get_numeric_func(ctx, rec, false); } else if (type_ipx == FDS_ET_BOOLEAN) { return translate_bool; } @@ -746,7 +1074,7 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) case UR_TYPE_DOUBLE: // Floating-point number if (fds_iemgr_is_type_float(type_ipx)) { - return translator_get_float_func(ctx, rec); + return translator_get_float_func(ctx, rec, false); } break; case UR_TYPE_IP: @@ -767,6 +1095,46 @@ translator_get_func(ipx_ctx_t *ctx, const struct map_rec *rec) return translate_time; } break; + case UR_TYPE_A_INT8: + case UR_TYPE_A_INT16: + case UR_TYPE_A_INT32: + case UR_TYPE_A_INT64: + case UR_TYPE_A_UINT8: + case UR_TYPE_A_UINT16: + case UR_TYPE_A_UINT32: + case UR_TYPE_A_UINT64: + // Char and (un)signed integer + if (fds_iemgr_is_type_unsigned(type_ipx) || fds_iemgr_is_type_signed(type_ipx)) { + return translator_get_numeric_func(ctx, rec, true); + } else if (type_ipx == FDS_ET_BOOLEAN) { + return translate_array_bool; + } + break; + case UR_TYPE_A_FLOAT: + case UR_TYPE_A_DOUBLE: + // Floating-point number + if (fds_iemgr_is_type_float(type_ipx)) { + return translator_get_float_func(ctx, rec, true); + } + break; + case UR_TYPE_A_IP: + // IP addresses + if (fds_iemgr_is_type_ip(type_ipx)) { + return translate_array_ip; + } + break; + case UR_TYPE_A_MAC: + // MAC address + if (type_ipx == FDS_ET_MAC_ADDRESS) { + return translate_array_mac; + } + break; + case UR_TYPE_A_TIME: + // Timestamp + if (fds_iemgr_is_type_time(type_ipx)) { + return translate_array_time; + } + break; default: break; } @@ -937,15 +1305,31 @@ translator_destroy_record(translator_t *trans) static int translator_cmp(const void *p1, const void *p2) { - const struct translator_rec *elem1, *elem2; - elem1 = (const struct translator_rec *) p1; - elem2 = (const struct translator_rec *) p2; + const struct tr_ipfix_s *ipfix1 = &((const struct translator_rec *) p1)->ipfix; + const struct tr_ipfix_s *ipfix2 = &((const struct translator_rec *) p2)->ipfix; - uint64_t elem1_val = ((uint64_t) elem1->ipfix.pen) << 16 | elem1->ipfix.id; - uint64_t elem2_val = ((uint64_t) elem2->ipfix.pen) << 16 | elem2->ipfix.id; + uint64_t elem1_val = ((uint64_t) ipfix1->pen) << 16 | ipfix1->id; + uint64_t elem2_val = ((uint64_t) ipfix2->pen) << 16 | ipfix2->id; if (elem1_val == elem2_val) { - return 0; + // TODO: check all ipfix elements + + ipfix1 = ipfix1->next; + ipfix2 = ipfix2->next; + if (ipfix1 && ipfix2) { + elem1_val = ((uint64_t) ipfix1->pen) << 16 | ipfix1->id; + elem2_val = ((uint64_t) ipfix2->pen) << 16 | ipfix2->id; + if (elem1_val == elem2_val) { + return 0; + } else { + return (elem1_val < elem2_val) ? (-1) : 1; + } + } else if (ipfix1) { + return -1; + } else if (ipfix2) { + return 1; + } + return 0; } else { return (elem1_val < elem2_val) ? (-1) : 1; } @@ -988,10 +1372,21 @@ translator_table_fill_rec(translator_t *trans, const struct map_rec *map_rec, trans_rec->unirec.size = ur_get_size(ur_id); trans_rec->unirec.type = ur_get_type(ur_id); trans_rec->unirec.req_idx = field_idx; - trans_rec->ipfix.pen = map_rec->ipfix.en; - trans_rec->ipfix.id = map_rec->ipfix.id; - trans_rec->ipfix.type = map_rec->ipfix.def->data_type; - trans_rec->ipfix.sem = map_rec->ipfix.def->data_semantic; + + const struct map_ipfix_s *ipfix = &map_rec->ipfix; + struct tr_ipfix_s *tr_ipfix = &trans_rec->ipfix; + while (ipfix) { + tr_ipfix->pen = ipfix->en; + tr_ipfix->id = ipfix->id; + tr_ipfix->type = ipfix->def->data_type; + tr_ipfix->sem = ipfix->def->data_semantic; + if (ipfix->next) { + tr_ipfix->next = malloc(sizeof(struct tr_ipfix_s)); + } else { + tr_ipfix->next = NULL; + } + ipfix = ipfix->next; + } IPX_CTX_DEBUG(trans->ctx, "Added conversion from IPFIX IE '%s:%s' to UniRec '%s'", map_rec->ipfix.def->scope->name, map_rec->ipfix.def->name, map_rec->unirec.name); @@ -1181,6 +1576,14 @@ translator_init_table(translator_t *trans, const map_t *map, const ur_template_t static void translator_destroy_table(translator_t *trans) { + for (size_t i = 0; i < trans->table.size; ++i) { + struct tr_ipfix_s *ipfix = trans->table.recs[i].ipfix.next; + while (ipfix) { + struct tr_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; + } + } free(trans->table.recs); } From 32f03d55903dc74ec9bb06abf546292569e63317 Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Wed, 18 Mar 2020 22:41:39 +0100 Subject: [PATCH 05/16] UniRec output: all elements of IPFIX linked list are now checked in both map and conversion comparator functions --- extra_plugins/output/unirec/src/map.c | 45 +++++++++----------- extra_plugins/output/unirec/src/translator.c | 35 +++++++-------- 2 files changed, 34 insertions(+), 46 deletions(-) diff --git a/extra_plugins/output/unirec/src/map.c b/extra_plugins/output/unirec/src/map.c index bf0620a5..de954d0a 100644 --- a/extra_plugins/output/unirec/src/map.c +++ b/extra_plugins/output/unirec/src/map.c @@ -455,39 +455,32 @@ map_load_line(map_t *map, const char *line, size_t line_id) static int map_sort_fn(const void *p1, const void *p2) { - struct map_rec *rec1 = *(struct map_rec **) p1; - struct map_rec *rec2 = *(struct map_rec **) p2; + const struct map_ipfix_s *ipfix1 = &(*(struct map_rec **) p1)->ipfix; + const struct map_ipfix_s *ipfix2 = &(*(struct map_rec **) p2)->ipfix; - if (rec1->ipfix.source != rec2->ipfix.source) { - return (rec1->ipfix.source < rec2->ipfix.source) ? (-1) : 1; - } - - // Primary sort by PEN - if (rec1->ipfix.en != rec2->ipfix.en) { - return (rec1->ipfix.en < rec2->ipfix.en) ? (-1) : 1; - } - - // Secondary sort by ID - if (rec1->ipfix.id != rec2->ipfix.id) { - return (rec1->ipfix.id < rec2->ipfix.id) ? (-1) : 1; - } + while (ipfix1 && ipfix2) { + if (ipfix1->source != ipfix2->source) { + return (ipfix1->source < ipfix2->source) ? (-1) : 1; + } - // TODO: better and more readable code - // TODO: check all elements in linked list - if (rec1->ipfix.next != NULL && rec2->ipfix.next != NULL) { // Primary sort by PEN - if (rec1->ipfix.next->en != rec2->ipfix.next->en) { - return (rec1->ipfix.next->en < rec2->ipfix.next->en) ? (-1) : 1; + if (ipfix1->en != ipfix2->en) { + return (ipfix1->en < ipfix2->en) ? (-1) : 1; } // Secondary sort by ID - if (rec1->ipfix.next->id != rec2->ipfix.next->id) { - return (rec1->ipfix.next->id < rec2->ipfix.next->id) ? (-1) : 1; + if (ipfix1->id != ipfix2->id) { + return (ipfix1->id < ipfix2->id) ? (-1) : 1; } - } else if (rec1->ipfix.next != NULL) { - return -1; - } else if (rec2->ipfix.next != NULL) { - return 1; + + ipfix1 = ipfix1->next; + ipfix2 = ipfix2->next; + } + + if (ipfix1->next) { + return -1; + } else if (ipfix2->next) { + return 1; } return 0; diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index 6d37a2ea..b72b3638 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -1308,31 +1308,26 @@ translator_cmp(const void *p1, const void *p2) const struct tr_ipfix_s *ipfix1 = &((const struct translator_rec *) p1)->ipfix; const struct tr_ipfix_s *ipfix2 = &((const struct translator_rec *) p2)->ipfix; - uint64_t elem1_val = ((uint64_t) ipfix1->pen) << 16 | ipfix1->id; - uint64_t elem2_val = ((uint64_t) ipfix2->pen) << 16 | ipfix2->id; + while (ipfix1 && ipfix2) { + uint64_t elem1_val = ((uint64_t) ipfix1->pen) << 16 | ipfix1->id; + uint64_t elem2_val = ((uint64_t) ipfix2->pen) << 16 | ipfix2->id; - if (elem1_val == elem2_val) { - // TODO: check all ipfix elements + if (elem1_val < elem2_val) { + return -1; + } else if (elem1_val > elem2_val) { + return 1; + } ipfix1 = ipfix1->next; ipfix2 = ipfix2->next; - if (ipfix1 && ipfix2) { - elem1_val = ((uint64_t) ipfix1->pen) << 16 | ipfix1->id; - elem2_val = ((uint64_t) ipfix2->pen) << 16 | ipfix2->id; - if (elem1_val == elem2_val) { - return 0; - } else { - return (elem1_val < elem2_val) ? (-1) : 1; - } - } else if (ipfix1) { - return -1; - } else if (ipfix2) { - return 1; - } - return 0; - } else { - return (elem1_val < elem2_val) ? (-1) : 1; } + + if (ipfix1) { + return -1; + } else if (ipfix2) { + return 1; + } + return 0; } /** From 4ea806b3f347f53480c874576369bb875afc90f5 Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Wed, 18 Mar 2020 22:42:20 +0100 Subject: [PATCH 06/16] UniRec output: added missing documentation --- extra_plugins/output/unirec/src/map.h | 1 + extra_plugins/output/unirec/src/translator.c | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/extra_plugins/output/unirec/src/map.h b/extra_plugins/output/unirec/src/map.h index 1bc8eda5..04c2caee 100644 --- a/extra_plugins/output/unirec/src/map.h +++ b/extra_plugins/output/unirec/src/map.h @@ -81,6 +81,7 @@ struct map_rec { uint16_t id; /** Definition of the IE (MUST not be NULL) */ const struct fds_iemgr_elem *def; + /** Points to an element of list, NULL if field is not list. */ struct map_ipfix_s *next; } ipfix; /**< IPFIX specific parameters */ diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index b72b3638..870bf428 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -77,6 +77,7 @@ struct translator_rec { enum fds_iemgr_element_type type; /** Field semantic */ enum fds_iemgr_element_semantic sem; + /** Pointer to element of IPFIX list */ struct tr_ipfix_s *next; } ipfix; /** IPFIX field identification */ @@ -304,6 +305,10 @@ translate_int(translator_t *trans, const struct translator_rec *rec, return 0; } +/** + * \brief Convert IPFIX list of uint elements to UniRec uint array + * \copydetails translate_uint() + */ static int translate_array_uint(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) @@ -354,6 +359,10 @@ translate_array_uint(translator_t *trans, const struct translator_rec *rec, return rc != FDS_OK; } +/** + * \brief Convert IPFIX list of int elements to UniRec int array + * \copydetails translate_uint() + */ static int translate_array_int(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) @@ -405,7 +414,7 @@ translate_array_int(translator_t *trans, const struct translator_rec *rec, } /** - * \brief Convert IPFIX boolean to UniRec char/(un)signed integer + * \brief Convert IPFIX list of bool elements to UniRec bool array * \copydetails translate_uint() */ static int @@ -448,7 +457,7 @@ translate_array_bool(translator_t *trans, const struct translator_rec *rec, } /** - * \brief Convert IPFIX float to UniRec float + * \brief Convert IPFIX list of float elements to UniRec float array * \copydetails translate_uint() */ static int @@ -490,7 +499,7 @@ translate_array_float(translator_t *trans, const struct translator_rec *rec, } /** - * \brief Convert IPFIX IPv4/IPv6 address to UniRec IPv4/IPv6 address + * \brief Convert IPFIX list of IPv4/IPv6 elements to UniRec IP array * \copydetails translate_uint() */ static int @@ -522,7 +531,7 @@ translate_array_ip(translator_t *trans, const struct translator_rec *rec, } /** - * \brief Convert IPFIX MAC address to UniRec MAC address + * \brief Convert IPFIX list of MAC elements to UniRec MAC array * \copydetails translate_uint() */ static int @@ -547,7 +556,7 @@ translate_array_mac(translator_t *trans, const struct translator_rec *rec, } /** - * \brief Convert IPFIX timestamp to UniRec timestamp + * \brief Convert IPFIX list of timestamp elements to UniRec timestamp array * \copydetails translate_uint() */ static int From a075dbc961392ccff4cc2ff456e435f946fdb441 Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Tue, 24 Mar 2020 17:16:02 +0100 Subject: [PATCH 07/16] UniRec output: code refactoring --- extra_plugins/output/unirec/src/translator.c | 650 +++++++++---------- 1 file changed, 307 insertions(+), 343 deletions(-) diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index 870bf428..e360cd2a 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -172,27 +172,10 @@ struct translator_s { *((dst_type *) (dst_ptr)) = (dst_type) (src_val); \ } -/** - * \brief Convert IPFIX unsigned integer to UniRec (un)signed integer - * \param[in,out] trans Translator instance (UniRec record will be modified) - * \param[in] rec Translator record (description of source IPFIX and destination UniRec fields) - * \param[in] field IPFIX field data - * \return On success returns 0. Otherwise returns a non-zero value. - */ -static int -translate_uint(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) +int +translator_store_uint(ur_field_type_t ur_type, void *field_ptr, uint64_t value, const enum fds_iemgr_element_semantic ipx_sem) { - uint64_t value; - if (fds_get_uint_be(field->data, field->size, &value) != FDS_OK) { - return 1; // Conversion failed - } - - ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; - - switch (rec->unirec.type) { + switch (ur_type) { case UR_TYPE_UINT64: *((uint64_t *) field_ptr) = value; break; @@ -221,6 +204,32 @@ translate_uint(translator_t *trans, const struct translator_rec *rec, default: return 1; // Unsupported data type } + return 0; +} + +/** + * \brief Convert IPFIX unsigned integer to UniRec (un)signed integer + * \param[in,out] trans Translator instance (UniRec record will be modified) + * \param[in] rec Translator record (description of source IPFIX and destination UniRec fields) + * \param[in] field IPFIX field data + * \return On success returns 0. Otherwise returns a non-zero value. + */ +static int +translate_uint(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + uint64_t value; + if (fds_get_uint_be(field->data, field->size, &value) != FDS_OK) { + return 1; // Conversion failed + } + + ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; + + if (translator_store_uint(ur_id, field_ptr, value, ipx_sem)) { + return 1; + } return 0; } @@ -255,24 +264,10 @@ translate_uint(translator_t *trans, const struct translator_rec *rec, *((dst_type *) (dst_ptr)) = (dst_type) (src_val); \ } -/** - * \brief Convert IPFIX signed integer to UniRec (un)signed integer - * \copydetails translate_uint() - */ -static int -translate_int(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) +int +translator_store_int(ur_field_type_t ur_type, void *field_ptr, int64_t value, const enum fds_iemgr_element_semantic ipx_sem) { - int64_t value; - if (fds_get_int_be(field->data, field->size, &value) != FDS_OK) { - return 1; // Conversion failed - } - - ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; - - switch (rec->unirec.type) { + switch (ur_type) { case UR_TYPE_INT64: *((int64_t *) field_ptr) = value; break; @@ -301,6 +296,253 @@ translate_int(translator_t *trans, const struct translator_rec *rec, default: return 1; // Unsupported data type } + return 0; +} + +/** + * \brief Convert IPFIX signed integer to UniRec (un)signed integer + * \copydetails translate_uint() + */ +static int +translate_int(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + int64_t value; + if (fds_get_int_be(field->data, field->size, &value) != FDS_OK) { + return 1; // Conversion failed + } + + ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; + + if (translator_store_int(ur_id, field_ptr, value, ipx_sem)) { + return 1; + } + + return 0; +} + +/** + * \brief Convert IPFIX bytes/string to UniRec bytes/string + * \copydetails translate_uint() + */ +static int +translate_bytes(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + ur_set_var(trans->record.ur_tmplt, trans->record.data, ur_id, field->data, field->size); + return 0; +} + +/** + * \brief Convert IPFIX string to trimmed UniRec string + * + * The function will copy only characters up to the first occurrence of '\0' (excluding). + * \copydetails translate_uint() + */ +static int +translator_string_trim(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + ur_field_id_t ur_id = rec->unirec.id; + size_t copy_len = strnlen((const char *) field->data, field->size); + ur_set_var(trans->record.ur_tmplt, trans->record.data, ur_id, field->data, copy_len); + return 0; +} + +int +translator_store_bool(int ur_size, void *field_ptr, bool value) +{ + const uint8_t res = value ? 1U : 0U; + switch (ur_size) { + case 1: + *((uint8_t *) field_ptr) = res; + break; + case 2: + *((uint16_t *) field_ptr) = res; + break; + case 4: + *((uint32_t *) field_ptr) = res; + break; + case 8: + *((uint64_t *) field_ptr) = res; + break; + default: + // Invalid size of the field + return 1; + } + return 0; +} + + +/** + * \brief Convert IPFIX boolean to UniRec char/(un)signed integer + * \copydetails translate_uint() + */ +static int +translate_bool(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + bool value; + if (fds_get_bool(field->data, field->size, &value) != FDS_OK) { + return 1; + } + + const ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (translator_store_bool(rec->unirec.size, field_ptr, value)) { + return 1; + } + + return 0; +} + +int +translator_store_float(ur_field_type_t ur_type, void *field_ptr, double value) +{ + switch (ur_type) { + case UR_TYPE_FLOAT: + if (value < -FLT_MAX && isnormal(value)) { + *((float *) field_ptr) = -FLT_MAX; + } else if (value > FLT_MAX && isnormal(value)) { + *((float *) field_ptr) = FLT_MAX; + } else { + *((float *) field_ptr) = (float) value; + } + break; + case UR_TYPE_DOUBLE: + *((double *) field_ptr) = value; + break; + default: + // Invalid type of the field + return 1; + } + + return 0; +} + +/** + * \brief Convert IPFIX float to UniRec float + * \copydetails translate_uint() + */ +static int +translate_float(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + double value; + if (fds_get_float_be(field->data, field->size, &value) != FDS_OK) { + return 1; // Conversion failed + } + + const ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (translator_store_float(rec->unirec.type, field_ptr, value)) { + return 1; + } + + return 0; +} + +int +translator_store_ip(uint8_t *ip_bytes, uint16_t data_size, void *field_ptr) +{ + switch (data_size) { + case 4: // IPv4 + *((ip_addr_t *) field_ptr) = ip_from_4_bytes_be((char *) ip_bytes); + break; + case 16: // IPv6 + *((ip_addr_t *) field_ptr) = ip_from_16_bytes_be((char *) ip_bytes); + break; + default: + // Invalid size of the field + return 1; + } + + return 0; +} + +/** + * \brief Convert IPFIX IPv4/IPv6 address to UniRec IPv4/IPv6 address + * \copydetails translate_uint() + */ +static int +translate_ip(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + const ur_field_id_t ur_id = rec->unirec.id; + void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (translator_store_ip(field->data, field->size, field_ptr)) { + return 1; + } + + return 0; +} + +/** + * \brief Convert IPFIX MAC address to UniRec MAC address + * \copydetails translate_uint() + */ +static int +translate_mac(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + if (field->size != 6U) { + return 1; + } + + const ur_field_id_t ur_id = rec->unirec.id; + ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + memcpy(field_ptr, field->data, 6U); + return 0; +} + +int +translator_store_time(const enum fds_iemgr_element_type type_ipx, const uint8_t *time_bytes, uint16_t time_size, ur_time_t *field_ptr) +{ + if (type_ipx == FDS_ET_DATE_TIME_MILLISECONDS || type_ipx == FDS_ET_DATE_TIME_SECONDS) { + // Low precision timestamp + uint64_t ts; + if (fds_get_datetime_lp_be(time_bytes, time_size, type_ipx, &ts) != FDS_OK) { + return 1; + } + + *field_ptr = ur_time_from_sec_msec(ts / 1000, ts % 1000); + return 0; + } else if (type_ipx == FDS_ET_DATE_TIME_MICROSECONDS || type_ipx == FDS_ET_DATE_TIME_NANOSECONDS) { + // High precision timestamp + struct timespec ts; + if (fds_get_datetime_hp_be(time_bytes, time_size, type_ipx, &ts) != FDS_OK) { + return 1; + } + + *field_ptr = ur_time_from_sec_msec(ts.tv_sec, ts.tv_nsec / 1000000); + return 0; + } + + return 1; +} + +/** + * \brief Convert IPFIX timestamp to UniRec timestamp + * \copydetails translate_uint() + */ +static int +translate_time(translator_t *trans, const struct translator_rec *rec, + const struct fds_drec_field *field) +{ + // Get the value + const enum fds_iemgr_element_type type_ipx = rec->ipfix.type; + const ur_field_id_t ur_id = rec->unirec.id; + ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + + if (translator_store_time(type_ipx, field->data, field->size, field_ptr)) { + return 1; + } return 0; } @@ -323,40 +565,17 @@ translate_array_uint(translator_t *trans, const struct translator_rec *rec, uint64_t value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); if (fds_get_uint_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } - switch (rec->unirec.type) { - case UR_TYPE_A_UINT64: - *((uint64_t *) field_ptr) = value; - break; - case UR_TYPE_A_UINT32: - UINT_CONV(uint32_t, field_ptr, value, UINT32_MAX, ipx_sem); - break; - case UR_TYPE_A_UINT16: - UINT_CONV(uint16_t, field_ptr, value, UINT16_MAX, ipx_sem); - break; - case UR_TYPE_A_UINT8: - UINT_CONV(uint8_t, field_ptr, value, UINT8_MAX, ipx_sem); - break; - case UR_TYPE_A_INT64: - UINT_CONV(int64_t, field_ptr, value, INT64_MAX, ipx_sem); - break; - case UR_TYPE_A_INT32: - UINT_CONV(int32_t, field_ptr, value, INT32_MAX, ipx_sem); - break; - case UR_TYPE_A_INT16: - UINT_CONV(int16_t, field_ptr, value, INT16_MAX, ipx_sem); - break; - case UR_TYPE_A_INT8: - UINT_CONV(int8_t, field_ptr, value, INT8_MAX, ipx_sem); - break; - default: - return 1; // Unsupported data type + if (translator_store_uint(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; } } - return rc != FDS_OK; + return 0; } /** @@ -377,40 +596,17 @@ translate_array_int(translator_t *trans, const struct translator_rec *rec, int64_t value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); if (fds_get_int_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } - switch (rec->unirec.type) { - case UR_TYPE_A_INT64: - *((int64_t *) field_ptr) = value; - break; - case UR_TYPE_A_INT32: - INT_CONV(int32_t, field_ptr, value, INT32_MIN, INT32_MAX, ipx_sem); - break; - case UR_TYPE_A_INT16: - INT_CONV(int16_t, field_ptr, value, INT16_MIN, INT16_MAX, ipx_sem); - break; - case UR_TYPE_A_INT8: - INT_CONV(int8_t, field_ptr, value, INT8_MIN, INT8_MAX, ipx_sem); - break; - case UR_TYPE_A_UINT64: - INT_CONV(uint64_t, field_ptr, value, 0, INT64_MAX, ipx_sem); // Must be int64! - break; - case UR_TYPE_A_UINT32: - INT_CONV(uint32_t, field_ptr, value, 0, UINT32_MAX, ipx_sem); - break; - case UR_TYPE_A_UINT16: - INT_CONV(uint16_t, field_ptr, value, 0, UINT16_MAX, ipx_sem); - break; - case UR_TYPE_A_UINT8: - INT_CONV(uint8_t, field_ptr, value, 0, UINT8_MAX, ipx_sem); - break; - default: - return 1; // Unsupported data type + if (translator_store_int(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; } } - return rc != FDS_OK; + return 0; } /** @@ -430,30 +626,17 @@ translate_array_bool(translator_t *trans, const struct translator_rec *rec, bool value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); if (fds_get_bool(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } - const uint8_t res = value ? 1U : 0U; - switch (ur_array_get_elem_size(ur_id)) { - case 1: - *((uint8_t *) field_ptr) = res; - break; - case 2: - *((uint16_t *) field_ptr) = res; - break; - case 4: - *((uint32_t *) field_ptr) = res; - break; - case 8: - *((uint64_t *) field_ptr) = res; - break; - default: - // Invalid size of the field - return 1; + if (translator_store_bool(ur_array_get_elem_size(ur_id), field_ptr, value)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; } } - return rc != FDS_OK; + return 0; } /** @@ -473,29 +656,17 @@ translate_array_float(translator_t *trans, const struct translator_rec *rec, double value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); if (fds_get_float_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } - switch (rec->unirec.type) { - case UR_TYPE_A_FLOAT: - if (value < -FLT_MAX && isnormal(value)) { - *((float *) field_ptr) = -FLT_MAX; - } else if (value > FLT_MAX && isnormal(value)) { - *((float *) field_ptr) = FLT_MAX; - } else { - *((float *) field_ptr) = (float) value; - } - break; - case UR_TYPE_A_DOUBLE: - *((double *) field_ptr) = value; - break; - default: - // Invalid type of the field - return 1; + if (translator_store_float(ur_array_get_elem_type(ur_id), field_ptr, value)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; } } - return rc != FDS_OK; + return 0; } /** @@ -514,20 +685,13 @@ translate_array_ip(translator_t *trans, const struct translator_rec *rec, while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - switch (list_it.field.size) { - case 4: // IPv4 - *((ip_addr_t *) field_ptr) = ip_from_4_bytes_be((char *) list_it.field.data); - break; - case 16: // IPv6 - *((ip_addr_t *) field_ptr) = ip_from_16_bytes_be((char *) list_it.field.data); - break; - default: - // Invalid size of the field - return 1; + if (translator_store_ip(list_it.field.data, list_it.field.size, field_ptr)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; } } - return rc != FDS_OK; + return 0; } /** @@ -552,7 +716,7 @@ translate_array_mac(translator_t *trans, const struct translator_rec *rec, memcpy(field_ptr, list_it.field.data, 6U); } - return rc != FDS_OK; + return 0; } /** @@ -573,215 +737,15 @@ translate_array_time(translator_t *trans, const struct translator_rec *rec, // Get the value ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - if (type_ipx == FDS_ET_DATE_TIME_MILLISECONDS || type_ipx == FDS_ET_DATE_TIME_SECONDS) { - // Low precision timestamp - uint64_t ts; - if (fds_get_datetime_lp_be(list_it.field.data, list_it.field.size, type_ipx, &ts) != FDS_OK) { - return 1; - } - - *field_ptr = ur_time_from_sec_msec(ts / 1000, ts % 1000); - continue; - } else if (type_ipx == FDS_ET_DATE_TIME_MICROSECONDS || type_ipx == FDS_ET_DATE_TIME_NANOSECONDS) { - // High precision timestamp - struct timespec ts; - if (fds_get_datetime_hp_be(list_it.field.data, list_it.field.size, type_ipx, &ts) != FDS_OK) { - return 1; - } - - *field_ptr = ur_time_from_sec_msec(ts.tv_sec, ts.tv_nsec / 1000000); - continue; + if (translator_store_time(type_ipx, list_it.field.data, list_it.field.size, field_ptr)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; } - return 1; } - return rc != FDS_OK; -} - -/** - * \brief Convert IPFIX bytes/string to UniRec bytes/string - * \copydetails translate_uint() - */ -static int -translate_bytes(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) -{ - ur_field_id_t ur_id = rec->unirec.id; - ur_set_var(trans->record.ur_tmplt, trans->record.data, ur_id, field->data, field->size); return 0; } -/** - * \brief Convert IPFIX string to trimmed UniRec string - * - * The function will copy only characters up to the first occurrence of '\0' (excluding). - * \copydetails translate_uint() - */ -static int -translator_string_trim(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) -{ - ur_field_id_t ur_id = rec->unirec.id; - size_t copy_len = strnlen((const char *) field->data, field->size); - ur_set_var(trans->record.ur_tmplt, trans->record.data, ur_id, field->data, copy_len); - return 0; -} - -/** - * \brief Convert IPFIX boolean to UniRec char/(un)signed integer - * \copydetails translate_uint() - */ -static int -translate_bool(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) -{ - bool value; - if (fds_get_bool(field->data, field->size, &value) != FDS_OK) { - return 1; - } - - const ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - - const uint8_t res = value ? 1U : 0U; - switch (rec->unirec.size) { - case 1: - *((uint8_t *) field_ptr) = res; - break; - case 2: - *((uint16_t *) field_ptr) = res; - break; - case 4: - *((uint32_t *) field_ptr) = res; - break; - case 8: - *((uint64_t *) field_ptr) = res; - break; - default: - // Invalid size of the field - return 1; - } - - return 0; -} - -/** - * \brief Convert IPFIX float to UniRec float - * \copydetails translate_uint() - */ -static int -translate_float(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) -{ - double value; - if (fds_get_float_be(field->data, field->size, &value) != FDS_OK) { - return 1; // Conversion failed - } - - const ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - - switch (rec->unirec.type) { - case UR_TYPE_FLOAT: - if (value < -FLT_MAX && isnormal(value)) { - *((float *) field_ptr) = -FLT_MAX; - } else if (value > FLT_MAX && isnormal(value)) { - *((float *) field_ptr) = FLT_MAX; - } else { - *((float *) field_ptr) = (float) value; - } - break; - case UR_TYPE_DOUBLE: - *((double *) field_ptr) = value; - break; - default: - // Invalid type of the field - return 1; - } - - return 0; -} - -/** - * \brief Convert IPFIX IPv4/IPv6 address to UniRec IPv4/IPv6 address - * \copydetails translate_uint() - */ -static int -translate_ip(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) -{ - const ur_field_id_t ur_id = rec->unirec.id; - void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - - switch (field->size) { - case 4: // IPv4 - *((ip_addr_t *) field_ptr) = ip_from_4_bytes_be((char *) field->data); - break; - case 16: // IPv6 - *((ip_addr_t *) field_ptr) = ip_from_16_bytes_be((char *) field->data); - break; - default: - // Invalid size of the field - return 1; - } - - return 0; -} - -/** - * \brief Convert IPFIX MAC address to UniRec MAC address - * \copydetails translate_uint() - */ -static int -translate_mac(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) -{ - if (field->size != 6U) { - return 1; - } - - const ur_field_id_t ur_id = rec->unirec.id; - ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - memcpy(field_ptr, field->data, 6U); - return 0; -} - -/** - * \brief Convert IPFIX timestamp to UniRec timestamp - * \copydetails translate_uint() - */ -static int -translate_time(translator_t *trans, const struct translator_rec *rec, - const struct fds_drec_field *field) -{ - // Get the value - const enum fds_iemgr_element_type type_ipx = rec->ipfix.type; - const ur_field_id_t ur_id = rec->unirec.id; - ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); - - if (type_ipx == FDS_ET_DATE_TIME_MILLISECONDS || type_ipx == FDS_ET_DATE_TIME_SECONDS) { - // Low precision timestamp - uint64_t ts; - if (fds_get_datetime_lp_be(field->data, field->size, type_ipx, &ts) != FDS_OK) { - return 1; - } - - *field_ptr = ur_time_from_sec_msec(ts / 1000, ts % 1000); - return 0; - } else if (type_ipx == FDS_ET_DATE_TIME_MICROSECONDS || type_ipx == FDS_ET_DATE_TIME_NANOSECONDS) { - // High precision timestamp - struct timespec ts; - if (fds_get_datetime_hp_be(field->data, field->size, type_ipx, &ts) != FDS_OK) { - return 1; - } - - *field_ptr = ur_time_from_sec_msec(ts.tv_sec, ts.tv_nsec / 1000000); - return 0; - } - - return 1; -} - /** * \brief Convert "ingressInterface" (EN: 0, ID: 10) to dir_bif_field * \copydetails translate_uint() From ab5e8d9e9202016f7212c722163c87d4d1baedbd Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Tue, 24 Mar 2020 22:33:33 +0100 Subject: [PATCH 08/16] UniRec output: basicList to unirec array conversion fix --- extra_plugins/output/unirec/src/translator.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index e360cd2a..e6816432 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -1350,6 +1350,7 @@ translator_table_fill_rec(translator_t *trans, const struct map_rec *map_rec, tr_ipfix->sem = ipfix->def->data_semantic; if (ipfix->next) { tr_ipfix->next = malloc(sizeof(struct tr_ipfix_s)); + tr_ipfix = tr_ipfix->next; } else { tr_ipfix->next = NULL; } @@ -1647,12 +1648,29 @@ translator_translate(translator_t *trans, struct fds_drec *ipfix_rec, uint16_t f // First, call special internal conversion functions, if enabled int converted_fields = translator_call_internals(trans); + struct tr_ipfix_s ipx_list_elem; // Try to convert all IPFIX fields while (fds_drec_iter_next(&it) != FDS_EOC) { // Find the conversion function const struct fds_tfield *info = it.field.info; key.ipfix.id = info->id; key.ipfix.pen = info->en; + key.ipfix.next = NULL; + + if (info->def && info->def->data_type == FDS_ET_BASIC_LIST) { + struct fds_blist_iter list_it; + + fds_blist_iter_init(&list_it, &it.field, NULL); + if (fds_blist_iter_next(&list_it) != FDS_OK) { + continue; + } + const struct fds_tfield *tmp = list_it.field.info; + ipx_list_elem.id = tmp->id; + ipx_list_elem.pen = tmp->en; + ipx_list_elem.next = NULL; + + key.ipfix.next = &ipx_list_elem; + } def = bsearch(&key, trans->table.recs, table_rec_cnt, table_rec_size, translator_cmp); if (!def) { From 5624dce1975ccee4347f615f688bc279f5c6c936 Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Wed, 25 Mar 2020 15:16:41 +0100 Subject: [PATCH 09/16] UniRec output: fixed empty basicList not being properly processed --- extra_plugins/output/unirec/src/translator.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index e6816432..f3d772a2 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -1658,12 +1658,12 @@ translator_translate(translator_t *trans, struct fds_drec *ipfix_rec, uint16_t f key.ipfix.next = NULL; if (info->def && info->def->data_type == FDS_ET_BASIC_LIST) { - struct fds_blist_iter list_it; + struct fds_blist_iter list_it; - fds_blist_iter_init(&list_it, &it.field, NULL); - if (fds_blist_iter_next(&list_it) != FDS_OK) { - continue; - } + fds_blist_iter_init(&list_it, &it.field, NULL); + if (fds_blist_iter_next(&list_it) == FDS_ERR_FORMAT) { + continue; + } const struct fds_tfield *tmp = list_it.field.info; ipx_list_elem.id = tmp->id; ipx_list_elem.pen = tmp->en; From 43c6ece5c1a0e48e2d30d0f607affc6b01bec1b9 Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Wed, 25 Mar 2020 15:53:47 +0100 Subject: [PATCH 10/16] UniRec output: fixed invalid params passed to store function --- extra_plugins/output/unirec/src/translator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index f3d772a2..c9c3d6b1 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -227,7 +227,7 @@ translate_uint(translator_t *trans, const struct translator_rec *rec, void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; - if (translator_store_uint(ur_id, field_ptr, value, ipx_sem)) { + if (translator_store_uint(rec->unirec.type, field_ptr, value, ipx_sem)) { return 1; } @@ -316,7 +316,7 @@ translate_int(translator_t *trans, const struct translator_rec *rec, void *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.sem; - if (translator_store_int(ur_id, field_ptr, value, ipx_sem)) { + if (translator_store_int(rec->unirec.type, field_ptr, value, ipx_sem)) { return 1; } From e15a3d919f266e845be55370293e1848c8c4012b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C4=8Cejka?= Date: Fri, 24 Apr 2020 10:49:33 +0200 Subject: [PATCH 11/16] UniRec output: update list of UniRec elements --- .../output/unirec/config/unirec-elements.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 445d2c4f..842b9ced 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -151,7 +151,11 @@ TLS_PUBLIC_KEYALG uint16 flowmon:tlsPublicKeyAlg # tlsP TLS_PUBLIC_KEYLENGTH int32 flowmon:tlsPublicKeyLength # tlsPublicKeyLength TLS_JA_3FINGERPRINT bytes flowmon:tlsJa3Fingerprint # tlsJa3Fingerprint -# --- Cisco Joy exporter --- -JOY_TLS_RECORD_LENGTHS int16* e0id291/e9id12188 # basicList of packet lengths -JOY_TLS_RECORD_TIMES uint16* e0id291/e9id12189 # basicList of packet timestamps -JOY_TLS_CONTENT_TYPES uint8* e0id291/e9id12190 # basicList of packet types \ No newline at end of file +# --- Per-Packet Information elements --- +PPI_TLS_REC_LENGTHS int16* e0id291/e8057id1010 # basicList of TLS record lengths +PPI_TLS_REC_TIMES uint16* e0id291/e8057id1011 # basicList of TLS record timestamps +PPI_TLS_CONTENT_TYPES uint8* e0id291/e8057id1012 # basicList of TLS record content types +PPI_PKT_LENGTHS int16* e0id291/e8057id1013 # basicList of packet lengths +PPI_PKT_TIMES uint16* e0id291/e8057id1014 # basicList of packet timestamps +PPI_PKT_FLAGS int8* e0id291/e8057id1015 # basicList of packet TCP flags +PPI_PKT_DIRECTIONS int8* e0id291/e8057id1016 # basicList of packet directions From a9be6329a8bb7cfdfadf926c7912a996e4ddd34d Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Fri, 5 Jun 2020 17:32:55 +0200 Subject: [PATCH 12/16] UniRec output: fixed internal lbf function --- extra_plugins/output/unirec/src/translator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index c9c3d6b1..4d2af207 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -1437,6 +1437,7 @@ translator_table_fill_internal(translator_t *trans, const struct map_rec *map_re trans_rec->ipfix.id = 10; // iana:ingressInterface trans_rec->ipfix.type = FDS_ET_UNSIGNED_32; trans_rec->ipfix.sem = FDS_ES_IDENTIFIER; + trans_rec->ipfix.next = NULL; trans_rec->func = translate_internal_dbf; IPX_CTX_DEBUG(trans->ctx, "Added conversion from internal 'dir_bit_field' to UniRec '%s'", map_rec->unirec.name); From 6c0c313f592fe9504fb3143220237195b56a8cff Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Fri, 5 Jun 2020 17:56:35 +0200 Subject: [PATCH 13/16] Unirec output: fixed store of array elements to unirec in some functions --- extra_plugins/output/unirec/src/translator.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index 4d2af207..1867e041 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -712,7 +712,7 @@ translate_array_mac(translator_t *trans, const struct translator_rec *rec, return 1; } - ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + ur_time_t *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); memcpy(field_ptr, list_it.field.data, 6U); } @@ -734,8 +734,7 @@ translate_array_time(translator_t *trans, const struct translator_rec *rec, fds_blist_iter_init(&list_it, field, NULL); while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - // Get the value - ur_time_t *field_ptr = ur_get_ptr_by_id(trans->record.ur_tmplt, trans->record.data, ur_id); + ur_time_t *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); if (translator_store_time(type_ipx, list_it.field.data, list_it.field.size, field_ptr)) { ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); From 950fcfaf35e75eda894369f54c1fe45ff3631666 Mon Sep 17 00:00:00 2001 From: Jiri Havranek Date: Fri, 5 Jun 2020 18:43:33 +0200 Subject: [PATCH 14/16] Unirec output: added check of failed array allocation for some conversion functions --- extra_plugins/output/unirec/src/translator.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index 1867e041..2f91b356 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -564,7 +564,7 @@ translate_array_uint(translator_t *trans, const struct translator_rec *rec, while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { uint64_t value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (fds_get_uint_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + if (field_ptr == NULL || fds_get_uint_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } @@ -595,7 +595,7 @@ translate_array_int(translator_t *trans, const struct translator_rec *rec, while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { int64_t value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (fds_get_int_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + if (field_ptr == NULL || fds_get_int_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } @@ -625,7 +625,7 @@ translate_array_bool(translator_t *trans, const struct translator_rec *rec, while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { bool value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (fds_get_bool(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + if (field_ptr == NULL || fds_get_bool(list_it.field.data, list_it.field.size, &value) != FDS_OK) { ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } @@ -655,7 +655,7 @@ translate_array_float(translator_t *trans, const struct translator_rec *rec, while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { double value; void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (fds_get_float_be(list_it.field.data, list_it.field.size, &value) != FDS_OK || field_ptr == NULL) { + if (field_ptr == NULL || fds_get_float_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; // Conversion failed } @@ -685,7 +685,7 @@ translate_array_ip(translator_t *trans, const struct translator_rec *rec, while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (translator_store_ip(list_it.field.data, list_it.field.size, field_ptr)) { + if (field_ptr == NULL || translator_store_ip(list_it.field.data, list_it.field.size, field_ptr)) { ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; } @@ -713,6 +713,11 @@ translate_array_mac(translator_t *trans, const struct translator_rec *rec, } ur_time_t *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + memcpy(field_ptr, list_it.field.data, 6U); } @@ -735,8 +740,7 @@ translate_array_time(translator_t *trans, const struct translator_rec *rec, fds_blist_iter_init(&list_it, field, NULL); while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { ur_time_t *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - - if (translator_store_time(type_ipx, list_it.field.data, list_it.field.size, field_ptr)) { + if (field_ptr == NULL || translator_store_time(type_ipx, list_it.field.data, list_it.field.size, field_ptr)) { ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); return 1; } From afbaf0fb0648386640f535a61ba9a7c70c6a89c8 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Wed, 1 Jul 2020 15:33:44 +0200 Subject: [PATCH 15/16] Unirec output: update conversion map (add reverse fields, fix invalid PPI type) --- extra_plugins/output/unirec/config/unirec-elements.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extra_plugins/output/unirec/config/unirec-elements.txt b/extra_plugins/output/unirec/config/unirec-elements.txt index 842b9ced..6de046bb 100644 --- a/extra_plugins/output/unirec/config/unirec-elements.txt +++ b/extra_plugins/output/unirec/config/unirec-elements.txt @@ -35,6 +35,13 @@ TIME_FIRST time e0id150,e0id152,e0id154,e0id156 # T TIME_LAST time e0id151,e0id153,e0id155,e0id157 # Time of the last packet of a flow DIR_BIT_FIELD uint8 _internal_dbf_ # Bit field used for determining incoming/outgoing flow (1 => Incoming, 0 => Outgoing) LINK_BIT_FIELD uint64 _internal_lbf_ # Bit field of links on which was flow seen +SRC_MAC macaddr e0id56 +DST_MAC macaddr e0id80 + +# --- Additional biflow fields --- +BYTES_REV uint64 e29305id1 +PACKETS_REV uint32 e29305id2 +TCP_FLAGS_REV uint8 e29305id6 # --- DNS specific fields --- DNS_ANSWERS uint16 e8057id0 # DNS answers @@ -156,6 +163,6 @@ PPI_TLS_REC_LENGTHS int16* e0id291/e8057id1010 # basi PPI_TLS_REC_TIMES uint16* e0id291/e8057id1011 # basicList of TLS record timestamps PPI_TLS_CONTENT_TYPES uint8* e0id291/e8057id1012 # basicList of TLS record content types PPI_PKT_LENGTHS int16* e0id291/e8057id1013 # basicList of packet lengths -PPI_PKT_TIMES uint16* e0id291/e8057id1014 # basicList of packet timestamps +PPI_PKT_TIMES time* e0id291/e8057id1014 # basicList of packet timestamps PPI_PKT_FLAGS int8* e0id291/e8057id1015 # basicList of packet TCP flags PPI_PKT_DIRECTIONS int8* e0id291/e8057id1016 # basicList of packet directions From b058cd39d241d3cf48d150ca8cbbdbed1744ec68 Mon Sep 17 00:00:00 2001 From: Lukas Hutak Date: Thu, 2 Jul 2020 13:34:16 +0200 Subject: [PATCH 16/16] UniRec output: code style polishing --- extra_plugins/output/unirec/src/map.c | 126 ++++---- extra_plugins/output/unirec/src/translator.c | 315 ++++++++++--------- 2 files changed, 228 insertions(+), 213 deletions(-) diff --git a/extra_plugins/output/unirec/src/map.c b/extra_plugins/output/unirec/src/map.c index de954d0a..415facda 100644 --- a/extra_plugins/output/unirec/src/map.c +++ b/extra_plugins/output/unirec/src/map.c @@ -291,44 +291,47 @@ map_load_line_ie_defs(map_t *map, char *ur_name, int ur_type, char *ur_type_str, char *subsave_ptr_list = NULL; struct map_ipfix_s *ipfix = &rec.ipfix; rec.ipfix.next = NULL; + for (char *list = subtoken; ;list = NULL) { - char *list_token = strtok_r(list, "/", &subsave_ptr_list); - if (!list_token) { - break; - } - - elem_def = map_elem_get_ipfix(map->iemgr, list_token); - if (elem_def != NULL) { - if (list == NULL) { - ipfix->next = malloc(sizeof(struct map_ipfix_s)); - if (!ipfix->next) { - rc = IPX_ERR_NOMEM; - elem_def = NULL; - break; - } - ipfix = ipfix->next; - } - // Store the "IPFIX element" record - ipfix->source = MAP_SRC_IPFIX; - ipfix->def = elem_def; - ipfix->id = elem_def->id; - ipfix->en = elem_def->scope->pen; - ipfix->next = NULL; - - continue; - } - break; + // Test if the specifier describes basicList + char *list_token = strtok_r(list, "/", &subsave_ptr_list); + if (!list_token) { + break; + } + + elem_def = map_elem_get_ipfix(map->iemgr, list_token); + if (elem_def != NULL) { + if (list == NULL) { + ipfix->next = malloc(sizeof(struct map_ipfix_s)); + if (!ipfix->next) { + rc = IPX_ERR_NOMEM; + elem_def = NULL; + break; + } + ipfix = ipfix->next; + } + + // Store the "IPFIX element" record + ipfix->source = MAP_SRC_IPFIX; + ipfix->def = elem_def; + ipfix->id = elem_def->id; + ipfix->en = elem_def->scope->pen; + ipfix->next = NULL; + continue; + } + break; } + if (elem_def != NULL) { - rc = map_rec_add(map, &rec); - continue; + rc = map_rec_add(map, &rec); + continue; } ipfix = rec.ipfix.next; while (ipfix) { - struct map_ipfix_s *tmp = ipfix->next; - free(ipfix); - ipfix = tmp; + struct map_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; } enum MAP_SRC fn_id = map_elem_get_internal(subtoken); @@ -459,28 +462,28 @@ map_sort_fn(const void *p1, const void *p2) const struct map_ipfix_s *ipfix2 = &(*(struct map_rec **) p2)->ipfix; while (ipfix1 && ipfix2) { - if (ipfix1->source != ipfix2->source) { - return (ipfix1->source < ipfix2->source) ? (-1) : 1; - } - - // Primary sort by PEN - if (ipfix1->en != ipfix2->en) { - return (ipfix1->en < ipfix2->en) ? (-1) : 1; - } - - // Secondary sort by ID - if (ipfix1->id != ipfix2->id) { - return (ipfix1->id < ipfix2->id) ? (-1) : 1; - } - - ipfix1 = ipfix1->next; - ipfix2 = ipfix2->next; + if (ipfix1->source != ipfix2->source) { + return (ipfix1->source < ipfix2->source) ? (-1) : 1; + } + + // Primary sort by PEN + if (ipfix1->en != ipfix2->en) { + return (ipfix1->en < ipfix2->en) ? (-1) : 1; + } + + // Secondary sort by ID + if (ipfix1->id != ipfix2->id) { + return (ipfix1->id < ipfix2->id) ? (-1) : 1; + } + + ipfix1 = ipfix1->next; + ipfix2 = ipfix2->next; } if (ipfix1->next) { - return -1; + return -1; } else if (ipfix2->next) { - return 1; + return 1; } return 0; @@ -563,20 +566,21 @@ map_load(map_t *map, const char *file) const struct map_ipfix_s *ipfix_prev = &rec_prev->ipfix; const struct map_ipfix_s *ipfix_now = &rec_now->ipfix; while (1) { - if (ipfix_prev->en != ipfix_now->en || ipfix_prev->id != ipfix_now->id || !ipfix_prev || !ipfix_now) { - rec_prev = rec_now; - break; - } - - ipfix_prev = ipfix_prev->next; - ipfix_now = ipfix_now->next; - if (!ipfix_prev && !ipfix_now) { - collision = true; - break; - } + if (!ipfix_prev || !ipfix_now || ipfix_prev->en != ipfix_now->en + || ipfix_prev->id != ipfix_now->id) { + rec_prev = rec_now; + break; + } + + ipfix_prev = ipfix_prev->next; + ipfix_now = ipfix_now->next; + if (!ipfix_prev && !ipfix_now) { + collision = true; + break; + } } if (!collision) { - continue; + continue; } // Collision detected! diff --git a/extra_plugins/output/unirec/src/translator.c b/extra_plugins/output/unirec/src/translator.c index 2f91b356..36ea75d0 100644 --- a/extra_plugins/output/unirec/src/translator.c +++ b/extra_plugins/output/unirec/src/translator.c @@ -2,10 +2,11 @@ * \file translator.c * \author Lukas Hutak * \author Tomas Cejka + * \author Jiri Havranek * \brief Conversion of IPFIX to UniRec format (source file) */ -/* Copyright (C) 2015 - 2017 CESNET, z.s.p.o. +/* Copyright (C) 2015 - 2020 CESNET, z.s.p.o. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -172,8 +173,9 @@ struct translator_s { *((dst_type *) (dst_ptr)) = (dst_type) (src_val); \ } -int -translator_store_uint(ur_field_type_t ur_type, void *field_ptr, uint64_t value, const enum fds_iemgr_element_semantic ipx_sem) +static inline int +translator_store_uint(ur_field_type_t ur_type, void *field_ptr, uint64_t value, + const enum fds_iemgr_element_semantic ipx_sem) { switch (ur_type) { case UR_TYPE_UINT64: @@ -264,8 +266,9 @@ translate_uint(translator_t *trans, const struct translator_rec *rec, *((dst_type *) (dst_ptr)) = (dst_type) (src_val); \ } -int -translator_store_int(ur_field_type_t ur_type, void *field_ptr, int64_t value, const enum fds_iemgr_element_semantic ipx_sem) +static inline int +translator_store_int(ur_field_type_t ur_type, void *field_ptr, int64_t value, + const enum fds_iemgr_element_semantic ipx_sem) { switch (ur_type) { case UR_TYPE_INT64: @@ -352,7 +355,7 @@ translator_string_trim(translator_t *trans, const struct translator_rec *rec, return 0; } -int +static inline int translator_store_bool(int ur_size, void *field_ptr, bool value) { const uint8_t res = value ? 1U : 0U; @@ -400,7 +403,7 @@ translate_bool(translator_t *trans, const struct translator_rec *rec, return 0; } -int +static inline int translator_store_float(ur_field_type_t ur_type, void *field_ptr, double value) { switch (ur_type) { @@ -447,8 +450,8 @@ translate_float(translator_t *trans, const struct translator_rec *rec, return 0; } -int -translator_store_ip(uint8_t *ip_bytes, uint16_t data_size, void *field_ptr) +static inline int +translator_store_ip(const uint8_t *ip_bytes, uint16_t data_size, void *field_ptr) { switch (data_size) { case 4: // IPv4 @@ -501,8 +504,9 @@ translate_mac(translator_t *trans, const struct translator_rec *rec, return 0; } -int -translator_store_time(const enum fds_iemgr_element_type type_ipx, const uint8_t *time_bytes, uint16_t time_size, ur_time_t *field_ptr) +static inline int +translator_store_time(const enum fds_iemgr_element_type type_ipx, const uint8_t *time_bytes, + uint16_t time_size, ur_time_t *field_ptr) { if (type_ipx == FDS_ET_DATE_TIME_MILLISECONDS || type_ipx == FDS_ET_DATE_TIME_SECONDS) { // Low precision timestamp @@ -555,24 +559,24 @@ static int translate_array_uint(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - ur_field_id_t ur_id = rec->unirec.id; - const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; - struct fds_blist_iter list_it; - int rc; - - fds_blist_iter_init(&list_it, field, NULL); - while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - uint64_t value; - void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (field_ptr == NULL || fds_get_uint_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; // Conversion failed - } - - if (translator_store_uint(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; - } + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + uint64_t value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_uint_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } + + if (translator_store_uint(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } } return 0; @@ -586,24 +590,24 @@ static int translate_array_int(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - ur_field_id_t ur_id = rec->unirec.id; - const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; - struct fds_blist_iter list_it; - int rc; - - fds_blist_iter_init(&list_it, field, NULL); - while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - int64_t value; - void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (field_ptr == NULL || fds_get_int_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; // Conversion failed - } - - if (translator_store_int(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; - } + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_semantic ipx_sem = rec->ipfix.next->sem; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + int64_t value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_int_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } + + if (translator_store_int(ur_array_get_elem_type(ur_id), field_ptr, value, ipx_sem)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } } return 0; @@ -617,24 +621,24 @@ static int translate_array_bool(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - ur_field_id_t ur_id = rec->unirec.id; - struct fds_blist_iter list_it; - int rc; - - fds_blist_iter_init(&list_it, field, NULL); - while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - bool value; - void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (field_ptr == NULL || fds_get_bool(list_it.field.data, list_it.field.size, &value) != FDS_OK) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; // Conversion failed - } + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + bool value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_bool(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } - if (translator_store_bool(ur_array_get_elem_size(ur_id), field_ptr, value)) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; - } - } + if (translator_store_bool(ur_array_get_elem_size(ur_id), field_ptr, value)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } return 0; } @@ -647,24 +651,24 @@ static int translate_array_float(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - ur_field_id_t ur_id = rec->unirec.id; - struct fds_blist_iter list_it; - int rc; - - fds_blist_iter_init(&list_it, field, NULL); - while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - double value; - void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (field_ptr == NULL || fds_get_float_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; // Conversion failed - } + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + double value; + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || fds_get_float_be(list_it.field.data, list_it.field.size, &value) != FDS_OK) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; // Conversion failed + } - if (translator_store_float(ur_array_get_elem_type(ur_id), field_ptr, value)) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; - } - } + if (translator_store_float(ur_array_get_elem_type(ur_id), field_ptr, value)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } return 0; } @@ -677,21 +681,21 @@ static int translate_array_ip(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - ur_field_id_t ur_id = rec->unirec.id; - struct fds_blist_iter list_it; - int rc; + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; - fds_blist_iter_init(&list_it, field, NULL); - while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (field_ptr == NULL || translator_store_ip(list_it.field.data, list_it.field.size, field_ptr)) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; - } - } + if (field_ptr == NULL || translator_store_ip(list_it.field.data, list_it.field.size, field_ptr)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } - return 0; + return 0; } /** @@ -702,26 +706,26 @@ static int translate_array_mac(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - ur_field_id_t ur_id = rec->unirec.id; - struct fds_blist_iter list_it; - int rc; + ur_field_id_t ur_id = rec->unirec.id; + struct fds_blist_iter list_it; + int rc; - fds_blist_iter_init(&list_it, field, NULL); - while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - if (list_it.field.size != 6U) { - return 1; - } + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + if (list_it.field.size != 6U) { + return 1; + } - ur_time_t *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (field_ptr == NULL) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; - } + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } - memcpy(field_ptr, list_it.field.data, 6U); - } + memcpy(field_ptr, list_it.field.data, 6U); + } - return 0; + return 0; } /** @@ -732,19 +736,19 @@ static int translate_array_time(translator_t *trans, const struct translator_rec *rec, const struct fds_drec_field *field) { - ur_field_id_t ur_id = rec->unirec.id; - const enum fds_iemgr_element_type type_ipx = rec->ipfix.next->type; - struct fds_blist_iter list_it; - int rc; - - fds_blist_iter_init(&list_it, field, NULL); - while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { - ur_time_t *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); - if (field_ptr == NULL || translator_store_time(type_ipx, list_it.field.data, list_it.field.size, field_ptr)) { - ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); - return 1; - } - } + ur_field_id_t ur_id = rec->unirec.id; + const enum fds_iemgr_element_type type_ipx = rec->ipfix.next->type; + struct fds_blist_iter list_it; + int rc; + + fds_blist_iter_init(&list_it, (struct fds_drec_field *) field, NULL); + while ((rc = fds_blist_iter_next(&list_it)) == FDS_OK) { + void *field_ptr = ur_array_append_get_ptr(trans->record.ur_tmplt, trans->record.data, ur_id); + if (field_ptr == NULL || translator_store_time(type_ipx, list_it.field.data, list_it.field.size, field_ptr)) { + ur_array_clear(trans->record.ur_tmplt, trans->record.data, ur_id); + return 1; + } + } return 0; } @@ -910,10 +914,10 @@ translator_get_numeric_func(ipx_ctx_t *ctx, const struct map_rec *rec, bool is_a enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; if (is_array) { - if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { - return NULL; - } - type_ipx = rec->ipfix.next->def->data_type; + if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { + return NULL; + } + type_ipx = rec->ipfix.next->def->data_type; } uint16_t size_ur = translator_size_ur_int(type_ur); @@ -975,10 +979,10 @@ translator_get_float_func(ipx_ctx_t *ctx, const struct map_rec *rec, bool is_arr enum fds_iemgr_element_type type_ipx = rec->ipfix.def->data_type; if (is_array) { - if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { - return NULL; - } - type_ipx = rec->ipfix.next->def->data_type; + if (rec->ipfix.next == NULL || rec->ipfix.next->next != NULL) { + return NULL; + } + type_ipx = rec->ipfix.next->def->data_type; } uint16_t size_ur = translator_size_ur_int(type_ur); @@ -1347,17 +1351,17 @@ translator_table_fill_rec(translator_t *trans, const struct map_rec *map_rec, const struct map_ipfix_s *ipfix = &map_rec->ipfix; struct tr_ipfix_s *tr_ipfix = &trans_rec->ipfix; while (ipfix) { - tr_ipfix->pen = ipfix->en; - tr_ipfix->id = ipfix->id; - tr_ipfix->type = ipfix->def->data_type; - tr_ipfix->sem = ipfix->def->data_semantic; - if (ipfix->next) { - tr_ipfix->next = malloc(sizeof(struct tr_ipfix_s)); - tr_ipfix = tr_ipfix->next; - } else { - tr_ipfix->next = NULL; - } - ipfix = ipfix->next; + tr_ipfix->pen = ipfix->en; + tr_ipfix->id = ipfix->id; + tr_ipfix->type = ipfix->def->data_type; + tr_ipfix->sem = ipfix->def->data_semantic; + if (ipfix->next) { + tr_ipfix->next = malloc(sizeof(struct tr_ipfix_s)); + tr_ipfix = tr_ipfix->next; + } else { + tr_ipfix->next = NULL; + } + ipfix = ipfix->next; } IPX_CTX_DEBUG(trans->ctx, "Added conversion from IPFIX IE '%s:%s' to UniRec '%s'", @@ -1451,6 +1455,21 @@ translator_table_fill_internal(translator_t *trans, const struct map_rec *map_re return IPX_ERR_DENIED; } +static inline void +translator_table_free_recs(struct translator_rec *recs, size_t recs_cnt) +{ + for (size_t i = 0; i < recs_cnt; ++i) { + struct tr_ipfix_s *ipfix = recs[i].ipfix.next; + while (ipfix) { + struct tr_ipfix_s *tmp = ipfix->next; + free(ipfix); + ipfix = tmp; + } + } + + free(recs); +} + /** * \brief Initialize a conversion table * @@ -1525,7 +1544,7 @@ translator_init_table(translator_t *trans, const map_t *map, const ur_template_t } if (ret_val != IPX_OK) { // Failed - free(table); + translator_table_free_recs(table, rec_cnt); return ret_val; } @@ -1549,15 +1568,7 @@ translator_init_table(translator_t *trans, const map_t *map, const ur_template_t static void translator_destroy_table(translator_t *trans) { - for (size_t i = 0; i < trans->table.size; ++i) { - struct tr_ipfix_s *ipfix = trans->table.recs[i].ipfix.next; - while (ipfix) { - struct tr_ipfix_s *tmp = ipfix->next; - free(ipfix); - ipfix = tmp; - } - } - free(trans->table.recs); + translator_table_free_recs(trans->table.recs, trans->table.size); } /** @@ -1699,7 +1710,7 @@ translator_translate(translator_t *trans, struct fds_drec *ipfix_rec, uint16_t f return NULL; } - // Check if conversion filled are required fields + // Check if conversion filled all required fields size_t idx; for (idx = 0; idx < trans->progress.size && trans->progress.req_fields[idx] == 0; ++idx); if (idx < trans->progress.size) {