Merge branch 'rc-4.0.4'

AUTHORS

@@ -35,6 +35,8 @@ In no particular order, we would like to recognise:
  * Robert Zeh for resolving some wandio errors that were causing crashes
  * Anthony Coddington for adding ERF provenance support and fixing a number
    of other ERF/DAG issues
+ * Tim Dawson for fixing issues with building against musl libc
+ * Hendrik Leppelsack for reporting and fixing some errors in the tool manpages
  * Jamie Curtis for fixing a couple of bugs many many years ago
  * Brendon Jones for creating the original Windows DLLs and writing bits of
    code here and there

README

@@ -1,4 +1,4 @@
-libtrace 4.0.3
+libtrace 4.0.4
 
 ---------------------------------------------------------------------------
 Copyright (c) 2007-2018 The University of Waikato, Hamilton, New Zealand.

configure.in

@@ -3,11 +3,11 @@
 # Now you only need to update the version number in two places - below,
 # and in the README
 
-AC_INIT([libtrace],[4.0.3],[[email protected]],[libtrace])
+AC_INIT([libtrace],[4.0.4],[[email protected]],[libtrace])
 
 LIBTRACE_MAJOR=4
 LIBTRACE_MID=0
-LIBTRACE_MINOR=3
+LIBTRACE_MINOR=4
 
 # OpenSolaris hides libraries like libncurses in /usr/gnu/lib, which is not
 # searched by default - add it to LDFLAGS so we at least have a chance of 
@@ -86,10 +86,6 @@ AC_CONFIG_LIBOBJ_DIR(lib)
 AC_FUNC_MALLOC
 AC_FUNC_REALLOC
 
-# *BSD doesn't have strndup. Currently provide our own.
-AC_REPLACE_FUNCS(strndup)
-
-
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
 AC_C_INLINE
@@ -125,7 +121,7 @@ AC_PROG_LIBTOOL
 AC_PROG_GCC_TRADITIONAL
 
 # Fail if any of these functions are missing
-AC_CHECK_FUNCS(socket strdup strlcpy strcasecmp strncasecmp snprintf vsnprintf recvmmsg)
+AC_CHECK_DECLS([socket, strdup, strlcpy, strcasecmp, strncasecmp, snprintf, vsnprintf, recvmmsg, strndup])
 
 AC_CHECK_SIZEOF([long int])
 
@@ -201,16 +197,20 @@ AC_CHECK_HEADER(net/bpf.h)
 AC_CHECK_HEADER(gdc.h,AC_DEFINE(HAVE_LIBGDC,1,[has gdc.h]))
 
 # Check for libwandio (no longer bundled with libtrace)
-AC_CHECK_LIB(wandio,wandio_create,wandiofound=1,wandiofound=0)
+AC_CHECK_LIB(wandio,wandio_wflush,wandiofound=1,wandiofound=0)
 if test "$wandiofound" = 0; then
-	AC_MSG_ERROR(libwandio is required to compile libtrace. If you have installed it in a non-standard location please use LDFLAGS to specify the location of the library. WANDIO can be obtained from http://research.wand.net.nz/software/libwandio.php)
+	AC_MSG_ERROR(libwandio 1.0.5 or better is required to compile this version of libtrace. If you have installed libwandio in a non-standard location please use LDFLAGS to specify the location of the library. WANDIO can be obtained from http://research.wand.net.nz/software/libwandio.php)
 else
 	LIBTRACE_LIBS="$LIBTRACE_LIBS -lwandio"
         TOOLS_LIBS="$TOOLS_LIBS -lwandio"
 	AC_DEFINE([HAVE_LIBWANDIO],1,[compile with libwandio support])
 fi
 
 AC_CHECK_LIB(crypto, EVP_EncryptInit_ex, cryptofound=1, cryptofound=0)
+if test "$cryptofound" = 1; then
+	AC_CHECK_HEADER(openssl/evp.h, cryptofound=1, cryptofound=0)
+fi
+AC_CHECK_LIB(tasn1, asn1_array2tree, asn1found=1, asn1found=0)
 
 # Check for libpcap
 AC_CHECK_LIB(pcap,pcap_next_ex,pcapfound=1,pcapfound=0)
@@ -454,6 +454,9 @@ AC_CHECK_DECL([PACKET_FANOUT],
 # If we use DPDK we might be able to use libnuma
 AC_CHECK_LIB(numa, numa_node_to_cpus, have_numa=1, have_numa=0)
 
+# Need libwandder for ETSI live decoding
+AC_CHECK_LIB(wandder, init_wandder_decoder, have_wandder=1, have_wandder=0)
+
 # Checks for various "optional" libraries
 AC_CHECK_LIB(pthread, pthread_create, have_pthread=1, have_pthread=0)
 
@@ -481,10 +484,17 @@ if test "$have_numa" = 1; then
 	AC_DEFINE(HAVE_LIBNUMA, 1, [Set to 1 if libnuma is supported])
 	with_numa=yes
 else
-	AC_DEFINE(HAVE_LIBNUMA, 0, [Set to 1 if libnuma is supported])
 	with_numa=no
 fi
 
+if test "$have_wandder" = 1; then
+        LIBTRACE_LIBS="$LIBTRACE_LIBS -lwandder"
+        AC_DEFINE(HAVE_WANDDER, 1, [Set to 1 if libwandder is available])
+        wandder_avail=yes
+else
+        wandder_avail=no
+fi
+
 if test "$dlfound" = 0; then
 	AC_MSG_ERROR("Unable to find dlopen. Please use LDFLAGS to specify the location of libdl and re-run configure")
 fi
@@ -532,11 +542,23 @@ if test "$have_clock_gettime" = 1; then
 	AC_DEFINE(HAVE_CLOCK_GETTIME, 1, [Set to 1 if clock_gettime is supported])
 	with_clock_gettime=yes
 else
-	AC_DEFINE(HAVE_CLOCK_GETTIME, 0, [Set to 1 if clock_gettime is supported])
 	with_clock_gettime=no
 fi
 
 
+have_memfd_create=no
+AC_RUN_IFELSE([AC_LANG_PROGRAM([
+#include <sys/syscall.h>
+],[
+#ifndef __NR_memfd_create
+exit(1)
+#endif
+])], [have_memfd_create=yes])
+
+if test "$have_memfd_create" = "yes"; then
+        AC_DEFINE(HAVE_MEMFD_CREATE, 1, [Set to 1 if kernel supports memfd_create syscall])
+fi
+
 # Stupid AC_SEARCH_LIB appends stuff to LIBS even when I don't want it to, so
 # just set libs to null here to avoid linking against them by default
 LIBS=
@@ -610,6 +632,7 @@ AS_IF([test "x$have_ncurses" = "xyes"], [
 AM_CONDITIONAL([HAVE_BPF_CAPTURE], [test "$ac_cv_have_decl_BIOCSETIF" = yes ])
 AM_CONDITIONAL([HAVE_DAG], [test "$libtrace_dag" = true])
 AM_CONDITIONAL([HAVE_DPDK], [test "$libtrace_dpdk" = true])
+AM_CONDITIONAL([HAVE_WANDDER], [test "x$wandder_avail" = "xyes"])
 AM_CONDITIONAL([DAG2_4], [test "$libtrace_dag_version" = 24])
 AM_CONDITIONAL([DAG2_5], [test "$libtrace_dag_version" = 25])
 AM_CONDITIONAL([HAVE_NETPACKET_PACKET_H], [test "$libtrace_netpacket_packet_h" = true])
@@ -691,9 +714,10 @@ elif test x"$want_dpdk" != "xno"; then
 	AC_MSG_NOTICE([Note: Requires DPDK v1.5 or newer])
 fi
 reportopt "Compiled with LLVM BPF JIT support" $JIT
+reportopt "Compiled with live ETSI LI support (requires libwandder)" $wandder_avail
 reportopt "Building man pages/documentation" $libtrace_doxygen
 reportopt "Building tracetop (requires libncurses)" $with_ncurses
-reportopt "Building traceanon with CryptoPan (requires libcrypto)" $have_crypto
+reportopt "Building traceanon with CryptoPan (requires libcrypto and openssl/evp.h)" $have_crypto
 
 # Report any errors relating to missing bison, flex, etc.
 echo 

lib/Makefile.am

@@ -6,7 +6,8 @@ pkginclude_HEADERS = dagformat.h lt_inttypes.h daglegacy.h \
         data-struct/vector.h \
         data-struct/deque.h data-struct/linked_list.h \
         data-struct/buckets.h data-struct/sliding_window.h \
-	data-struct/message_queue.h hash_toeplitz.h
+	data-struct/message_queue.h hash_toeplitz.h \
+        data-struct/simple_circular_buffer.h
 
 AM_CFLAGS=@LIBCFLAGS@ @CFLAG_VISIBILITY@ -pthread
 AM_CXXFLAGS=@LIBCXXFLAGS@ @CFLAG_VISIBILITY@ -pthread
@@ -50,6 +51,12 @@ export CFLAGS += $(SAVED_CFLAGS)
 export CXXFLAGS += $(SAVED_CXXFLAGS)
 endif
 
+if HAVE_WANDDER
+ETSISOURCES=format_etsilive.c
+else
+ETSISOURCES=
+endif
+
 libtrace_la_SOURCES = trace.c trace_parallel.c common.h \
 		format_erf.c format_pcap.c format_legacy.c \
 		format_rt.c format_helper.c format_helper.h format_pcapfile.c \
@@ -62,15 +69,16 @@ libtrace_la_SOURCES = trace.c trace_parallel.c common.h \
 		protocols_transport.c protocols.h protocols_ospf.c \
 		protocols_application.c \
 		$(DAGSOURCE) format_erf.h format_ndag.c format_ndag.h \
-		$(BPFJITSOURCE) \
+		$(BPFJITSOURCE) $(ETSISOURCES) \
 		libtrace_arphrd.h \
 		data-struct/ring_buffer.c data-struct/vector.c \
 		data-struct/message_queue.c data-struct/deque.c \
 		data-struct/sliding_window.c data-struct/object_cache.c \
 		data-struct/linked_list.c hash_toeplitz.c combiner_ordered.c \
-                data-struct/buckets.c \
+                data-struct/buckets.c data-struct/simple_circular_buffer.c \
 		combiner_sorted.c combiner_unordered.c \
-		pthread_spinlock.c pthread_spinlock.h
+		pthread_spinlock.c pthread_spinlock.h \
+		strndup.c
 
 if DAG2_4
 nodist_libtrace_la_SOURCES = dagopts.c dagapi.c

lib/data-struct/simple_circular_buffer.c

@@ -0,0 +1,104 @@
+#define _GNU_SOURCE
+
+#include <sys/mman.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <sys/syscall.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/socket.h>
+#include <errno.h>
+
+#include "simple_circular_buffer.h"
+
+DLLEXPORT int libtrace_scb_init(libtrace_scb_t *buf, uint32_t size,
+                uint16_t id) {
+
+        char anonname[32];
+
+        if (size % getpagesize() != 0) {
+                size = ((size / getpagesize()) + 1) * getpagesize();
+        }
+
+        snprintf(anonname, 32, "lt_scb_%u", id);
+#ifdef HAVE_MEMFD_CREATE
+        buf->fd = syscall(__NR_memfd_create, anonname, 0);
+#else
+        buf->fd = shm_open(anonname, O_RDWR | O_CREAT, 0600);
+#endif
+        if (ftruncate(buf->fd, size) < 0) {
+                perror("ftruncate in libtrace_scb_init");
+                close(buf->fd);
+                buf->fd = -1;
+                buf->address = NULL;
+        } else {
+                buf->address = mmap(NULL, 2 * size, PROT_NONE,
+                                MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+                mmap(buf->address, size, PROT_READ | PROT_WRITE,
+                                MAP_SHARED | MAP_FIXED, buf->fd, 0);
+                mmap(buf->address + size, size, PROT_READ | PROT_WRITE,
+                                MAP_SHARED | MAP_FIXED, buf->fd, 0);
+        }
+        buf->read_offset = 0;
+        buf->write_offset = 0;
+        buf->count_bytes = size;
+
+        if (buf->address) {
+                return 0;
+        }
+        return -1;
+}
+
+DLLEXPORT void libtrace_scb_destroy(libtrace_scb_t *buf) {
+        /* TODO shm_unlink the file name if we used shm_open? */
+
+        if (buf->address) {
+                munmap(buf->address, buf->count_bytes * 2);
+        }
+        if (buf->fd != -1) {
+                close(buf->fd);
+        }
+}
+
+DLLEXPORT int libtrace_scb_recv_sock(libtrace_scb_t *buf, int sock,
+                int recvflags) {
+        int space = buf->count_bytes - (buf->write_offset - buf->read_offset);
+        int ret;
+
+        if (buf->address == NULL) {
+                return -1;
+        }
+
+        if (space == 0) {
+                return buf->count_bytes;
+        }
+
+        ret = recv(sock, buf->address + buf->write_offset, space, recvflags);
+        if (ret < 0) {
+                return ret;
+        }
+        buf->write_offset += ret;
+        return (buf->write_offset - buf->read_offset);
+}
+
+DLLEXPORT uint8_t *libtrace_scb_get_read(libtrace_scb_t *buf,
+                uint32_t *available) {
+
+        if (buf->address == NULL) {
+                return NULL;
+        }
+        *available = buf->write_offset - buf->read_offset;
+        return buf->address + buf->read_offset;
+}
+
+DLLEXPORT void libtrace_scb_advance_read(libtrace_scb_t *buf,
+                uint32_t forward) {
+
+        buf->read_offset += forward;
+        if (buf->read_offset >= buf->count_bytes) {
+                buf->read_offset -= buf->count_bytes;
+                buf->write_offset -= buf->count_bytes;
+        }
+}

lib/data-struct/simple_circular_buffer.h

@@ -0,0 +1,24 @@
+#ifndef LIBTRACE_SCB_H_
+#define LIBTRACE_SCB_H_
+
+#include "libtrace.h"
+
+typedef struct libtracescb {
+        uint8_t *address;
+        uint32_t count_bytes;
+        uint32_t write_offset;
+        uint32_t read_offset;
+        int fd;
+} libtrace_scb_t;
+
+
+DLLEXPORT int libtrace_scb_init(libtrace_scb_t *buf, uint32_t size,
+                uint16_t id);
+DLLEXPORT void libtrace_scb_destroy(libtrace_scb_t *buf);
+DLLEXPORT int libtrace_scb_recv_sock(libtrace_scb_t *buf, int sock,
+                int recvflags);
+DLLEXPORT uint8_t *libtrace_scb_get_read(libtrace_scb_t *buf,
+                uint32_t *available);
+DLLEXPORT void libtrace_scb_advance_read(libtrace_scb_t *buf, uint32_t forward);
+
+#endif

lib/format_atmhdr.c

@@ -201,6 +201,7 @@ static struct libtrace_format_t atmhdr = {
         atmhdr_prepare_packet,		/* prepare_packet */
 	NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         atmhdr_get_link_type,        	/* get_link_type */
         NULL,                           /* get_direction */
         NULL,                           /* set_direction */

lib/format_bpf.c

@@ -370,7 +370,8 @@ static int bpf_config_input(libtrace_t *libtrace,
 		case TRACE_OPTION_HASHER:
 			/* TODO investigate hashing in BSD? */
 			break;
-
+		case TRACE_OPTION_REPLAY_SPEEDUP:
+			break;
 		/* Avoid default: so that future options will cause a warning
 		 * here to remind us to implement it, or flag it as
 		 * unimplementable
@@ -615,6 +616,7 @@ static struct libtrace_format_t bpf = {
 	bpf_prepare_packet, 	/* prepare_packet */
 	NULL,			/* fin_packet */
 	NULL,			/* write_packet */
+	NULL,			/* flush_output */
 	bpf_get_link_type,	/* get_link_type */
 	bpf_get_direction,	/* get_direction */
 	NULL,			/* set_direction */
@@ -665,6 +667,7 @@ static struct libtrace_format_t bpf = {
 	bpf_prepare_packet, 	/* prepare_packet */
 	NULL,			/* fin_packet */
 	NULL,			/* write_packet */
+	NULL,			/* flush_output */
 	bpf_get_link_type,	/* get_link_type */
 	bpf_get_direction,	/* get_direction */
 	NULL,			/* set_direction */

lib/format_dag24.c

@@ -528,6 +528,7 @@ static struct libtrace_format_t dag = {
         dag_prepare_packet,		/* prepare_packet */
 	NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         erf_get_link_type,              /* get_link_type */
         erf_get_direction,              /* get_direction */
         erf_set_direction,              /* set_direction */

lib/format_dag25.c

@@ -607,6 +607,7 @@ static int dag_config_input(libtrace_t *libtrace, trace_option_t option,
 		 * cards */
 		return -1;
 	case TRACE_OPTION_EVENT_REALTIME:
+        case TRACE_OPTION_REPLAY_SPEEDUP:
 		/* Live capture is always going to be realtime */
 		return -1;
 	case TRACE_OPTION_HASHER:
@@ -1562,6 +1563,7 @@ static struct libtrace_format_t dag = {
 	dag_prepare_packet,		/* prepare_packet */
 	NULL,                           /* fin_packet */
 	dag_write_packet,               /* write_packet */
+	NULL,                           /* flush_output */
 	erf_get_link_type,              /* get_link_type */
 	erf_get_direction,              /* get_direction */
 	erf_set_direction,              /* set_direction */