Merge branch 'rc-4.0.3'

AUTHORS

@@ -32,6 +32,9 @@ In no particular order, we would like to recognise:
  * "EaseTheWorld" for their work with improving the packet statistics API, as
    well as reporting several bugs
  * Richard Cziva for contributing to the DPDK support
+ * Robert Zeh for resolving some wandio errors that were causing crashes
+ * Anthony Coddington for adding ERF provenance support and fixing a number
+   of other ERF/DAG issues
  * Jamie Curtis for fixing a couple of bugs many many years ago
  * Brendon Jones for creating the original Windows DLLs and writing bits of
    code here and there

README

@@ -1,7 +1,7 @@
-libtrace 4.0.2
+libtrace 4.0.3
 
 ---------------------------------------------------------------------------
-Copyright (c) 2007-2017 The University of Waikato, Hamilton, New Zealand.
+Copyright (c) 2007-2018 The University of Waikato, Hamilton, New Zealand.
 All rights reserved.
 
 This code has been developed by the University of Waikato WAND
@@ -26,14 +26,14 @@ Further information about libtrace, see
 http://research.wand.net.nz/software/libtrace.php
 
 Bugs should be reported by either emailing [email protected] or filing
-an issue at https://github.com/wanduow/libtrace
+an issue at https://github.com/LibtraceTeam/libtrace
 
 It is licensed under the GNU Lesser General Public License (GPL) version 3.
 Please see the included files COPYING and COPYING.LESSER for details of this
 license.
 
 A detailed ChangeLog can be found on the libtrace wiki:
-https://github.com/wanduow/libtrace/wiki/ChangeLog
+https://github.com/LibtraceTeam/libtrace/wiki/ChangeLog
 
 Documentation, usage instructions and a detailed tutorial can also found
 on the libtrace wiki.

configure.in

@@ -3,11 +3,11 @@
 # Now you only need to update the version number in two places - below,
 # and in the README
 
-AC_INIT([libtrace],[4.0.2],[[email protected]],[libtrace])
+AC_INIT([libtrace],[4.0.3],[[email protected]],[libtrace])
 
 LIBTRACE_MAJOR=4
 LIBTRACE_MID=0
-LIBTRACE_MINOR=2
+LIBTRACE_MINOR=3
 
 # OpenSolaris hides libraries like libncurses in /usr/gnu/lib, which is not
 # searched by default - add it to LDFLAGS so we at least have a chance of 
@@ -246,7 +246,7 @@ AC_ARG_WITH(man,
 if test "$pcapfound" = 0; then
 	AC_MSG_ERROR(libpcap0.8 or greater is required to compile libtrace. If you have installed it in a non-standard location please use LDFLAGS to specify the location of the library)
 else
-	TOOL_LIBS="$TOOL_LIBS -lpcap"
+	TOOLS_LIBS="$TOOLS_LIBS -lpcap"
 	LIBTRACE_LIBS="$LIBTRACE_LIBS -lpcap"
 	AC_DEFINE([HAVE_LIBPCAP],1,[compile with libpcap support])
 
@@ -402,18 +402,21 @@ libtrace_dpdk=false
 if test "$want_dpdk" != no; then
 	# So instead simply check for existence
 	if test "$RTE_SDK" != ""; then
-		AC_CHECK_FILE("$RTE_SDK/$RTE_TARGET/lib/libintel_dpdk.a", dpdk_found="libintel_dpdk.a", dpdk_found=0)
+		AC_CHECK_FILE("$RTE_SDK/$RTE_TARGET/lib/libintel_dpdk.a", dpdk_found=":libintel_dpdk.a", dpdk_found=0)
 	fi
 	# DPDK 2.1.0+ renames this to libdpdk from libintel_dpdk
 	if test "$dpdk_found" = 0 -a "$RTE_SDK" != ""; then
-		AC_CHECK_FILE("$RTE_SDK/$RTE_TARGET/lib/libdpdk.a", dpdk_found="libdpdk.a", dpdk_found=0)
+		AC_CHECK_FILE("$RTE_SDK/$RTE_TARGET/lib/libdpdk.a", dpdk_found=":libdpdk.a", dpdk_found=0)
+	fi
+	if test "$dpdk_found" = 0 -a "$RTE_SDK" != ""; then
+		AC_CHECK_FILE("$RTE_SDK/$RTE_TARGET/lib/libdpdk.so", dpdk_found="dpdk", dpdk_found=0)
 	fi
 	if test "$dpdk_found" != 0 -a "$RTE_SDK" != ""; then
 		# Save these now so that they can be re-exported later
 		AC_SUBST([RTE_TARGET])
 		AC_SUBST([RTE_SDK])
 		# Force dpdk library to be statically linked to allow compiler optimisations
-		LIBTRACE_LIBS="$LIBTRACE_LIBS -Wl,--whole-archive -Wl,-l:$dpdk_found -Wl,--no-whole-archive -Wl,-lm"
+		LIBTRACE_LIBS="$LIBTRACE_LIBS -Wl,--whole-archive -Wl,-l$dpdk_found -Wl,--no-whole-archive -Wl,-lm"
 		AC_DEFINE(HAVE_DPDK,1,[conditional for building with DPDK live capture support])
 		libtrace_dpdk=true
 	fi

lib/Makefile.am

@@ -1,7 +1,7 @@
 lib_LTLIBRARIES = libtrace.la
 include_HEADERS = libtrace.h libtrace_parallel.h
 pkginclude_HEADERS = dagformat.h lt_inttypes.h daglegacy.h \
-        rt_protocol.h erftypes.h \
+        rt_protocol.h erftypes.h pthread_spinlock.h \
         data-struct/ring_buffer.h data-struct/object_cache.h \
         data-struct/vector.h \
         data-struct/deque.h data-struct/linked_list.h \
@@ -33,7 +33,7 @@ BPFJITSOURCE=
 endif
 
 if HAVE_DPDK
-NATIVEFORMATS+= format_dpdk.c
+NATIVEFORMATS+= format_dpdk.c format_dpdkndag.c
 # So we also make libtrace.mk in dpdk otherwise automake tries to expand
 # it too early which I cannot seem to stop unless we use a path that
 # doesn't exist currently

lib/data-struct/message_queue.h

@@ -25,8 +25,8 @@
  */
 #include <pthread.h>
 #include <limits.h>
-#include "../libtrace.h"
-#include "../pthread_spinlock.h"
+#include "libtrace.h"
+#include "pthread_spinlock.h"
 
 #ifndef LIBTRACE_MESSAGE_QUEUE
 #define LIBTRACE_MESSAGE_QUEUE
@@ -39,12 +39,16 @@ typedef struct libtrace_message_queue_t {
 	pthread_spinlock_t spin;
 } libtrace_message_queue_t;
 
-void libtrace_message_queue_init(libtrace_message_queue_t *mq, size_t message_len);
-int libtrace_message_queue_put(libtrace_message_queue_t *mq, const void *message);
-int libtrace_message_queue_count(const libtrace_message_queue_t *mq);
-int libtrace_message_queue_get(libtrace_message_queue_t *mq, void *message);
-int libtrace_message_queue_try_get(libtrace_message_queue_t *mq, void *message);
-void libtrace_message_queue_destroy(libtrace_message_queue_t *mq);
-int libtrace_message_queue_get_fd(libtrace_message_queue_t *mq);
+DLLEXPORT void libtrace_message_queue_init(libtrace_message_queue_t *mq,
+        size_t message_len);
+DLLEXPORT int libtrace_message_queue_put(libtrace_message_queue_t *mq,
+        const void *message);
+DLLEXPORT int libtrace_message_queue_count(const libtrace_message_queue_t *mq);
+DLLEXPORT int libtrace_message_queue_get(libtrace_message_queue_t *mq,
+        void *message);
+DLLEXPORT int libtrace_message_queue_try_get(libtrace_message_queue_t *mq,
+        void *message);
+DLLEXPORT void libtrace_message_queue_destroy(libtrace_message_queue_t *mq);
+DLLEXPORT int libtrace_message_queue_get_fd(libtrace_message_queue_t *mq);
 
 #endif

lib/data-struct/ring_buffer.h

@@ -25,8 +25,8 @@
  */
 #include <pthread.h>
 #include <semaphore.h>
-#include "../libtrace.h"
-#include "../pthread_spinlock.h"
+#include "libtrace.h"
+#include "pthread_spinlock.h"
 
 #ifndef LIBTRACE_RINGBUFFER_H
 #define LIBTRACE_RINGBUFFER_H

lib/erftypes.h

@@ -161,10 +161,22 @@
 #define TYPE_RAW_LINK		24
 #endif
 
+/** Provenance Metadata Record */
+#ifndef ERF_TYPE_META
+#define ERF_TYPE_META   27
+#endif
+/* TODO: Endace has deprecated TYPE_* in favour of ERF_TYPE_*. New types do not have TYPE_* aliases. */
+#ifndef TYPE_META
+#define TYPE_META       ERF_TYPE_META
+#endif
+
 /** Padding record */
 #ifndef TYPE_PAD
 #define TYPE_PAD		48
 #endif
 
+#ifndef ERF_TYPE_MAX
+#define ERF_TYPE_MAX    TYPE_PAD
+#endif
 
 #endif /* _ERFTYPES_H_ */

lib/format_atmhdr.c

@@ -136,7 +136,7 @@ static int atmhdr_read_packet(libtrace_t *libtrace, libtrace_packet_t *packet) {
 	if ((numbytes=wandio_read(libtrace->io, buffer, (size_t)12)) != 12)
 	{
 		if (numbytes != 0) {
-			trace_set_err(libtrace,errno,"read(%s)",libtrace->uridata);
+			trace_set_err(libtrace,TRACE_ERR_WANDIO_FAILED,"read(%s)",libtrace->uridata);
 		}
 		return numbytes;
 	}

lib/format_dag25.c

@@ -186,7 +186,9 @@ static int dag_get_padding(const libtrace_packet_t *packet)
 		dag_record_t *erfptr = (dag_record_t *)packet->header;
 		switch(erfptr->type) {
 			case TYPE_ETH:
+			case TYPE_COLOR_ETH:
 			case TYPE_DSM_COLOR_ETH:
+			case TYPE_COLOR_HASH_ETH:
 				return 2;
 			default: 		return 0;
 		}
@@ -625,7 +627,7 @@ static int dag_start_output(libtrace_out_t *libtrace)
 	nopoll = zero;
 
 	/* Attach and start the DAG stream */
-	if (dag_attach_stream(FORMAT_DATA_OUT->device->fd,
+	if (dag_attach_stream64(FORMAT_DATA_OUT->device->fd,
 			FORMAT_DATA_OUT->dagstream, 0, 4 * 1024 * 1024) < 0) {
 		trace_set_err_out(libtrace, errno, "Cannot attach DAG stream");
 		return -1;
@@ -639,7 +641,7 @@ static int dag_start_output(libtrace_out_t *libtrace)
 	FORMAT_DATA_OUT->stream_attached = 1;
 
 	/* We don't want the dag card to do any sleeping */
-	dag_set_stream_poll(FORMAT_DATA_OUT->device->fd,
+	dag_set_stream_poll64(FORMAT_DATA_OUT->device->fd,
 			FORMAT_DATA_OUT->dagstream, 0, &zero,
 			&nopoll);
 
@@ -657,7 +659,7 @@ static int dag_start_input_stream(libtrace_t *libtrace,
 	nopoll = zero;
 
 	/* Attach and start the DAG stream */
-	if (dag_attach_stream(FORMAT_DATA->device->fd,
+	if (dag_attach_stream64(FORMAT_DATA->device->fd,
 			      stream->dagstream, 0, 0) < 0) {
 		trace_set_err(libtrace, errno, "Cannot attach DAG stream #%u",
 		              stream->dagstream);
@@ -673,7 +675,7 @@ static int dag_start_input_stream(libtrace_t *libtrace,
 	FORMAT_DATA->stream_attached = 1;
 
 	/* We don't want the dag card to do any sleeping */
-	if (dag_set_stream_poll(FORMAT_DATA->device->fd,
+	if (dag_set_stream_poll64(FORMAT_DATA->device->fd,
 			    stream->dagstream, 0, &zero,
 			    &nopoll) < 0) {
 		trace_set_err(libtrace, errno,
@@ -865,10 +867,10 @@ static int dag_fin_output(libtrace_out_t *libtrace)
 
 	/* Wait until the buffer is nearly clear before exiting the program,
 	 * as we will lose packets otherwise */
-	dag_tx_get_stream_space
+	dag_tx_get_stream_space64
 		(FORMAT_DATA_OUT->device->fd,
 		 FORMAT_DATA_OUT->dagstream,
-		 dag_get_stream_buffer_size(FORMAT_DATA_OUT->device->fd,
+		 dag_get_stream_buffer_size64(FORMAT_DATA_OUT->device->fd,
 					    FORMAT_DATA_OUT->dagstream) - 8);
 
 	/* Need the lock, since we're going to be handling the device list */
@@ -1049,7 +1051,7 @@ static int dag_prepare_packet_stream(libtrace_t *libtrace,
 	/* Update the dropped packets counter */
 	/* No loss counter for DSM coloured records - have to use some
 	 * other API */
-	if (erfptr->type == TYPE_DSM_COLOR_ETH) {
+	if (erf_is_color_type(erfptr->type)) {
 		/* TODO */
 	} else {
 		/* Use the ERF loss counter */
@@ -1099,7 +1101,7 @@ static int dag_dump_packet(libtrace_out_t *libtrace,
 	 */
 	if (FORMAT_DATA_OUT->waiting == 0) {
 		FORMAT_DATA_OUT->txbuffer =
-			dag_tx_get_stream_space(FORMAT_DATA_OUT->device->fd,
+			dag_tx_get_stream_space64(FORMAT_DATA_OUT->device->fd,
 						FORMAT_DATA_OUT->dagstream,
 						16908288);
 	}
@@ -1285,7 +1287,7 @@ static int dag_read_packet_stream(libtrace_t *libtrace,
 		packet->buffer = 0;
 	}
 
-	if (dag_set_stream_poll(FORMAT_DATA->device->fd, stream_data->dagstream,
+	if (dag_set_stream_poll64(FORMAT_DATA->device->fd, stream_data->dagstream,
 				sizeof(dag_record_t), &maxwait,
 				&pollwait) == -1) {
 		trace_set_err(libtrace, errno, "dag_set_stream_poll");
@@ -1391,7 +1393,7 @@ static libtrace_eventobj_t trace_event_dag(libtrace_t *libtrace,
 		return event;
 	}
 
-	if (dag_set_stream_poll(FORMAT_DATA->device->fd,
+	if (dag_set_stream_poll64(FORMAT_DATA->device->fd,
 				FORMAT_DATA_FIRST->dagstream, 0, &minwait,
 				&minwait) == -1) {
 		trace_set_err(libtrace, errno, "dag_set_stream_poll");