Changes in / [314239a:97d0351]

Files:

• AUTHORS (modified) (1 diff)
• README (modified) (1 diff)
• configure.in (modified) (10 diffs)
• lib/Makefile.am (modified) (4 diffs)
• lib/data-struct/simple_circular_buffer.c (added)
• lib/data-struct/simple_circular_buffer.h (added)
• lib/format_atmhdr.c (modified) (1 diff)
• lib/format_bpf.c (modified) (3 diffs)
• lib/format_dag24.c (modified) (1 diff)
• lib/format_dag25.c (modified) (2 diffs)
• lib/format_dpdk.c (modified) (13 diffs)
• lib/format_dpdk.h (modified) (1 diff)
• lib/format_dpdkndag.c (modified) (1 diff)
• lib/format_duck.c (modified) (1 diff)
• lib/format_erf.c (modified) (3 diffs)
• lib/format_etsilive.c (added)
• lib/format_helper.c (modified) (4 diffs)
• lib/format_legacy.c (modified) (4 diffs)
• lib/format_linux_common.c (modified) (4 diffs)
• lib/format_linux_common.h (modified) (1 diff)
• lib/format_linux_int.c (modified) (2 diffs)
• lib/format_linux_ring.c (modified) (8 diffs)
• lib/format_ndag.c (modified) (12 diffs)
• lib/format_pcap.c (modified) (3 diffs)
• lib/format_pcapfile.c (modified) (3 diffs)
• lib/format_pcapng.c (modified) (39 diffs)
• lib/format_rt.c (modified) (1 diff)
• lib/format_tsh.c (modified) (2 diffs)
• lib/libtrace.h.in (modified) (8 diffs)
• lib/libtrace_int.h (modified) (6 diffs)
• lib/libtrace_parallel.h (modified) (2 diffs)
• lib/linktypes.c (modified) (2 diffs)
• lib/protocols_l2.c (modified) (5 diffs)
• lib/protocols_pktmeta.c (modified) (4 diffs)
• lib/strndup.c (modified) (1 diff)
• lib/trace.c (modified) (13 diffs)
• lib/trace_parallel.c (modified) (8 diffs)
• libpacketdump/Makefile.am (modified) (2 diffs)
• libpacketdump/eth_2054.c (modified) (4 diffs)
• libpacketdump/link_22.c (added)
• test/do-test-build-dpdk.sh (modified) (4 diffs)
• tools/traceanon/traceanon.1 (modified) (1 diff)
• tools/tracemerge/tracemerge.1 (modified) (1 diff)
• tools/tracepktdump/tracepktdump.cc (modified) (1 diff)
• tools/tracereplay/tracereplay.1 (modified) (3 diffs)
• tools/tracereplay/tracereplay.c (modified) (7 diffs)
• tools/tracereport/misc_report.c (modified) (1 diff)
• tools/tracesplit/tracesplit.1 (modified) (1 diff)
• tools/tracesplit/tracesplit.c (modified) (1 diff)

AUTHORS

@@ -36 +36 @@
  * Anthony Coddington for adding ERF provenance support and fixing a number
    of other ERF/DAG issues
+ * Tim Dawson for fixing issues with building against musl libc
+ * Hendrik Leppelsack for reporting and fixing some errors in the tool manpages
  * Jamie Curtis for fixing a couple of bugs many many years ago
  * Brendon Jones for creating the original Windows DLLs and writing bits of

README

@@ -1 @@
-libtrace 4.0.3
+libtrace 4.0.4
 
 ---------------------------------------------------------------------------

configure.in

@@ -4 +4 @@
 # and in the README
 
-AC_INIT([libtrace],[4.0.3],[contact@wand.net.nz],[libtrace])
+AC_INIT([libtrace],[4.0.4],[contact@wand.net.nz],[libtrace])
 
 LIBTRACE_MAJOR=4
 LIBTRACE_MID=0
-LIBTRACE_MINOR=3
+LIBTRACE_MINOR=4
 
 # OpenSolaris hides libraries like libncurses in /usr/gnu/lib, which is not
@@ -87 +87 @@
 AC_FUNC_REALLOC
 
-# *BSD doesn't have strndup. Currently provide our own.
-AC_REPLACE_FUNCS(strndup)
-
-
 # Checks for typedefs, structures, and compiler characteristics.
 AC_C_CONST
@@ -126 +122 @@
 
 # Fail if any of these functions are missing
-AC_CHECK_FUNCS(socket strdup strlcpy strcasecmp strncasecmp snprintf vsnprintf recvmmsg)
+AC_CHECK_DECLS([socket, strdup, strlcpy, strcasecmp, strncasecmp, snprintf, vsnprintf, recvmmsg, strndup])
 
 AC_CHECK_SIZEOF([long int])
@@ -202 +198 @@
 
 # Check for libwandio (no longer bundled with libtrace)
-AC_CHECK_LIB(wandio,wandio_create,wandiofound=1,wandiofound=0)
+AC_CHECK_LIB(wandio,wandio_wflush,wandiofound=1,wandiofound=0)
 if test "$wandiofound" = 0; then
-        AC_MSG_ERROR(libwandio is required to compile libtrace. If you have installed it in a non-standard location please use LDFLAGS to specify the location of the library. WANDIO can be obtained from http://research.wand.net.nz/software/libwandio.php)
+        AC_MSG_ERROR(libwandio 1.0.5 or better is required to compile this version of libtrace. If you have installed libwandio in a non-standard location please use LDFLAGS to specify the location of the library. WANDIO can be obtained from http://research.wand.net.nz/software/libwandio.php)
 else
         LIBTRACE_LIBS="$LIBTRACE_LIBS -lwandio"
@@ -212 +208 @@
 
 AC_CHECK_LIB(crypto, EVP_EncryptInit_ex, cryptofound=1, cryptofound=0)
+if test "$cryptofound" = 1; then
+        AC_CHECK_HEADER(openssl/evp.h, cryptofound=1, cryptofound=0)
+fi
+AC_CHECK_LIB(tasn1, asn1_array2tree, asn1found=1, asn1found=0)
 
 # Check for libpcap
@@ -455 +455 @@
 AC_CHECK_LIB(numa, numa_node_to_cpus, have_numa=1, have_numa=0)
 
+# Need libwandder for ETSI live decoding
+AC_CHECK_LIB(wandder, init_wandder_decoder, have_wandder=1, have_wandder=0)
+
 # Checks for various "optional" libraries
 AC_CHECK_LIB(pthread, pthread_create, have_pthread=1, have_pthread=0)
@@ -482 +485 @@
         with_numa=yes
 else
-        AC_DEFINE(HAVE_LIBNUMA, 0, [Set to 1 if libnuma is supported])
         with_numa=no
+fi
+
+if test "$have_wandder" = 1; then
+        LIBTRACE_LIBS="$LIBTRACE_LIBS -lwandder"
+        AC_DEFINE(HAVE_WANDDER, 1, [Set to 1 if libwandder is available])
+        wandder_avail=yes
+else
+        wandder_avail=no
 fi
 
@@ -533 +543 @@
         with_clock_gettime=yes
 else
-        AC_DEFINE(HAVE_CLOCK_GETTIME, 0, [Set to 1 if clock_gettime is supported])
         with_clock_gettime=no
 fi
 
+
+have_memfd_create=no
+AC_RUN_IFELSE([AC_LANG_PROGRAM([
+#include <sys/syscall.h>
+],[
+#ifndef __NR_memfd_create
+exit(1)
+#endif
+])], [have_memfd_create=yes])
+
+if test "$have_memfd_create" = "yes"; then
+        AC_DEFINE(HAVE_MEMFD_CREATE, 1, [Set to 1 if kernel supports memfd_create syscall])
+fi
 
 # Stupid AC_SEARCH_LIB appends stuff to LIBS even when I don't want it to, so
@@ -611 +633 @@
 AM_CONDITIONAL([HAVE_DAG], [test "$libtrace_dag" = true])
 AM_CONDITIONAL([HAVE_DPDK], [test "$libtrace_dpdk" = true])
+AM_CONDITIONAL([HAVE_WANDDER], [test "x$wandder_avail" = "xyes"])
 AM_CONDITIONAL([DAG2_4], [test "$libtrace_dag_version" = 24])
 AM_CONDITIONAL([DAG2_5], [test "$libtrace_dag_version" = 25])
@@ -692 +715 @@
 fi
 reportopt "Compiled with LLVM BPF JIT support" $JIT
+reportopt "Compiled with live ETSI LI support (requires libwandder)" $wandder_avail
 reportopt "Building man pages/documentation" $libtrace_doxygen
 reportopt "Building tracetop (requires libncurses)" $with_ncurses
-reportopt "Building traceanon with CryptoPan (requires libcrypto)" $have_crypto
+reportopt "Building traceanon with CryptoPan (requires libcrypto and openssl/evp.h)" $have_crypto
 
 # Report any errors relating to missing bison, flex, etc.

lib/Makefile.am

@@ -7 +7 @@
         data-struct/deque.h data-struct/linked_list.h \
         data-struct/buckets.h data-struct/sliding_window.h \
-        data-struct/message_queue.h hash_toeplitz.h
+        data-struct/message_queue.h hash_toeplitz.h \
+        data-struct/simple_circular_buffer.h
 
 AM_CFLAGS=@LIBCFLAGS@ @CFLAG_VISIBILITY@ -pthread
@@ -51 +52 @@
 endif
 
+if HAVE_WANDDER
+ETSISOURCES=format_etsilive.c
+else
+ETSISOURCES=
+endif
+
 libtrace_la_SOURCES = trace.c trace_parallel.c common.h \
                 format_erf.c format_pcap.c format_legacy.c \
@@ -63 +70 @@
                 protocols_application.c \
                 $(DAGSOURCE) format_erf.h format_ndag.c format_ndag.h \
-                $(BPFJITSOURCE) \
+                $(BPFJITSOURCE) $(ETSISOURCES) \
                 libtrace_arphrd.h \
                 data-struct/ring_buffer.c data-struct/vector.c \
@@ -69 +76 @@
                 data-struct/sliding_window.c data-struct/object_cache.c \
                 data-struct/linked_list.c hash_toeplitz.c combiner_ordered.c \
-                data-struct/buckets.c \
+                data-struct/buckets.c data-struct/simple_circular_buffer.c \
                 combiner_sorted.c combiner_unordered.c \
-                pthread_spinlock.c pthread_spinlock.h
+                pthread_spinlock.c pthread_spinlock.h \
+                strndup.c
 
 if DAG2_4

lib/format_atmhdr.c

@@ -202 +202 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         atmhdr_get_link_type,           /* get_link_type */
         NULL,                           /* get_direction */

lib/format_bpf.c

@@ -371 +371 @@
                         /* TODO investigate hashing in BSD? */
                         break;
-
+                case TRACE_OPTION_REPLAY_SPEEDUP:
+                        break;
                 /* Avoid default: so that future options will cause a warning
                  * here to remind us to implement it, or flag it as
@@ -616 +617 @@
         NULL,                   /* fin_packet */
         NULL,                   /* write_packet */
+        NULL,                   /* flush_output */
         bpf_get_link_type,      /* get_link_type */
         bpf_get_direction,      /* get_direction */
@@ -666 +668 @@
         NULL,                   /* fin_packet */
         NULL,                   /* write_packet */
+        NULL,                   /* flush_output */
         bpf_get_link_type,      /* get_link_type */
         bpf_get_direction,      /* get_direction */

lib/format_dag24.c

@@ -529 +529 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         erf_get_link_type,              /* get_link_type */
         erf_get_direction,              /* get_direction */

lib/format_dag25.c

@@ -608 +608 @@
                 return -1;
         case TRACE_OPTION_EVENT_REALTIME:
+        case TRACE_OPTION_REPLAY_SPEEDUP:
                 /* Live capture is always going to be realtime */
                 return -1;
@@ -1563 +1564 @@
         NULL,                           /* fin_packet */
         dag_write_packet,               /* write_packet */
+        NULL,                           /* flush_output */
         erf_get_link_type,              /* get_link_type */
         erf_get_direction,              /* get_direction */

lib/format_dpdk.c

@@ -57 +57 @@
 #include <math.h>
 
-#if HAVE_LIBNUMA
+#ifdef HAVE_LIBNUMA
 #include <numa.h>
 #endif
@@ -90 +90 @@
 struct dpdk_format_data_t {
         int8_t promisc; /* promiscuous mode - RX only */
-        uint8_t port; /* Always 0 we only whitelist a single port - Shared TX & RX */
-        uint8_t nb_ports; /* Total number of usable ports on system should be 1 */
         uint8_t paused; /* See paused_state */
+        portid_t port; /* Always 0 we only whitelist a single port - Shared TX & RX */
+        portid_t nb_ports; /* Total number of usable ports on system should be 1 */
         uint16_t link_speed; /* Link speed 10,100,1000,10000 etc. */
         int snaplen; /* The snap length for the capture - RX only */
@@ -214 +214 @@
         int matches;
         assert(str);
+#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 1)
         matches = sscanf(str, "%8"SCNx32":%2"SCNx8":%2"SCNx8".%2"SCNx8"-%ld",
                          &addr->domain, &addr->bus, &addr->devid,
                          &addr->function, core);
+#else
+        matches = sscanf(str, "%4"SCNx16":%2"SCNx8":%2"SCNx8".%2"SCNx8"-%ld",
+                         &addr->domain, &addr->bus, &addr->devid,
+                         &addr->function, core);
+#endif
         if (matches >= 4) {
                 return 0;
@@ -454 +460 @@
         }
 
-#if HAVE_LIBNUMA
+#ifdef HAVE_LIBNUMA
         format_data->nic_numa_node = pci_to_numa(&use_addr);
         if (my_cpu < 0) {
@@ -696 +702 @@
         case TRACE_OPTION_META_FREQ:
         case TRACE_OPTION_EVENT_REALTIME:
+        case TRACE_OPTION_REPLAY_SPEEDUP:
                 break;
         /* Avoid default: so that future options will cause a warning
@@ -816 +823 @@
  * @param cb_arg The dpdk_format_data_t structure associated with the format
  */
-static int dpdk_lsc_callback(uint16_t port, enum rte_eth_event_type event,
+#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 1)
+static int dpdk_lsc_callback(portid_t port, enum rte_eth_event_type event,
                               void *cb_arg, void *retparam UNUSED) {
+#else
+static void dpdk_lsc_callback(portid_t port, enum rte_eth_event_type event,
+                              void *cb_arg) {
+#endif
         struct dpdk_format_data_t * format_data = cb_arg;
         struct rte_eth_link link_info;
@@ -852 +864 @@
         }
 #endif
+#if RTE_VERSION >= RTE_VERSION_NUM(17, 8, 0, 1)
         return 0;
+#endif
 }
 
@@ -878 +892 @@
          * these to any particular physical core */
         if (real) {
-#if HAVE_LIBNUMA
+#ifdef HAVE_LIBNUMA
                 for (i = 0; i < RTE_MAX_LCORE; ++i) {
                         if (!rte_lcore_is_enabled(i) && numa_node_of_cpu(i) == socket) {
@@ -1171 +1185 @@
         if (ret < 0) {
                 snprintf(err, errlen, "Intel DPDK - Cannot configure TX queue"
-                         " on port %"PRIu8" : %s", format_data->port,
+                         " on port %d : %s", (int)format_data->port,
                          strerror(-ret));
                 return -1;
@@ -1220 +1234 @@
                 if (ret < 0) {
                         snprintf(err, errlen, "Intel DPDK - Cannot configure"
-                                 " RX queue on port %"PRIu8" : %s",
-                                 format_data->port,
+                                 " RX queue on port %d : %s",
+                                 (int)format_data->port,
                                  strerror(-ret));
                         return -1;
@@ -1290 +1304 @@
 }
 
-static inline size_t dpdk_get_max_rx_queues (uint8_t port_id) {
+static inline size_t dpdk_get_max_rx_queues (portid_t port_id) {
         struct rte_eth_dev_info dev_info;
         rte_eth_dev_info_get(port_id, &dev_info);
@@ -1511 +1525 @@
                 memcpy(rte_pktmbuf_append(m_buff[0], caplen), packet->payload, caplen);
                 do {
-                        ret = rte_eth_tx_burst(0 /*queue TODO*/, FORMAT(trace)->port, m_buff, 1);
+                        ret = rte_eth_tx_burst(FORMAT(trace)->port, 0 /*queue TODO*/, m_buff, 1);
                 } while (ret != 1);
         }
@@ -2146 +2160 @@
         dpdk_fin_packet,                    /* fin_packet */
         dpdk_write_packet,                  /* write_packet */
+        NULL,                               /* flush_output */
         dpdk_get_link_type,                 /* get_link_type */
         dpdk_get_direction,                 /* get_direction */

lib/format_dpdk.h

@@ -113 +113 @@
 #define rte_mempool_in_use_count rte_mempool_free_count
 #endif
+
+/* 17.05-rc1 deprecated, 17.08 removed
+ * rte_set_log_level -> rte_log_set_global_level
+ */
+#if RTE_VERSION < RTE_VERSION_NUM(17, 5, 0, 1)
+#define rte_log_set_global_level rte_set_log_level
+#endif
+
+/* 17.11-rc1 increases port size from 8 to 16bits
+ */
+#if RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 1)
+typedef uint16_t portid_t;
+#else
+typedef uint8_t portid_t;
+#endif
+
 
 #include <rte_per_lcore.h>

lib/format_dpdkndag.c

@@ -733 +733 @@
         NULL,                   /* fin_packet */
         NULL,                   /* write_packet */
+        NULL,                   /* flush_output */
         erf_get_link_type,      /* get_link_type */
         erf_get_direction,      /* get_direction */

lib/format_duck.c

@@ -337 +337 @@
         NULL,                           /* fin_packet */
         duck_write_packet,              /* write_packet */
+        NULL,                           /* flush_output */
         duck_get_link_type,             /* get_link_type */
         NULL,                           /* get_direction */

lib/format_erf.c

@@ -597 +597 @@
         }
         return numbytes + framinglen;
+}
+
+static int erf_flush_output(libtrace_out_t *libtrace) {
+        return wandio_wflush(OUTPUT->file);
 }
 
@@ -868 +872 @@
         NULL,                           /* fin_packet */
         erf_write_packet,               /* write_packet */
+        erf_flush_output,               /* flush_output */
         erf_get_link_type,              /* get_link_type */
         erf_get_direction,              /* get_direction */
@@ -912 +917 @@
         NULL,                           /* fin_packet */
         erf_write_packet,               /* write_packet */
+        erf_flush_output,               /* flush_output */
         erf_get_link_type,              /* get_link_type */
         erf_get_direction,              /* get_direction */

lib/format_helper.c

@@ -127 +127 @@
         double ts;
         double now;
+        double sincebeginnow = 0;
+        double sincebegintrace = 0;
+
 #ifdef WIN32
         struct __timeb64 tstruct;
@@ -182 +185 @@
 
         
-        if (fabs(trace->event.tdelta)>1e-9) {
-                /* Subtract the tdelta from the walltime to get a suitable
+        if (fabs(trace->event.first_ts)>1e-9) {
+                /* Subtract the tdelta from the starting times to get a suitable
                  * "relative" time */
-                now -= trace->event.tdelta; 
+                sincebeginnow = (now - trace->event.first_now);
+                sincebegintrace = (ts - trace->event.first_ts);
 
                 /* If the trace timestamp is still in the future, return a 
                  * SLEEP event, otherwise return the packet */
-                if (ts > now) {
-                        event.seconds = ts - 
-                                trace->event.trace_last_ts;
-                        trace->event.trace_last_ts = ts;
+                if (sincebeginnow <= sincebegintrace / (double)trace->replayspeedup) {
+                        event.seconds = ((sincebegintrace / (double)trace->replayspeedup) - sincebeginnow);
                         event.type = TRACE_EVENT_SLEEP;
                         trace->event.waiting = true;
@@ -204 +206 @@
                  * trace file.
                  */
-                trace->event.tdelta = now - ts;
+                trace->event.first_now = (double)now;
+                trace->event.first_ts = (double)ts;
         }
 
@@ -219 +222 @@
         event.type = TRACE_EVENT_PACKET;
 
-        trace->event.trace_last_ts = ts;
         trace->event.waiting = false;
 

lib/format_legacy.c

@@ -523 +523 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         legacyatm_get_link_type,        /* get_link_type */
         NULL,                           /* get_direction */
@@ -567 +568 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         legacyeth_get_link_type,        /* get_link_type */
         NULL,                           /* get_direction */
@@ -611 +613 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         legacypos_get_link_type,        /* get_link_type */
         NULL,                           /* get_direction */
@@ -655 +658 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         legacynzix_get_link_type,       /* get_link_type */
         NULL,                           /* get_direction */

lib/format_linux_common.c

@@ -49 +49 @@
 
 #include "format_linux_common.h"
+
+unsigned int rand_seedp = 0;
 
 #ifdef HAVE_NETPACKET_PACKET_H
@@ -85 +87 @@
                 sock = socket(PF_INET, SOCK_STREAM, 0);
                 memset(&ifr, 0, sizeof(struct ifreq));
-                strncpy(ifr.ifr_name, libtrace->uridata, IF_NAMESIZE);
+                strncpy(ifr.ifr_name, libtrace->uridata, IF_NAMESIZE - 1);
                 if (ioctl(sock, SIOCGIFHWADDR, &ifr) != 0) {
                         perror("Can't get HWADDR for interface");
@@ -164 +166 @@
                         /* Live captures are always going to be in trace time */
                         break;
+                case TRACE_OPTION_REPLAY_SPEEDUP:
+                        break;
                 /* Avoid default: so that future options will cause a warning
                  * here to remind us to implement it, or flag it as
@@ -199 +203 @@
         /* Some examples use pid for the group however that would limit a single
          * application to use only int/ring format, instead using rand */
-        FORMAT_DATA->fanout_group = (uint16_t) rand();
+        FORMAT_DATA->fanout_group = (uint16_t) (rand_r(&rand_seedp) % 65536);
         return 0;
 }

lib/format_linux_common.h

@@ -421 +421 @@
                                         struct linux_per_stream_t *stream)
 {
-        int fanout_opt = ((int)FORMAT_DATA->fanout_flags << 16) |
-                         (int)FORMAT_DATA->fanout_group;
-        if (setsockopt(stream->fd, SOL_PACKET, PACKET_FANOUT,
-                        &fanout_opt, sizeof(fanout_opt)) == -1) {
-                trace_set_err(libtrace, TRACE_ERR_INIT_FAILED,
+        int fanout_opt;
+        int attempts = 0;
+        while (attempts < 5) {
+                fanout_opt = ((int)FORMAT_DATA->fanout_flags << 16) |
+                                 (int)FORMAT_DATA->fanout_group;
+
+                if (setsockopt(stream->fd, SOL_PACKET, PACKET_FANOUT,
+                                &fanout_opt, sizeof(fanout_opt)) == -1) {
+                        trace_set_err(libtrace, TRACE_ERR_INIT_FAILED,
                               "Converting the fd to a socket fanout failed %s",
                               libtrace->uridata);
-                return -1;
+                        FORMAT_DATA->fanout_group ++;
+                        attempts ++;
+                        continue;
+                }
+                return 0;
         }
-        return 0;
+        return -1;
 }
 #endif /* HAVE_NETPACKET_PACKET_H */

lib/format_linux_int.c

@@ -496 +496 @@
         NULL,                           /* fin_packet */
         linuxnative_write_packet,       /* write_packet */
+        NULL,                           /* flush_output */
         linuxnative_get_link_type,      /* get_link_type */
         linuxnative_get_direction,      /* get_direction */
@@ -556 +557 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         linuxnative_get_link_type,      /* get_link_type */
         linuxnative_get_direction,      /* get_direction */

lib/format_linux_ring.c

@@ -61 +61 @@
         ((mac) > (hdrend) && (mac) < (net) ? (mac) : (net))
 
+static pthread_mutex_t pagesize_mutex;
 #ifdef HAVE_NETPACKET_PACKET_H
 /* Get current frame in the ring buffer*/
@@ -70 +71 @@
 /* Cached page size, the page size shouldn't be changing */
 static int pagesize = 0;
+
 
 /*
@@ -88 +90 @@
         struct ifreq ifr;
         unsigned max_frame = LIBTRACE_PACKET_BUFSIZE;
-        pagesize = getpagesize();
+        pthread_mutex_lock(&pagesize_mutex);
+        if (pagesize == 0) {
+                pagesize = getpagesize();
+        }
+        pthread_mutex_unlock(&pagesize_mutex);
 
         strcpy(ifr.ifr_name, uri);
@@ -461 +467 @@
         struct pollfd pollset[2];
 
-        ring_release_frame(libtrace, packet);
-        
         packet->buf_control = TRACE_CTRL_EXTERNAL;
         packet->type = TRACE_RT_DATA_LINUX_RING;
@@ -547 +551 @@
 
         stream->last_timestamp = packet->order;
-                
 
         /* We just need to get prepare_packet to set all our packet pointers
@@ -745 +748 @@
         linuxring_fin_packet,           /* fin_packet */
         linuxring_write_packet,         /* write_packet */
+        NULL,                           /* flush_output */
         linuxring_get_link_type,        /* get_link_type */
         linuxring_get_direction,        /* get_direction */
@@ -807 +811 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         linuxring_get_link_type,        /* get_link_type */
         linuxring_get_direction,        /* get_direction */
@@ -837 +842 @@
 void linuxring_constructor(void)
 {
+        pthread_mutex_init(&pagesize_mutex, NULL);
         register_format(&linuxring);
 }

lib/format_ndag.c

@@ -114 +114 @@
         uint64_t received_packets;
 
-        fd_set allsocks;
         int maxfd;
 } recvstream_t;
@@ -488 +487 @@
                 FORMAT_DATA->receivers[i].received_packets = 0;
                 FORMAT_DATA->receivers[i].missing_records = 0;
-                FD_ZERO(&(FORMAT_DATA->receivers[i].allsocks));
                 FORMAT_DATA->receivers[i].maxfd = -1;
 
@@ -771 +769 @@
         ssock->recordcount = 0;
         rt->sourcecount += 1;
-        FD_SET(ssock->sock, &(rt->allsocks));
         if (ssock->sock > rt->maxfd) {
                 rt->maxfd = ssock->sock;
@@ -881 +878 @@
                 fprintf(stderr, "Received invalid record on the channel for %s:%u.\n",
                                 ssock->groupaddr, ssock->port);
-                FD_CLR(ssock->sock, &(rt->allsocks));
                 close(ssock->sock);
                 ssock->sock = -1;
@@ -943 +939 @@
 #if HAVE_RECVMMSG
         int i;
-#endif
-
-        if (ssock->sock == -1) {
-                return 0;
-        }
-
-#if HAVE_RECVMMSG
-        /* Plenty of full buffers, just use the packets in those */
-        if (ssock->bufavail < RECV_BATCH_SIZE / 2) {
-                return 1;
-        }
-#else
-        if (ssock->bufavail == 0) {
-                return 1;
-        }
 #endif
 
@@ -987 +968 @@
                                         ssock->port);
 
-                                FD_CLR(ssock->sock, &(rt->allsocks));
                                 close(ssock->sock);
                                 ssock->sock = -1;
@@ -997 +977 @@
                                 ssock->groupaddr, ssock->port,
                                 strerror(errno));
-                        FD_CLR(ssock->sock, &(rt->allsocks));
                         close(ssock->sock);
                         ssock->sock = -1;
@@ -1035 +1014 @@
         struct timeval tv;
         fd_set fds;
+        int maxfd = 0;
         struct timeval zerotv;
 
@@ -1040 +1020 @@
         gottime = 0;
 
-        fds = rt->allsocks;
+        FD_ZERO(&fds);
 
         if (rt->maxfd == -1) {
@@ -1049 +1029 @@
         zerotv.tv_usec = 0;
 
-        if (select(rt->maxfd + 1, &fds, NULL, NULL, &zerotv) == -1) {
+        for (i = 0; i < rt->sourcecount; i++) {
+                if (rt->sources[i].sock == -1) {
+                        continue;
+                }
+
+#if HAVE_RECVMMSG
+                /* Plenty of full buffers, just use the packets in those */
+                if (rt->sources[i].bufavail < RECV_BATCH_SIZE / 2) {
+                        readybufs ++;
+                        continue;
+                }
+#else
+                if (rt->sources[i].bufavail == 0) {
+                        readybufs ++;
+                        continue;
+                }
+#endif
+                FD_SET(rt->sources[i].sock, &fds);
+                if (maxfd < rt->sources[i].sock) {
+                        maxfd = rt->sources[i].sock;
+                }
+        }
+
+
+        if (maxfd <= 0) {
+                return readybufs;
+        }
+
+        if (select(maxfd + 1, &fds, NULL, NULL, &zerotv) == -1) {
                 /* log the error? XXX */
                 return -1;
@@ -1056 +1064 @@
         for (i = 0; i < rt->sourcecount; i++) {
                 if (!FD_ISSET(rt->sources[i].sock, &fds)) {
+                        if (rt->sources[i].bufavail < ENCAP_BUFFERS) {
+                                readybufs ++;
+                        }
                         continue;
                 }
@@ -1423 +1434 @@
         NULL,                   /* fin_packet */
         NULL,                   /* write_packet */
+        NULL,                   /* flush_output */
         erf_get_link_type,      /* get_link_type */
         erf_get_direction,      /* get_direction */

lib/format_pcap.c

@@ -583 +583 @@
 }
 
+static int pcap_flush_output(libtrace_out_t *libtrace) {
+        return pcap_dump_flush(OUTPUT.trace.dump);
+}
+
 static int pcapint_write_packet(libtrace_out_t *libtrace,
                 libtrace_packet_t *packet) 
@@ -779 +783 @@
         NULL,                           /* fin_packet */
         pcap_write_packet,              /* write_packet */
+        pcap_flush_output,              /* flush_output */
         pcap_get_link_type,             /* get_link_type */
         pcapint_get_direction,          /* get_direction */
@@ -823 +828 @@
         NULL,                           /* fin_packet */
         pcapint_write_packet,           /* write_packet */
+        NULL,                           /* flush_output */
         pcap_get_link_type,             /* get_link_type */
         pcapint_get_direction,          /* get_direction */

lib/format_pcapfile.c

@@ -257 +257 @@
                 case TRACE_OPTION_FILTER:
                 case TRACE_OPTION_HASHER:
+                case TRACE_OPTION_REPLAY_SPEEDUP:
                         /* All these are either unsupported or handled
                          * by trace_config */
@@ -531 +532 @@
 
         return numbytes+ret;
+}
+
+static int pcapfile_flush_output(libtrace_out_t *out) {
+
+        return wandio_wflush(DATAOUT(out)->file);
 }
 
@@ -695 +701 @@
         NULL,                           /* fin_packet */
         pcapfile_write_packet,          /* write_packet */
+        pcapfile_flush_output,          /* flush_output */
         pcapfile_get_link_type,         /* get_link_type */
         pcapfile_get_direction,         /* get_direction */

lib/format_pcapng.c

@@ -173 +173 @@
 };
 
+typedef struct pcapng_peeker pcapng_hdr_t;
+
 
 #define DATA(x) ((struct pcapng_format_data_t *)((x)->format_data))
@@ -260 +262 @@
                 case TRACE_OPTION_FILTER:
                 case TRACE_OPTION_HASHER:
+                case TRACE_OPTION_REPLAY_SPEEDUP:
                         break;
         }
@@ -286 +289 @@
 
 static char *pcapng_parse_next_option(libtrace_t *libtrace, char **pktbuf,
-                uint16_t *code, uint16_t *length) {
+                uint16_t *code, uint16_t *length, pcapng_hdr_t *blockhdr) {
 
         struct pcapng_optheader *opthdr = (struct pcapng_optheader *)*pktbuf;
         int to_skip;
         int padding = 0;
+        char *eob; //end of block
         char *optval;
+        if (DATA(libtrace)->byteswapped) {
+                eob = ((char *) blockhdr) + byteswap32(blockhdr->blocklen);
+        } else {
+                eob = ((char *) blockhdr) + blockhdr->blocklen;
+        }
+
+        assert((char *)blockhdr < *pktbuf);
+        // Check if we have reached the end of the block, +4 for trailing block-size
+        // We cannot assume a endofopt, so we add one
+        if (eob == (*pktbuf) + 4) {
+                *code = 0;
+                *length = 0;
+                return *pktbuf;
+        }
+        // If there is not enough space for another header we've encountered an error
+        if (eob < (*pktbuf) + 4 + sizeof(struct pcapng_optheader)) {
+                return NULL;
+        }
 
         if (DATA(libtrace)->byteswapped) {
@@ -310 +332 @@
 
         to_skip = (*length) + padding;
+        // Check the value we return is within the block length
+        if (eob < optval + to_skip + 4) {
+                return NULL;
+        }
         *pktbuf = optval + to_skip;
 
         return optval;
-}
-
-static inline int skip_block(libtrace_t *libtrace, uint32_t toread) {
-        int err;
-
-        while (toread > 0) {
-                char buf[4096];
-                int nextread;
-
-                if (toread < 4096) {
-                        nextread = toread;
-                } else {
-                        nextread = 4096;
-                }
-
-                err = wandio_read(libtrace->io, buf, nextread);
-                if (err < 0) {
-                        trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED,
-                                "Reading section header options");
-                        return -1;
-                }
-                if (err == 0) {
-                        return 0;
-                }
-                toread -= err;
-        }
-
-        return 1;
-
 }
 
@@ -352 +349 @@
         if (err < 0) {
                 trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED,
-                        "Failed to read pcapng interface options");
+                        "Failed reading pcapng block");
                 return err;
         }
@@ -362 +359 @@
         if (err < (int)to_read) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
-                        "Incomplete pcapng interface header block");
+                        "Incomplete pcapng block");
                 return -1;
         }
@@ -453 +450 @@
         }
 
-        if (err < (int)(sizeof(sechdr))) {
+        if (err < (int)(sizeof(pcapng_sec_t))) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
                         "Incomplete pcapng section header block");
@@ -490 +487 @@
         /* Read all of the options etc. -- we don't need them for now, but
          * we have to skip forward to the next useful header. */
-        bodyptr = packet->buffer + sizeof(pcapng_sec_t);
+        bodyptr = (char *) packet->buffer + sizeof(pcapng_sec_t);
         err = pcapng_read_body(libtrace, bodyptr, to_read);
         if (err <= 0) {
@@ -506 +503 @@
 
 static int pcapng_read_interface(libtrace_t *libtrace,
-                libtrace_packet_t *packet, uint32_t flags) {
+                libtrace_packet_t *packet, uint32_t blocklen, uint32_t flags) {
 
         pcapng_int_t *inthdr;
-        int err;
-        uint32_t to_read;
         pcapng_interface_t *newint;
         uint16_t optcode, optlen;
@@ -516 +511 @@
         char *bodyptr = NULL;
 
-        err = wandio_read(libtrace->io, packet->buffer, sizeof(pcapng_int_t));
-
-        if (err < 0) {
-                trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED,
-                        "Reading pcapng interface header block");
-                return -1;
-        }
-
-        if (err == 0) {
-                return 0;
-        }
-
-        if (err < (int)sizeof(inthdr)) {
+        if (blocklen < sizeof(pcapng_int_t) + 4) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
                         "Incomplete pcapng interface header block");
@@ -551 +534 @@
                 newint->snaplen = byteswap32(inthdr->snaplen);
                 newint->linktype = byteswap16(inthdr->linktype);
-                to_read = byteswap32(inthdr->blocklen) - sizeof(pcapng_int_t);
         } else {
                 assert(inthdr->blocktype == PCAPNG_INTERFACE_TYPE);
                 newint->snaplen = inthdr->snaplen;
                 newint->linktype = inthdr->linktype;
-                to_read = inthdr->blocklen - sizeof(pcapng_int_t);
         }
 
@@ -565 +546 @@
                         DATA(libtrace)->allocatedinterfaces * sizeof(
                                 pcapng_interface_t *));
-
-                /* Could memset the new memory to zero, if required */
+                memset(&DATA(libtrace)->interfaces[DATA(libtrace)->nextintid], 0, sizeof(void *) * 10);
         }
 
@@ -572 +552 @@
         DATA(libtrace)->nextintid += 1;
 
-        bodyptr = packet->buffer + sizeof(pcapng_int_t);
-        err = pcapng_read_body(libtrace, bodyptr, to_read);
-        if (err <= 0) {
-                return err;
-        }
+        bodyptr = (char *) packet->buffer + sizeof(pcapng_int_t);
 
         packet->type = TRACE_RT_PCAPNG_META;
@@ -587 +563 @@
         do {
                 optval = pcapng_parse_next_option(libtrace, &bodyptr,
-                                &optcode, &optlen);
+                                &optcode, &optlen, (pcapng_hdr_t *) packet->buffer);
                 if (optval == NULL) {
                         trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
@@ -607 +583 @@
         } while (optcode != 0);
 
-        return 1;
+        return (int) blocklen;
 
 }
 
 static int pcapng_read_nrb(libtrace_t *libtrace, libtrace_packet_t *packet,
-                uint32_t flags) {
+                uint32_t blocklen, uint32_t flags) {
 
         /* Just read the NR records and pass them off to the caller. If
@@ -619 +595 @@
          */
         pcapng_nrb_t *hdr = NULL;
-        int err;
-        uint32_t to_read;
-        char *bodyptr;
-
-        err = wandio_read(libtrace->io, packet->buffer, sizeof(pcapng_nrb_t));
-
-        if (err < 0) {
-                trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED, "reading pcapng name resolution block");
-                return -1;
-        }
-
-        if (err == 0) {
-                return 0;
-        }
-
-        if (err < (int)sizeof(pcapng_nrb_t)) {
+
+        if (blocklen < sizeof(pcapng_nrb_t) + 4) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
                                 "Incomplete pcapng name resolution block");
@@ -645 +607 @@
         if (DATA(libtrace)->byteswapped) {
                 assert(byteswap32(hdr->blocktype) == PCAPNG_NAME_RESOLUTION_TYPE);
-                to_read = byteswap32(hdr->blocklen) - sizeof(pcapng_nrb_t);
         } else {
                 assert(hdr->blocktype == PCAPNG_NAME_RESOLUTION_TYPE);
-                to_read = hdr->blocklen - sizeof(pcapng_nrb_t);
-        }
-
-        bodyptr = packet->buffer + sizeof(pcapng_nrb_t);
-        err = pcapng_read_body(libtrace, bodyptr, to_read);
-        if (err <= 0) {
-                return err;
         }
 
@@ -663 +617 @@
         }
 
-        return sizeof(pcapng_nrb_t) + to_read;
+        return (int) blocklen;
 
 }
 
 static int pcapng_read_custom(libtrace_t *libtrace, libtrace_packet_t *packet,
-                uint32_t flags) {
+                uint32_t blocklen, uint32_t flags) {
 
         /* Just read the custom records and pass them off to the caller. If
@@ -675 +629 @@
          */
         pcapng_custom_t *hdr = NULL;
-        int err;
-        uint32_t to_read;
-        char *bodyptr;
-
-        err = wandio_read(libtrace->io, packet->buffer, sizeof(pcapng_custom_t));
-
-        if (err < 0) {
-                trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED, "reading pcapng custom block");
-                return -1;
-        }
-
-        if (err == 0) {
-                return 0;
-        }
-
-        if (err < (int)sizeof(pcapng_custom_t)) {
+
+        if (blocklen < sizeof(pcapng_custom_t) + 4) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
                                 "Incomplete pcapng custom block");
@@ -702 +642 @@
                 assert(byteswap32(hdr->blocktype) == PCAPNG_CUSTOM_TYPE ||
                         byteswap32(hdr->blocktype) == PCAPNG_CUSTOM_NONCOPY_TYPE);
-                to_read = byteswap32(hdr->blocklen) - sizeof(pcapng_custom_t);
-        } else {
-                assert(hdr->blocktype == PCAPNG_NAME_RESOLUTION_TYPE ||
+        } else {
+                assert(hdr->blocktype == PCAPNG_CUSTOM_TYPE ||
                         hdr->blocktype == PCAPNG_CUSTOM_NONCOPY_TYPE);
-                to_read = hdr->blocklen - sizeof(pcapng_custom_t);
-        }
-
-        bodyptr = packet->buffer + sizeof(pcapng_custom_t);
-        err = pcapng_read_body(libtrace, bodyptr, to_read);
-        if (err <= 0) {
-                return err;
         }
 
@@ -721 +653 @@
         }
 
-        return sizeof(pcapng_custom_t) + to_read;
+        return (int) blocklen;
 
 }
 
 static int pcapng_read_stats(libtrace_t *libtrace, libtrace_packet_t *packet,
-                uint32_t flags) {
+                uint32_t blocklen, uint32_t flags) {
         pcapng_stats_t *hdr = NULL;
-        int err;
-        uint32_t to_read;
         uint32_t ifaceid;
         uint64_t timestamp;
@@ -737 +667 @@
         char *bodyptr;
 
-        err = wandio_read(libtrace->io, packet->buffer, sizeof(pcapng_stats_t));
-
-        if (err < 0) {
-                trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED, "reading pcapng interface stats");
-                return -1;
-        }
-
-        if (err == 0) {
-                return 0;
-        }
-
-        if (err < (int)sizeof(pcapng_stats_t)) {
+        if (blocklen < sizeof(pcapng_stats_t) + 4) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
                                 "Incomplete pcapng interface stats header");
@@ -759 +678 @@
         if (DATA(libtrace)->byteswapped) {
                 assert(byteswap32(hdr->blocktype) == PCAPNG_INTERFACE_STATS_TYPE);
-                to_read = byteswap32(hdr->blocklen) - sizeof(pcapng_stats_t);
                 ifaceid = byteswap32(hdr->interfaceid);
                 timestamp = ((uint64_t)(byteswap32(hdr->timestamp_high)) << 32) + byteswap32(hdr->timestamp_low);
         } else {
                 assert(hdr->blocktype == PCAPNG_INTERFACE_STATS_TYPE);
-                to_read = hdr->blocklen - sizeof(pcapng_stats_t);
                 ifaceid = hdr->interfaceid;
                 timestamp = ((uint64_t)(hdr->timestamp_high) << 32) +
                                 hdr->timestamp_low;
-        }
-
-        bodyptr = packet->buffer + sizeof(pcapng_stats_t);
-        err = pcapng_read_body(libtrace, bodyptr, to_read);
-        if (err <= 0) {
-                return err;
         }
 
@@ -790 +701 @@
 
         if (timestamp < interface->laststats) {
-                return sizeof(pcapng_stats_t) + to_read;
+                return (int) blocklen;
         }
 
@@ -798 +709 @@
         do {
                 optval = pcapng_parse_next_option(packet->trace, &bodyptr,
-                                &optcode, &optlen);
+                                &optcode, &optlen, (pcapng_hdr_t *) packet->buffer);
                 if (optval == NULL) {
                         trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
-                                "Failed to read options for pcapng enhanced packet");
+                                "Failed to read options for pcapng interface stats");
                         return -1;
                 }
@@ -844 +755 @@
         interface->laststats = timestamp;
 
-        return sizeof(pcapng_stats_t) + to_read;
+        return (int) blocklen;
 
 }
 
 static int pcapng_read_simple(libtrace_t *libtrace, libtrace_packet_t *packet,
-                uint32_t flags) {
-
-        int err;
-        uint32_t to_read;
+                uint32_t blocklen, uint32_t flags) {
+
         uint32_t caplen;
         pcapng_spkt_t *hdr = NULL;
         pcapng_interface_t *interface;
-        char *bodyptr;
-
-        err = wandio_read(libtrace->io, packet->buffer, sizeof(pcapng_spkt_t));
-
-        if (err < 0) {
-                trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED, "reading pcapng simple packet");
-                return -1;
-        }
-
-        if (err == 0) {
-                return 0;
-        }
-
-        if (err < (int)sizeof(pcapng_spkt_t)) {
+
+        if (blocklen < sizeof(pcapng_spkt_t) + 4) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
                                 "Incomplete pcapng simple packet header");
@@ -880 +777 @@
         if (DATA(libtrace)->byteswapped) {
                 assert(byteswap32(hdr->blocktype) == PCAPNG_SIMPLE_PACKET_TYPE);
-                to_read = byteswap32(hdr->blocklen) - sizeof(pcapng_spkt_t);
-                caplen = to_read - 4;   /* account for trailing length field */
+                caplen = byteswap32(hdr->blocklen) - sizeof(pcapng_spkt_t) - 4;
+                         /* account for trailing length field */
         } else {
                 assert(hdr->blocktype == PCAPNG_SIMPLE_PACKET_TYPE);
-                to_read = hdr->blocklen - sizeof(pcapng_spkt_t);
-                caplen = to_read - 4; /* account for trailing length field */
-        }
-
-        bodyptr = packet->buffer + sizeof(pcapng_spkt_t);
-        err = pcapng_read_body(libtrace, bodyptr, to_read);
-        if (err <= 0) {
-                return err;
+                caplen = hdr->blocklen - sizeof(pcapng_spkt_t) - 4;
+                         /* account for trailing length field */
         }
 
@@ -911 +802 @@
                 return -1;
         }
-        return sizeof(pcapng_spkt_t) + to_read;
+        return (int) blocklen;
 
 }
 
 static int pcapng_read_enhanced(libtrace_t *libtrace, libtrace_packet_t *packet,
-                uint32_t flags) {
+                uint32_t blocklen, uint32_t flags) {
         pcapng_epkt_t *hdr = NULL;
-        int err;
-        uint32_t to_read;
         uint32_t caplen;
         uint32_t ifaceid;
@@ -927 +816 @@
         char *bodyptr;
 
-        err = wandio_read(libtrace->io, packet->buffer, sizeof(pcapng_epkt_t));
-
-        if (err < 0) {
-                trace_set_err(libtrace, TRACE_ERR_WANDIO_FAILED, "reading pcapng enhanced packet");
-                return -1;
-        }
-
-        if (err == 0) {
-                return 0;
-        }
-
-        if (err < (int)sizeof(pcapng_epkt_t)) {
+        if (blocklen < (int)sizeof(pcapng_epkt_t) + 4) {
                 trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
                                 "Incomplete pcapng enhanced packet header");
@@ -950 +828 @@
                 assert(byteswap32(hdr->blocktype) == PCAPNG_ENHANCED_PACKET_TYPE);
                 caplen = byteswap32(hdr->caplen);
-                to_read = byteswap32(hdr->blocklen) - sizeof(pcapng_epkt_t);
                 ifaceid = byteswap32(hdr->interfaceid);
         } else {
                 assert(hdr->blocktype == PCAPNG_ENHANCED_PACKET_TYPE);
                 caplen = hdr->caplen;
-                to_read = hdr->blocklen - sizeof(pcapng_epkt_t);
                 ifaceid = hdr->interfaceid;
         }
 
-        bodyptr = packet->buffer + sizeof(pcapng_epkt_t);
-        err = pcapng_read_body(libtrace, bodyptr, to_read);
-        if (err <= 0) {
-                return err;
-        }
+        bodyptr = (char *) packet->buffer + sizeof(pcapng_epkt_t);
 
         /* Set packet type based on interface linktype */
@@ -984 +856 @@
         /* Make sure to parse any useful options */
         if ((caplen % 4) == 0) {
-                bodyptr = packet->payload + caplen;
-        } else {
-                bodyptr = packet->payload + caplen + (4 - (caplen % 4));
+                bodyptr = (char *) packet->payload + caplen;
+        } else {
+                bodyptr = (char *) packet->payload + caplen + (4 - (caplen % 4));
+        }
+        // Check the packet caplen actually fits within the block we read
+        if ((char *) packet->buffer + blocklen < bodyptr + 4) {
+                trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
+                                "Incomplete pcapng enhanced packet header");
+                return -1;
         }
 
         do {
                 optval = pcapng_parse_next_option(packet->trace, &bodyptr,
-                                &optcode, &optlen);
+                                &optcode, &optlen, (pcapng_hdr_t *) packet->buffer);
                 if (optval == NULL) {
                         trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
@@ -1008 +886 @@
 
         } while (optcode != 0);
-        return sizeof(pcapng_epkt_t) + to_read;
+        return (int) blocklen;
 
 }
@@ -1052 +930 @@
                 }
 
+                // Warning: the byteorder might not be set yet, the section header sets this
                 if (DATA(libtrace)->byteswapped) {
                         btype = byteswap32(peeker.blocktype);
+                        to_read = byteswap32(peeker.blocklen);
                 } else {
                         btype = peeker.blocktype;
+                        to_read = peeker.blocklen;
+                }
+
+                // Check we won't read off the end of the packet buffer. Assuming corruption.
+                // Exclude the SECTION header, as this is used to identify the byteorder
+                if (to_read > LIBTRACE_PACKET_BUFSIZE && btype != PCAPNG_SECTION_TYPE) {
+                        trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
+                                      "Oversized pcapng block found, is the trace corrupted?");
+                        return -1;
+                }
+                if (btype != PCAPNG_SECTION_TYPE) {
+                        // Read the entire block, unless it is a section as our byte ordering has
+                        // not been set yet.
+                        err = pcapng_read_body(libtrace, packet->buffer, to_read);
+                        if (err <= 0) {
+                                return err;
+                        }
+                        if (*((uint32_t *)((char *)packet->buffer+to_read-4)) != peeker.blocklen) {
+                                trace_set_err(libtrace, TRACE_ERR_BAD_PACKET,
+                                              "Mismatched pcapng block sizes found, trace is invalid.");
+                                return -1;
+                        }
                 }
 
@@ -1067 +969 @@
                         /* Interface Header */
                         case PCAPNG_INTERFACE_TYPE:
-                                err = pcapng_read_interface(libtrace, packet, flags);
+                                err = pcapng_read_interface(libtrace, packet, to_read, flags);
                                 gotpacket = 1;
                                 break;
@@ -1074 +976 @@
                         case PCAPNG_ENHANCED_PACKET_TYPE:
                                 err = pcapng_read_enhanced(libtrace, packet,
-                                                flags);
+                                                to_read, flags);
                                 gotpacket = 1;
                                 break;
 
                         case PCAPNG_SIMPLE_PACKET_TYPE:
-                                err = pcapng_read_simple(libtrace, packet, flags);
+                                err = pcapng_read_simple(libtrace, packet, to_read, flags);
                                 gotpacket = 1;
                                 break;
 
                         case PCAPNG_INTERFACE_STATS_TYPE:
-                                err = pcapng_read_stats(libtrace, packet, flags);
+                                err = pcapng_read_stats(libtrace, packet, to_read, flags);
                                 gotpacket = 1;
                                 break;
 
                         case PCAPNG_NAME_RESOLUTION_TYPE:
-                                err = pcapng_read_nrb(libtrace, packet, flags);
+                                err = pcapng_read_nrb(libtrace, packet, to_read, flags);
                                 gotpacket = 1;
                                 break;
@@ -1095 +997 @@
                         case PCAPNG_CUSTOM_TYPE:
                         case PCAPNG_CUSTOM_NONCOPY_TYPE:
-                                err = pcapng_read_custom(libtrace, packet, flags);
+                                err = pcapng_read_custom(libtrace, packet, to_read, flags);
                                 gotpacket = 1;
                                 break;
@@ -1105 +1007 @@
                         /* Everything else -- don't care, skip it */
                         default:
-                                if (DATA(libtrace)->byteswapped) {
-                                        to_read = byteswap32(peeker.blocklen);
-                                } else {
-                                        to_read = peeker.blocklen;
-                                }
-                                err = skip_block(libtrace, to_read);
                                 break;
                 }
@@ -1194 +1090 @@
 
         ts.tv_sec = (timestamp / interface->tsresol);
-        ts.tv_nsec = (uint64_t)(timestamp - (ts.tv_sec * interface->tsresol)) / interface->tsresol * 1000000000;
+        ts.tv_nsec = (uint64_t)(timestamp - (ts.tv_sec * interface->tsresol))
+                        / ((double)interface->tsresol) * 1000000000;
 
         return ts;
@@ -1468 +1365 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         pcapng_get_link_type,           /* get_link_type */
         pcapng_get_direction,           /* get_direction */

lib/format_rt.c

@@ -845 +845 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         rt_get_link_type,               /* get_link_type */
         NULL,                           /* get_direction */

lib/format_tsh.c

@@ -245 +245 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         tsh_get_link_type,              /* get_link_type */
         tsh_get_direction,              /* get_direction */
@@ -294 +295 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
+        NULL,                           /* flush_output */
         tsh_get_link_type,              /* get_link_type */
         tsh_get_direction,              /* get_direction */

lib/libtrace.h.in

@@ -277 +277 @@
 typedef struct trace_err_t{
         int err_num;            /**< error code */
-        char problem[255];      /**< the format, uri etc that caused the error for reporting purposes */
+        char problem[1024];     /**< the format, uri etc that caused the error for reporting purposes */
 } libtrace_err_t;
 
@@ -370 +370 @@
        TRACE_TYPE_NONDATA = 19,         /**< Not a data packet */
        TRACE_TYPE_OPENBSD_LOOP = 20,    /**< OpenBSD loopback */
-       TRACE_TYPE_ERF_META = 21 /**< ERF Provenance metadata record */
+       TRACE_TYPE_ERF_META = 21, /**< ERF Provenance metadata record */
+       TRACE_TYPE_ETSILI = 22   /**< ETSI Lawful Intercept */
 } libtrace_linktype_t;
 
@@ -398 +399 @@
         TRACE_FORMAT_NDAG       =19,    /**< DAG multicast over a network */
         TRACE_FORMAT_DPDK_NDAG       =20,    /**< DAG multicast over a network, received via DPDK */
+        TRACE_FORMAT_ETSILIVE     =21,  /**< ETSI LI over a network */
 };
 
@@ -453 +455 @@
         TRACE_RT_DATA_DPDK=TRACE_RT_DATA_SIMPLE+TRACE_FORMAT_DPDK,
 
+        TRACE_RT_DATA_ETSILI = TRACE_RT_DATA_SIMPLE + TRACE_FORMAT_ETSILIVE,
+
         /** As PCAP does not store the linktype with the packet, we need to 
          * create a separate RT type for each supported DLT, starting from
@@ -560 +564 @@
         uint64_t internalid;            /** Internal identifier for the pkt */
         void *srcbucket;
+
+        pthread_mutex_t ref_lock;       /**< Lock for reference counter */
+        int refcount;                 /**< Reference counter */
+
 } libtrace_packet_t;
 
@@ -1312 +1320 @@
         /** The hasher function for a parallel libtrace. It is recommended to
          * access this option via trace_set_hasher(). */
-        TRACE_OPTION_HASHER
+        TRACE_OPTION_HASHER,
+
+        /** Speed up trace file replays (via trace_event()) by this factor */
+        TRACE_OPTION_REPLAY_SPEEDUP,
 } trace_option_t;
 
@@ -1526 +1537 @@
 DLLEXPORT void trace_destroy_output(libtrace_out_t *trace);
 
+/** Flush an output trace, forcing any buffered packets to be written
+ * @param libtrace      The output trace to be flushed
+ */
+DLLEXPORT int trace_flush_output(libtrace_out_t *libtrace);
+
 /** Check (and clear) the current error state of an input trace
  * @param trace         The input trace to check the error state on
@@ -1774 +1790 @@
  */
 DLLEXPORT void trace_destroy_packet(libtrace_packet_t *packet);
-
 
 /** Read the next packet from an input trace 

lib/libtrace_int.h

@@ -99 +99 @@
 #endif
 
-#ifndef HAVE_STRNDUP
+#if !HAVE_DECL_STRNDUP
 char *strndup(const char *s, size_t size);
 #endif
 
-#ifndef HAVE_STRNCASECMP
+#if !HAVE_DECL_STRNCASECMP
 # ifndef HAVE__STRNICMP
 /** A local implementation of strncasecmp (as some systems do not have it) */
@@ -112 +112 @@
 #endif
 
-#ifndef HAVE_SNPRINTF
+#if !HAVE_DECL_SNPRINTF
 # ifndef HAVE_SPRINTF_S
 /** A local implementation of snprintf (as some systems do not have it) */
@@ -152 +152 @@
 //#define RP_BUFSIZE 65536U
 
+#define LIBTRACE_MAX_REPLAY_SPEEDUP 1000
+
 /** Data about the most recent event from a trace file */
 struct libtrace_event_status_t {
         /** A libtrace packet to store the packet when a PACKET event occurs */
         libtrace_packet_t *packet;
-        /** Time between the timestamp for the current packet and the current 
-         * walltime */
-        double tdelta;
-        /** The timestamp of the previous PACKET event */
-        double trace_last_ts;
+
+        /* The walltime when we processed the first packet from the trace */
+        double first_now;
+
+        /* The tracetime of the first packet in the trace */
+        double first_ts;
+
         /** The size of the current PACKET event */
         int psize;
@@ -313 +317 @@
          * used only if the capture format does not support snapping natively */
         size_t snaplen;                 
+        /** Speed up the packet rate when using trace_event() to process trace
+         * files by this factor. */
+        int replayspeedup;
         /** Count of the number of packets returned to the libtrace user */
         uint64_t accepted_packets;
@@ -617 +624 @@
          */
         int (*write_packet)(libtrace_out_t *libtrace, libtrace_packet_t *packet);
+
+        /** Flush any buffered output for an output trace.
+         *
+         * @param libtrace      The output trace to be flushed
+         */
+        int (*flush_output)(libtrace_out_t *libtrace);
+
         /** Returns the libtrace link type for a packet.
          *
@@ -1242 +1256 @@
 /** Constructor for the network DAG format module */
 void ndag_constructor(void);
+/** Constructor for the live ETSI over TCP format module */
+void etsilive_constructor(void);
 #ifdef HAVE_BPF
 /** Constructor for the BPF format module */

lib/libtrace_parallel.h

@@ -731 +731 @@
 /** Set the maximum number of perpkt threads to use in a trace.
  *
+ * Only valid on a new trace, that has not be started. Once started
+ * the number of threads cannot be changed without destroying the trace.
+ *
  * @param[in] trace The parallel input trace
  * @param[in] nb The number of threads to use. If set to 0, libtrace will
@@ -1271 +1274 @@
  */
 DLLEXPORT void trace_free_packet(libtrace_t * libtrace, libtrace_packet_t * packet);
+
+/** Increments the internal reference counter for a packet.
+ * @param packet        The packet opaque pointer
+ *
+ * You may wish to use this function (and its decrementing counterpart)
+ * in situations where you are retaining multiple references to a packet
+ * outside of the core packet processing function. This will ensure that
+ * the packet is not released until there are no more outstanding references
+ * to the packet anywhere in your program.
+ */
+DLLEXPORT void trace_increment_packet_refcount(libtrace_packet_t *packet);
+
+/** Decrements the internal reference counter for a packet.
+ * @param packet        The packet opaque pointer
+ *
+ * If the reference counter goes below one, trace_fin_packet() will be
+ * called on the packet.
+ *
+ * You may wish to use this function (and its incrementing counterpart)
+ * in situations where you are retaining multiple references to a packet
+ * outside of the core packet processing function. This will ensure that
+ * the packet is not released until there are no more outstanding references
+ * to the packet anywhere in your program.
+ */
+DLLEXPORT void trace_decrement_packet_refcount(libtrace_packet_t *packet);
+
 
 /** Provides some basic information about a trace based on its input format.

lib/linktypes.c

@@ -109 +109 @@
                 case TRACE_TYPE_METADATA:
                 case TRACE_TYPE_NONDATA:
+                case TRACE_TYPE_ETSILI:
                         break;
                 case TRACE_TYPE_UNKNOWN:
@@ -211 +212 @@
                 case TRACE_TYPE_NONDATA:
                 case TRACE_TYPE_OPENBSD_LOOP:
+                case TRACE_TYPE_ETSILI:
                 case TRACE_TYPE_UNKNOWN:
                         break;

lib/protocols_l2.c

@@ -483 +483 @@
                 case TRACE_TYPE_PFLOG:
                 case TRACE_TYPE_ERF_META:
+                case TRACE_TYPE_ETSILI:
                         break;
                 case TRACE_TYPE_UNKNOWN:
@@ -519 +520 @@
                                 case TRACE_TYPE_PFLOG:
                                 case TRACE_TYPE_ERF_META:
+                                case TRACE_TYPE_ETSILI:
                                         break;
                                 case TRACE_TYPE_UNKNOWN:
@@ -574 +576 @@
                 case TRACE_TYPE_PFLOG:
                 case TRACE_TYPE_LINUX_SLL:
+                case TRACE_TYPE_ETSILI:
                         return NULL;
 
@@ -696 +699 @@
                 case TRACE_TYPE_80211_PRISM:
                 case TRACE_TYPE_80211_RADIO:
+                case TRACE_TYPE_ETSILI:
                         assert(!"Metadata headers should already be skipped");
                         break;
@@ -746 +750 @@
                 case TRACE_TYPE_80211_PRISM:
                 case TRACE_TYPE_80211_RADIO:
+                case TRACE_TYPE_ETSILI:
                         assert(!"Metadata headers should already be skipped");
                         break;

lib/protocols_pktmeta.c

@@ -28 +28 @@
 #include "protocols.h"
 #include <assert.h>
+
+#ifdef HAVE_WANDDER
+#include <libwandder_etsili.h>
+#endif
 
 /* This file contains all the protocol decoding functions for the meta-data
@@ -121 +125 @@
 }
 
+static void *trace_get_payload_from_etsili(const void *link,
+                libtrace_linktype_t *type, uint32_t *remaining) {
+
+#ifdef HAVE_WANDDER
+        wandder_etsispec_t *dec;
+        uint8_t *ccptr;
+
+        /* XXX Bit annoying to be creating and freeing this every time */
+        dec = wandder_create_etsili_decoder();
+        wandder_attach_etsili_buffer(dec, (uint8_t *)link, *remaining, false);
+        ccptr = wandder_etsili_get_cc_contents(dec, remaining, NULL, 0);
+        /* Assuming all CCs are IP for now */
+        *type = TRACE_TYPE_NONE;
+        wandder_free_etsili_decoder(dec);
+        return ccptr;
+
+#else
+        (void)link;
+        (void)type;
+        *remaining = 0;
+        return NULL;
+#endif
+
+}
+
 DLLEXPORT void *trace_get_packet_meta(const libtrace_packet_t *packet, 
                 libtrace_linktype_t *linktype,
@@ -139 +168 @@
                 case TRACE_TYPE_80211_PRISM:
                 case TRACE_TYPE_ERF_META:
+                case TRACE_TYPE_ETSILI:
                         return pktbuf;
                 /* Non metadata packets */
@@ -199 +229 @@
                                         linktype, remaining);
                         return nexthdr;
+                case TRACE_TYPE_ETSILI:
+                        nexthdr = trace_get_payload_from_etsili(meta,
+                                        linktype, remaining);
+                        return nexthdr;
+
                 case TRACE_TYPE_HDLC_POS:
                 case TRACE_TYPE_ETH:

lib/strndup.c

@@ -24 +24 @@
  *
  */
-#ifndef HAVE_STRNDUP
+#include "config.h"
+#if !HAVE_DECL_STRNDUP
 
 #include <stdlib.h>

lib/trace.c

@@ -107 +107 @@
  * will use our own one that does
  */
-static void xstrncpy(char *dest, const char *src, size_t n)
-{
-        strncpy(dest,src,n);
-        dest[n]='\0';
+static inline void xstrncpy(char *dest, const char *src, size_t n,
+                size_t destlen)
+{
+        size_t slen = destlen - 1;
+        if (n < slen) {
+                slen = n;
+        }
+        strncpy(dest,src,slen);
+        dest[slen]='\0';
 }
 
@@ -120 +125 @@
                 exit(EXIT_FAILURE);
         }
-        xstrncpy(ret,src,n);
+        xstrncpy(ret,src,n,n+1);
         return ret;
 }
@@ -148 +153 @@
                 rt_constructor();
                 ndag_constructor();
+#ifdef HAVE_WANDDER
+                etsilive_constructor();
+#endif
 #ifdef HAVE_DAG
                 dag_constructor();
@@ -248 +256 @@
         libtrace->format=NULL;
 
-        libtrace->event.tdelta = 0.0;
         libtrace->event.packet = NULL;
         libtrace->event.psize = 0;
-        libtrace->event.trace_last_ts = 0.0;
+        libtrace->event.first_ts = 0.0;
+        libtrace->event.first_now = 0.0;
         libtrace->event.waiting = false;
         libtrace->filter = NULL;
         libtrace->snaplen = 0;
+        libtrace->replayspeedup = 1;
         libtrace->started=false;
         libtrace->uridata = NULL;
@@ -367 +376 @@
 
         if((uridata = strchr(uri,':')) == NULL) {
-                xstrncpy(scan, uri, strlen(uri));
+                xstrncpy(scan, uri, strlen(uri), URI_PROTO_LINE);
         } else {
-                xstrncpy(scan,uri, (size_t)(uridata - uri));
+                xstrncpy(scan,uri, (size_t)(uridata - uri), URI_PROTO_LINE);
         }
 
@@ -375 +384 @@
         libtrace->format=NULL;
 
-        libtrace->event.tdelta = 0.0;
         libtrace->event.packet = NULL;
         libtrace->event.psize = 0;
-        libtrace->event.trace_last_ts = 0.0;
+        libtrace->event.first_ts = 0;
+        libtrace->event.first_now = 0;
         libtrace->filter = NULL;
         libtrace->snaplen = 0;
@@ -592 +601 @@
          * deal with some options itself, so give that a go */
         switch(option) {
+                case TRACE_OPTION_REPLAY_SPEEDUP:
+                        /* Clear the error if there was one */
+                        if (trace_is_err(libtrace)) {
+                                trace_get_err(libtrace);
+                        }
+                        if (*(int*)value<1
+                                || *(int*)value>LIBTRACE_MAX_REPLAY_SPEEDUP) {
+                                trace_set_err(libtrace,TRACE_ERR_BAD_STATE,
+                                        "Invalid replay speed");
+                        }
+                        libtrace->replayspeedup=*(int*)value;
+                        return 0;
+
                 case TRACE_OPTION_SNAPLEN:
                         /* Clear the error if there was one */
@@ -724 +746 @@
                 libtrace_ocache_destroy(&libtrace->packet_freelist);
                 for (i = 0; i < libtrace->perpkt_thread_count; ++i) {
-                        libtrace_message_queue_destroy(&libtrace->perpkt_threads[i].messages);
-                }
-                libtrace_message_queue_destroy(&libtrace->hasher_thread.messages);
-                libtrace_message_queue_destroy(&libtrace->keepalive_thread.messages);
-                libtrace_message_queue_destroy(&libtrace->reporter_thread.messages);
+                        libtrace_message_queue_destroy(&libtrace->perpkt_threads[i].messages);
+                }
+                if (libtrace->hasher_thread.type == THREAD_HASHER)
+                        libtrace_message_queue_destroy(&libtrace->hasher_thread.messages);
+                if (libtrace->keepalive_thread.type == THREAD_KEEPALIVE)
+                        libtrace_message_queue_destroy(&libtrace->keepalive_thread.messages);
+                if (libtrace->reporter_thread.type == THREAD_REPORTER)
+                        libtrace_message_queue_destroy(&libtrace->reporter_thread.messages);
 
 
@@ -796 +821 @@
 }
 
+DLLEXPORT int trace_flush_output(libtrace_out_t *libtrace) {
+        if (!libtrace) {
+                return -1;
+        }
+        if (libtrace->format && libtrace->format->flush_output) {
+                return libtrace->format->flush_output(libtrace);
+        }
+        return 0;
+}
+
 DLLEXPORT libtrace_packet_t *trace_create_packet(void)
 {
@@ -805 +840 @@
 
         packet->buf_control=TRACE_CTRL_PACKET;
+        pthread_mutex_init(&(packet->ref_lock), NULL);
         trace_clear_cache(packet);
         return packet;
@@ -856 +892 @@
                 free(packet->buffer);
         }
+        pthread_mutex_destroy(&(packet->ref_lock));
         packet->buf_control=(buf_control_t)'\0';
                                 /* A "bad" value to force an assert
@@ -1538 +1575 @@
                                 trace_set_err(packet->trace,
                                                 TRACE_ERR_NO_CONVERSION,
-                                                "pcap does not support this format");
+                                                "pcap does not support this linktype so cannot apply BPF filters");
                                 if (free_packet_needed) {
                                         trace_destroy_packet(packet_copy);
@@ -2316 +2353 @@
         packet->l3_remaining = 0;
         packet->l4_remaining = 0;
+        packet->refcount = 0;
 
 }

lib/trace_parallel.c

@@ -690 +690 @@
                         if (!trace->pread) {
                                 assert(packets[0]);
-                                ASSERT_RET(pthread_mutex_lock(&trace->libtrace_lock), == 0);
                                 nb_packets = trace_read_packet(trace, packets[0]);
-                                ASSERT_RET(pthread_mutex_unlock(&trace->libtrace_lock), == 0);
                                 packets[0]->error = nb_packets;
                                 if (nb_packets > 0)
@@ -1048 +1046 @@
         ASSERT_RET(pthread_spin_unlock(&libtrace->first_packets.lock), == 0);
 
+        memset(&mesg, 0, sizeof(libtrace_message_t));
         mesg.code = MESSAGE_FIRST_PACKET;
         trace_message_reporter(libtrace, &mesg);
@@ -1195 +1194 @@
 
         gettimeofday(&prev, NULL);
+        memset(&message, 0, sizeof(libtrace_message_t));
         message.code = MESSAGE_TICK_INTERVAL;
 
@@ -1546 +1546 @@
                        const char *name) {
 #ifdef __linux__
-        pthread_attr_t attrib;
         cpu_set_t cpus;
         int i;
@@ -1564 +1563 @@
         for (i = 0; i < get_nb_cores(); i++)
                 CPU_SET(i, &cpus);
-        pthread_attr_init(&attrib);
-        pthread_attr_setaffinity_np(&attrib, sizeof(cpus), &cpus);
-        ret = pthread_create(&t->tid, &attrib, start_routine, (void *) trace);
-        pthread_attr_destroy(&attrib);
+
+        ret = pthread_create(&t->tid, NULL, start_routine, (void *) trace);
+        if( ret == 0 ) {
+                ret = pthread_setaffinity_np(t->tid, sizeof(cpus), &cpus);
+        }
+
 #else
         ret = pthread_create(&t->tid, NULL, start_routine, (void *) trace);
@@ -2004 +2005 @@
         }
         if (!libtrace->started || libtrace->state != STATE_RUNNING) {
+                ASSERT_RET(pthread_mutex_unlock(&libtrace->libtrace_lock), == 0);
                 trace_set_err(libtrace,TRACE_ERR_BAD_STATE, "You must call trace_start() before calling trace_ppause()");
                 return -1;
@@ -2402 +2404 @@
 
 DLLEXPORT int trace_set_perpkt_threads(libtrace_t *trace, int nb) {
-        if (!trace_is_configurable(trace)) return -1;
+        // Only supported on new traces not paused traces
+        if (trace->state != STATE_NEW) return -1;
 
         /* TODO consider allowing an offset from the total number of cores i.e.
@@ -2601 +2604 @@
 }
 
+DLLEXPORT void trace_increment_packet_refcount(libtrace_packet_t *packet) {
+        pthread_mutex_lock(&(packet->ref_lock));
+        if (packet->refcount < 0) {
+                packet->refcount = 1;
+        } else {
+                packet->refcount ++;
+        }
+        pthread_mutex_unlock(&(packet->ref_lock));
+}
+
+DLLEXPORT void trace_decrement_packet_refcount(libtrace_packet_t *packet) {
+        pthread_mutex_lock(&(packet->ref_lock));
+        packet->refcount --;
+
+        if (packet->refcount <= 0) {
+                trace_free_packet(packet->trace, packet);
+        }
+        pthread_mutex_unlock(&(packet->ref_lock));
+}
+
+
 DLLEXPORT libtrace_info_t *trace_get_information(libtrace_t * libtrace) {
         if (libtrace->format)

libpacketdump/Makefile.am

@@ -60 +60 @@
 # 17: PPP
 TXT_PROTOCOLS+=link_17.protocol
+
+# 22: ETSI LI
+if HAVE_WANDDER
+BIN_PROTOCOLS+=link_22.la
+endif
 
 # Decoders for various ethertypes (in decimal)
@@ -135 +140 @@
 link_11_la_LDFLAGS=$(modflags)
 link_15_la_LDFLAGS=$(modflags) 
+if HAVE_WANDDER
+link_22_la_LDFLAGS=$(modflags) 
+endif
 eth_0_la_LDFLAGS=$(modflags)
 eth_2048_la_LDFLAGS=$(modflags)

libpacketdump/eth_2054.c

@@ -47 +47 @@
 static char *format_hrd(const struct arphdr *arp, const char *hrd) {
         static char buffer[1024] = {0,};
-        int i;
+        int i, ret;
+        size_t bufused;
 
         if (!hrd) {
@@ -59 +60 @@
                         break;
                 default:
+                        bufused = 0;
                         for (i=0;i<arp->ar_hln;i++) {
-                                snprintf(buffer,sizeof(buffer),"%s %02x",
-                                                buffer,(unsigned char)hrd[i]);
+                                if (bufused >= sizeof(buffer)) {
+                                        break;
+                                }
+                                ret = snprintf(buffer + bufused,
+                                                sizeof(buffer) - bufused,
+                                                "%02x ",
+                                                (unsigned char)hrd[i]);
+                                if (ret > 0) {
+                                        bufused += ret;
+                                }
                         }
                         break;
@@ -77 +87 @@
 static char *format_pro(const struct arphdr *arp, const char *pro) {
         static char buffer[1024] = {0,};
-        int i;
+        int i, ret;
+        size_t bufused;
         
         if (!pro) {
@@ -91 +102 @@
                 default:
                         snprintf(buffer, sizeof(buffer), "%s", " (");
+                        bufused = 2;
                         for (i=0;i<arp->ar_pln;i++) {
-                                snprintf(buffer,sizeof(buffer),"%s %02x",
-                                                buffer,(unsigned char)pro[i]);
+                                if (bufused >= sizeof(buffer)) {
+                                        break;
+                                }
+                                ret = snprintf(buffer + bufused,
+                                                sizeof(buffer) - bufused,
+                                                "%02x ",
+                                                (unsigned char)pro[i]);
+                                if (ret > 0) {
+                                        bufused += ret;
+                                }
                         }
-                        strncat(buffer,")",sizeof(buffer) - strlen(buffer) - 1);
+                        if (bufused < sizeof(buffer)) {
+                                snprintf(buffer + bufused,
+                                                sizeof(buffer) - bufused,
+                                                ")");
+                        }
                         break;
         }

test/do-test-build-dpdk.sh

@@ -23 +23 @@
 }
 
-# Versions to check
+# Old kernel version jessie 3.16
 declare -a dpdk_versions=(
         "dpdk-1.7.1.tar.gz"
@@ -31 +31 @@
         "dpdk-2.2.0.tar.gz"
         "dpdk-16.04.tar.gz"
-        "dpdk-16.07.tar.gz"
-        "dpdk-16.11.tar.gz"
+        "dpdk-16.07.2.tar.gz"
+        "dpdk-16.11.6.tar.gz"
         )
+# Versions to check stretch linux 4.9
+declare -a dpdk_versions=(
+        "dpdk-2.2.0.tar.gz"
+        "dpdk-16.04.tar.gz"
+        "dpdk-16.07.2.tar.gz"
+        "dpdk-16.11.6.tar.gz"
+        "dpdk-17.02.1.tar.gz"
+        "dpdk-17.05.2.tar.gz"
+        "dpdk-17.08.2.tar.gz"
+        "dpdk-17.11.2.tar.gz"
+        "dpdk-18.02.1.tar.gz"
+        )
+
 
 mkdir "$DOWNLOAD_DIR" > /dev/null 2>&1
@@ -60 +73 @@
 # Build the DPDK libraries
 # We try to not overwrite these, so that a rebuild is faster
+# We build DPDK without KNI, as most kernel dependent code is there
+#   - also excluding makes the build faster
+# We also disable error on warning, to improve forwards compiler compatibility
 cd "$DOWNLOAD_DIR"
 for dpdk_build in $(ls -d */)
@@ -67 +83 @@
         do_test make install T=x86_64-native-linuxapp-gcc \
                              CONFIG_RTE_BUILD_COMBINE_LIBS=y \
-                             EXTRA_CFLAGS="-fPIC" -j $BUILD_THREADS \
+                             CONFIG_RTE_LIBRTE_KNI=n \
+                             CONFIG_RTE_KNI_KMOD=n \
+                             EXTRA_CFLAGS="-fPIC -w" -j $BUILD_THREADS \
                              > build_stdout.txt 2> build_stderr.txt
         cd ..

tools/traceanon/traceanon.1

@@ -101 +101 @@
 .BI \-\^\-compress-type=method
 compress the output trace using the compression algorithm "method". Possible
-algorithms are "gzip", "bzip2", "lzo", "xz" and "none". Default is "none".
+algorithms are "gz", "bz", "lzo", "xz" and "no". Default is "no".
 
 .TP

tools/tracemerge/tracemerge.1

@@ -53 +53 @@
 .BI \-\^\-compress-type method 
 Describes the compression algorithm to be used when writing the output trace.
-Possible methods are "gzip", "bzip2", "lzo", "xz" and "none". Defaults to 
-"none".
+Possible methods are "gz", "bz", "lzo", "xz" and "no". Defaults to 
+"no".
 
 

tools/tracepktdump/tracepktdump.cc

@@ -113 +113 @@
                                 /* Ignore RT messages */
                                 continue;
+                        if (trace_is_err(trace)) {
+                                break;
+                        }
                         trace_dump_packet(packet);
 

tools/tracereplay/tracereplay.1

@@ -4 +4 @@
 .SH SYNOPSIS
 .B tracereplay
-[\-b | \-\^\-broadcast] [-s \-\^\-snaplength [ snaplength] ] 
-[\-f | \-\^\-filter [ filter string ] ]
+[\-b | \-\^\-broadcast] [-s \-\^\-snaplength [ snaplength] ]
+[\-f | \-\^\-filter [ filter string ] ] [\-X | \-\^\-speedup [ factor] ]
 inputuri outputuri
 .SH DESCRPTION
@@ -23 +23 @@
 .TP
 .PD 0
-.BI \-s [snaplength]
+.BI \-s " snaplength"
 .TP
 .PD
-.BI \-\^\-snaplength [snaplength]
+.BI \-\^\-snaplength " snaplength"
 Truncate the packets to the length specified. This only makes a difference
 if the inputuri is a capture device. The value should be > 0. Inappropriate 
@@ -33 +33 @@
 .TP
 .PD 0
-.BI \-f [ filter string ]
+.BI \-f " filterstring"
 .TP
 .PD
-.BI \-\^\-filter [ filter ]
+.BI \-\^\-filter " filterstring"
 Apply a filter to the inputuri.
+
+.TP
+.PD 0
+.BI \-X " factor"
+.TP
+.PD
+.BI \-\^\-speedup " factor"
+Decrease the gaps between packets by the specified factor. This will accelerate
+the rate at which the replay is performed. By default, the factor is 1 (i.e.
+no acceleration).
 
 .SH LINKS

tools/tracereplay/tracereplay.c

@@ -98 +98 @@
         if (IS_LIBTRACE_META_PACKET(packet))
                 return NULL;
+        if (trace_get_wire_length(packet) == 0)
+                return NULL;
 
         pkt_buffer = trace_get_packet_buffer(packet,&linktype,&remaining);
@@ -194 +196 @@
         fprintf(stderr, " --broadcast\n");
         fprintf(stderr, "\t\tSend ethernet frames to broadcast address\n");
+        fprintf(stderr, " -X\n");
+        fprintf(stderr, " --speedup\n");
+        fprintf(stderr, "\t\tSpeed up replay by a factor of <speedup>\n");
 
 }
@@ -207 +212 @@
         libtrace_packet_t * new;
         int snaplen = 0;
+        int speedup = 1;
 
 
@@ -216 +222 @@
                         { "snaplen",    1, 0, 's'},
                         { "broadcast",  0, 0, 'b'},
+                        { "speedup",    1, 0, 'X'},
                         { NULL,         0, 0, 0}
                 };
 
-                int c = getopt_long(argc, argv, "bhs:f:",
+                int c = getopt_long(argc, argv, "bhs:f:X:",
                                 long_options, &option_index);
 
@@ -232 +239 @@
                                 snaplen = atoi(optarg);
                                 break;
-
+                        case 'X':
+                                speedup = atoi(optarg);
+                                break;
                         case 'b':
                                 broadcast = 1;
@@ -257 +266 @@
         }
 
+        if (speedup < 1) {
+                speedup = 1;
+        }
+
         uri = strdup(argv[optind]);
 
@@ -279 +292 @@
                 }
         }
+
+        if (trace_config(trace, TRACE_OPTION_REPLAY_SPEEDUP, &speedup)) {
+                trace_perror(trace, "error setting replay speedup factor");
+                return 1;
+        }
 
         /* Starting the trace */

tools/tracereport/misc_report.c

@@ -67 +67 @@
 {
         static char ret[1024];
-        char tmp[1024];
+        char tmp[128];
         ret[0]='\0';
         if (ts == 0) 

tools/tracesplit/tracesplit.1

@@ -69 +69 @@
 \fB-Z\fR compression-method
 Compress the data using the specified compression algorithm. Accepted methods
-are "gzip", "bzip2", "lzo", "xz" or "none". Default value is none unless a 
+are "gz", "bz", "lzo", "xz" or "no". Default value is "no" unless a 
 compression level is specified, in which case gzip will be used.
 

tools/tracesplit/tracesplit.c

@@ -61 +61 @@
 static char *strdupcat(char *str,char *app)
 {
-        str=realloc(str,strlen(str)+strlen(app)+1);
-        strncat(str,app,strlen(str) + strlen(app));
+        int newsize = strlen(str)+strlen(app)+1;
+        str=realloc(str,newsize);
+        strncat(str,app,newsize - strlen(str) - 1);
         return str;
 }