Changeset e26c1fc

Timestamp:

05/06/16 11:37:53 (5 years ago)

Author:

Shane Alcock <salcock@…>

Branches:

4.0.1-hotfixes, cachetimestamps, develop, dpdk-ndag, etsilive, master, ndag_format, rc-4.0.1, rc-4.0.2, rc-4.0.3, rc-4.0.4, ringdecrementfix, ringperformance, ringtimestampfixes

Children:

2b0eae9

Parents:

89aa13f (diff), ee0805a (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge remote-tracking branch 'origin/develop' into develop

Files:

• INSTALL (modified) (2 diffs)
• README (modified) (1 diff)
• config-win.h (deleted)
• examples/tutorial/sourcedemo.c (modified) (3 diffs)
• lib/format_dpdk.c (modified) (3 diffs)
• lib/libtrace.h.in (modified) (1 diff)
• lib/trace.c (modified) (82 diffs)
• libtrace.sln (deleted)
• libtrace.vcproj (deleted)
• test-convert.vcproj (deleted)
• test-format.vcproj (deleted)
• test-pcap-bpf.vcproj (deleted)
• test/vxlan.pcap (added)
• tools/tracesplit/tracesplit.1 (modified) (2 diffs)
• tools/tracesplit/tracesplit.c (modified) (17 diffs)

INSTALL

@@ -39 +39 @@
 
 
+Q. What operating systems do you support?
+
+A. Linux, FreeBSD, Mac OS X and OpenBSD. Windows is not supported.
+
 ----------------------------------
 Using libtrace:
@@ -44 +48 @@
 The best source of information on how to use libtrace and the tools that come
 with it is the libtrace wiki located at 
-http://www.wand.net.nz/trac/libtrace/wiki
+http://www.github.com/wanduow/libtrace/wiki
 
 

README

@@ -13 +13 @@
 This directory contains source code for libtrace, a userspace library for 
 processing of network traffic capture from live interfaces or from offline
-traces.
+traces. 
 
 libtrace was primarily designed for use with the real-time interface to the 
 Waikato DAG Capture Point software running at The University of Waikato, 
 and has been since extended to a range of other trace and interface formats.
+
+Libtrace should build and run on Linux, Mac OS X, FreeBSD and OpenBSD systems.
 
 Further information about libtrace see 

examples/tutorial/sourcedemo.c

@@ -28 +28 @@
 static inline void print_ip(struct sockaddr *ip) {
 
-        char str[20];
+        char str[40];
         
         /* Check the sockaddr family so we can cast it to the appropriate
@@ -37 +37 @@
                 /* Use inet_ntop to convert the address into a string using
                  * dotted decimal notation */
-                printf("%s ", inet_ntop(AF_INET, &(v4->sin_addr), str, 20));
+                printf("%s ", inet_ntop(AF_INET, &(v4->sin_addr), str, sizeof(str)));
         }
 
@@ -45 +45 @@
                 /* Use inet_ntop to convert the address into a string using
                  * IPv6 address notation */
-                printf("%s ", inet_ntop(AF_INET6, &(v6->sin6_addr), str, 20));
+                printf("%s ", inet_ntop(AF_INET6, &(v6->sin6_addr), str, sizeof(str)));
         }
 

lib/format_dpdk.c

@@ -135 +135 @@
 #else
 #       define DPDK_USE_LOG_LEVEL 0
+#endif
+
+/* 1.8.0-rc2
+ * rx/tx_conf thresholds can be set to NULL in rte_eth_rx/tx_queue_setup
+ * this uses the default values, which are better tuned per device
+ * See issue #26
+ */
+#if RTE_VERSION >= RTE_VERSION_NUM(1, 8, 0, 2)
+#       define DPDK_USE_NULL_QUEUE_CONFIG 1
+#else
+#       define DPDK_USE_NULL_QUEUE_CONFIG 0
 #endif
 
@@ -898 +909 @@
      */
     ret = rte_eth_tx_queue_setup(format_data->port, format_data->queue_id,
-                        format_data->nb_tx_buf, rte_socket_id(), &tx_conf);
+                        format_data->nb_tx_buf, rte_socket_id(),
+                        DPDK_USE_NULL_QUEUE_CONFIG ? NULL : &tx_conf);
     if (ret < 0) {
         snprintf(err, errlen, "Intel DPDK - Cannot configure TX queue on port"
@@ -908 +920 @@
     ret = rte_eth_rx_queue_setup(format_data->port, format_data->queue_id,
                             format_data->nb_rx_buf, rte_socket_id(),
-                            &rx_conf, format_data->pktmbuf_pool);
+                            DPDK_USE_NULL_QUEUE_CONFIG ? NULL : &rx_conf,
+                            format_data->pktmbuf_pool);
     if (ret < 0) {
         snprintf(err, errlen, "Intel DPDK - Cannot configure RX queue on port"

lib/libtrace.h.in

@@ -1425 +1425 @@
 /** Create a new packet object
  *
- * @return A pointer to an initialised libtrace_packet_t object
+ * @return A pointer to an initialised libtrace_packet_t object, or NULL if
+ * libtrace is unable to allocate space for a new packet.
  */
 DLLEXPORT libtrace_packet_t *trace_create_packet(void);

lib/trace.c

@@ -2 +2 @@
  * This file is part of libtrace
  *
- * Copyright (c) 2007,2008,2009,2010 The University of Waikato, Hamilton, 
+ * Copyright (c) 2007,2008,2009,2010 The University of Waikato, Hamilton,
  * New Zealand.
  *
- * Authors: Daniel Lawson 
+ * Authors: Daniel Lawson
  *          Perry Lorier
- *          Shane Alcock 
- *          
+ *          Shane Alcock
+ *
  * All rights reserved.
  *
- * This code has been developed by the University of Waikato WAND 
+ * This code has been developed by the University of Waikato WAND
  * research group. For further information please see http://www.wand.net.nz/
  *
@@ -115 +115 @@
         dest[n]='\0';
 }
- 
+
 static char *xstrndup(const char *src,size_t n)
-{       
+{
         char *ret=(char*)malloc(n+1);
         if (ret==NULL) {
@@ -153 +153 @@
 }
 
-/* Prints help information for libtrace 
+/* Prints help information for libtrace
  *
  * Function prints out some basic help information regarding libtrace,
@@ -176 +176 @@
 {
         struct libtrace_format_t *tmp;
-        
+
         /* Try and guess based on filename */
         for(tmp = formats_list; tmp; tmp=tmp->next) {
@@ -198 +198 @@
                 }
         }
-        
+
         /* No formats matched -- make sure we clean up the IO object we
          * used to probe the file magic */
@@ -213 +213 @@
  *  erf:/path/to/erf/file
  *  erf:/path/to/erf/file.gz
- *  erf:-                       (stdin)
+ *  erf:-                       (stdin)
  *  dag:/dev/dagcard
- *  pcapint:pcapinterface               (eg: pcapint:eth0)
+ *  pcapint:pcapinterface               (eg: pcapint:eth0)
  *  pcapfile:/path/to/pcap/file
  *  pcapfile:-
@@ -226 +226 @@
  */
 DLLEXPORT libtrace_t *trace_create(const char *uri) {
-        libtrace_t *libtrace = 
+        libtrace_t *libtrace =
                         (libtrace_t *)malloc(sizeof(libtrace_t));
         char *scan = 0;
-        const char *uridata = 0;                  
+        const char *uridata = 0;
 
         trace_init();
@@ -239 +239 @@
                 return NULL;
         }
-        
+
         libtrace->err.err_num = TRACE_ERR_NOERROR;
         libtrace->format=NULL;
-        
+
         libtrace->event.tdelta = 0.0;
         libtrace->event.packet = NULL;
@@ -289 +289 @@
          * libtrace->uridata contains the appropriate data for this
          */
-       
-        /* Call the init_input function for the matching capture format */ 
+
+        /* Call the init_input function for the matching capture format */
         if (libtrace->format->init_input) {
                 int err=libtrace->format->init_input(libtrace);
                 assert (err==-1 || err==0);
                 if (err==-1) {
-                        /* init_input should call trace_set_err to set 
-                         * the error message
+                        /* init_input should call trace_set_err to set the
+                         * error message
                          */
                         return libtrace;
@@ -305 +305 @@
                 return libtrace;
         }
-        
+
         if (scan)
                 free(scan);
@@ -328 +328 @@
 
         trace_init();
-        
+
         libtrace->err.err_num = TRACE_ERR_NOERROR;
 
@@ -336 +336 @@
                 xstrncpy(scan,uri, (size_t)(uridata - uri));
         }
-        
+
         libtrace->err.err_num = TRACE_ERR_NOERROR;
         libtrace->format=NULL;
-        
+
         libtrace->event.tdelta = 0.0;
         libtrace->event.packet = NULL;
@@ -350 +350 @@
         libtrace->io = NULL;
         libtrace->filtered_packets = 0;
-        
+
         for(tmp=formats_list;tmp;tmp=tmp->next) {
                 if (strlen(scan) == strlen(tmp->name) &&
@@ -371 +371 @@
 }
 
-/* Creates an output trace from a URI. 
+/* Creates an output trace from a URI.
  *
  * @param uri   the uri string describing the output format and destination
- * @returns opaque pointer to a libtrace_output_t 
+ * @returns opaque pointer to a libtrace_output_t
  *
  *  If an error occured when attempting to open the output trace, NULL is
- *  returned and trace_errno is set. 
- */
-        
+ *  returned and trace_errno is set.
+ */
+
 DLLEXPORT libtrace_out_t *trace_create_output(const char *uri) {
-        libtrace_out_t *libtrace = 
+        libtrace_out_t *libtrace =
                         (libtrace_out_t*)malloc(sizeof(libtrace_out_t));
-        
+
         char *scan = 0;
         const char *uridata = 0;
@@ -394 +394 @@
         libtrace->format = NULL;
         libtrace->uridata = NULL;
-        
+
         /* Parse the URI to determine what capture format we want to write */
 
@@ -402 +402 @@
                 return libtrace;
         }
-        
+
         /* Attempt to find the format in the list of supported formats */
         for(tmp=formats_list;tmp;tmp=tmp->next) {
@@ -471 +471 @@
 
 /* Start an output trace */
-DLLEXPORT int trace_start_output(libtrace_out_t *libtrace) 
+DLLEXPORT int trace_start_output(libtrace_out_t *libtrace)
 {
         assert(libtrace);
@@ -507 +507 @@
                 return -1;
         }
-        
+
         /* If the capture format supports configuration, try using their
          * native configuration first */
@@ -517 +517 @@
 
         /* If we get here, either the native configuration failed or the
-         * format did not support configuration. However, libtrace can 
+         * format did not support configuration. However, libtrace can
          * deal with some options itself, so give that a go */
         switch(option) {
@@ -525 +525 @@
                                 trace_get_err(libtrace);
                         }
-                        if (*(int*)value<0 
+                        if (*(int*)value<0
                                 || *(int*)value>LIBTRACE_PACKET_BUFSIZE) {
                                 trace_set_err(libtrace,TRACE_ERR_BAD_STATE,
@@ -547 +547 @@
                 case TRACE_OPTION_META_FREQ:
                         if (!trace_is_err(libtrace)) {
-                                trace_set_err(libtrace, 
+                                trace_set_err(libtrace,
                                                 TRACE_ERR_OPTION_UNAVAIL,
                                                 "This format does not support meta-data gathering");
@@ -554 +554 @@
                 case TRACE_OPTION_EVENT_REALTIME:
                         if (!trace_is_err(libtrace)) {
-                                trace_set_err(libtrace, 
+                                trace_set_err(libtrace,
                                                 TRACE_ERR_OPTION_UNAVAIL,
                                                 "This format does not support realtime events");
                         }
                         return -1;
-                        
+
         }
         if (!trace_is_err(libtrace)) {
@@ -568 +568 @@
 }
 
-DLLEXPORT int trace_config_output(libtrace_out_t *libtrace, 
+DLLEXPORT int trace_config_output(libtrace_out_t *libtrace,
                 trace_option_output_t option,
                 void *value) {
-        
+
         /* Unlike the input options, libtrace does not natively support any of
          * the output options - the format module must be able to deal with
@@ -625 +625 @@
  * @param libtrace      the output trace file to be destroyed
  */
-DLLEXPORT void trace_destroy_output(libtrace_out_t *libtrace) 
+DLLEXPORT void trace_destroy_output(libtrace_out_t *libtrace)
 {
         assert(libtrace);
@@ -635 +635 @@
 }
 
-DLLEXPORT libtrace_packet_t *trace_create_packet(void) 
-{
-        libtrace_packet_t *packet = 
+DLLEXPORT libtrace_packet_t *trace_create_packet(void)
+{
+        libtrace_packet_t *packet =
                 (libtrace_packet_t*)calloc((size_t)1,sizeof(libtrace_packet_t));
+
+        if (packet == NULL)
+                return NULL;
 
         packet->buf_control=TRACE_CTRL_PACKET;
@@ -646 +649 @@
 
 DLLEXPORT libtrace_packet_t *trace_copy_packet(const libtrace_packet_t *packet) {
-        libtrace_packet_t *dest = 
+        libtrace_packet_t *dest =
                 (libtrace_packet_t *)malloc(sizeof(libtrace_packet_t));
         if (!dest) {
@@ -665 +668 @@
         /* Reset the cache - better to recalculate than try to convert
          * the values over to the new packet */
-        trace_clear_cache(dest);        
+        trace_clear_cache(dest);
         /* Ooooh nasty memcpys! This is why we want to avoid copying packets
          * as much as possible */
@@ -680 +683 @@
                 free(packet->buffer);
         }
-        packet->buf_control=(buf_control_t)'\0'; 
+        packet->buf_control=(buf_control_t)'\0';
                                 /* A "bad" value to force an assert
                                  * if this packet is ever reused
                                  */
         free(packet);
-}       
+}
 
 /* Read one packet from the trace into buffer. Note that this function will
  * block until a packet is read (or EOF is reached).
  *
- * @param libtrace      the libtrace opaque pointer
- * @param packet        the packet opaque pointer
+ * @param libtrace      the libtrace opaque pointer
+ * @param packet        the packet opaque pointer
  * @returns 0 on EOF, negative value on error
  *
@@ -704 +707 @@
                 return -1;
         }
-        if (!(packet->buf_control==TRACE_CTRL_PACKET || packet->buf_control==TRACE_CTRL_EXTERNAL)) {
+        if (!(packet->buf_control==TRACE_CTRL_PACKET
+                    || packet->buf_control==TRACE_CTRL_EXTERNAL)) {
                 trace_set_err(libtrace,TRACE_ERR_BAD_STATE,"Packet passed to trace_read_packet() is invalid\n");
                 return -1;
         }
         assert(packet);
-      
-        /* Store the trace we are reading from into the packet opaque 
+
+        /* Store the trace we are reading from into the packet opaque
          * structure */
         packet->trace = libtrace;
@@ -741 +745 @@
                                         return ~0U;
                                 }
-                                
+
                                 if (filtret == 0) {
                                         ++libtrace->filtered_packets;
@@ -765 +769 @@
  * appropriate capture format header for the format type that the packet is
  * being converted to. This also allows for a packet to be converted into
- * just about capture format that is supported by libtrace, provided the 
+ * just about capture format that is supported by libtrace, provided the
  * format header is present in the buffer.
  *
  * This function is primarily used to convert packets received via the RT
  * protocol back into their original capture format. The RT header encapsulates
- * the original capture format header, so after removing it the packet must 
+ * the original capture format header, so after removing it the packet must
  * have it's header and payload pointers updated and the packet format and type
  * changed, amongst other things.
  *
- * Intended only for internal use at this point - this function is not 
+ * Intended only for internal use at this point - this function is not
  * available through the external libtrace API.
  */
@@ -782 +786 @@
         assert(packet);
         assert(trace);
-        
+
         /* XXX Proper error handling?? */
         if (buffer == NULL)
@@ -791 +795 @@
                 return -1;
         }
-        
+
         packet->trace = trace;
-        
+
         /* Clear packet cache */
         trace_clear_cache(packet);
@@ -801 +805 @@
                                 buffer, rt_type, flags);
         }
-        trace_set_err(trace, TRACE_ERR_UNSUPPORTED, 
+        trace_set_err(trace, TRACE_ERR_UNSUPPORTED,
                         "This format does not support preparing packets\n");
         return -1;
@@ -815 +819 @@
 DLLEXPORT int trace_write_packet(libtrace_out_t *libtrace, libtrace_packet_t *packet) {
         assert(libtrace);
-        assert(packet); 
+        assert(packet);
         /* Verify the packet is valid */
         if (!libtrace->started) {
@@ -843 +847 @@
                  * wire lengths to be the "remaining" value. If the packet has
                  * been padded to increase the capture length, we don't want
-                 * to allow subsequent protocol decoders to consider the 
+                 * to allow subsequent protocol decoders to consider the
                  * padding as part of the packet.
                  *
@@ -853 +857 @@
                  * better off returning an incomplete TCP header in that case.
                  */
-                
+
                 cap_len = trace_get_capture_length(packet);
                 wire_len = trace_get_wire_length(packet);
@@ -862 +866 @@
                  * massively negative wire lens. We could assert fail here on
                  * them, but we could at least try the capture length instead.
-                 * 
+                 *
                  * You may still run into problems if you try to write that
                  * packet, but at least reading should work OK.
@@ -878 +882 @@
 
 
-/* Get a pointer to the first byte of the packet payload 
+/* Get a pointer to the first byte of the packet payload
  *
  * DEPRECATED - use trace_get_packet_buffer() instead */
@@ -885 +889 @@
 }
 
-/* Get the current time in DAG time format 
- * @param packet        a pointer to a libtrace_packet structure
+/* Get the current time in DAG time format
+ * @param packet        a pointer to a libtrace_packet structure
  * @returns a 64 bit timestamp in DAG ERF format (upper 32 bits are the seconds
  * past 1970-01-01, the lower 32bits are partial seconds)
- */ 
+ */
 DLLEXPORT uint64_t trace_get_erf_timestamp(const libtrace_packet_t *packet) {
         if (packet->trace->format->get_erf_timestamp) {
@@ -915 +919 @@
                 return (uint64_t)0;
         }
-                      
 }
 
@@ -924 +927 @@
  * @author Daniel Lawson
  * @author Perry Lorier
- */ 
+ */
 DLLEXPORT struct timeval trace_get_timeval(const libtrace_packet_t *packet) {
         struct timeval tv;
@@ -936 +939 @@
                 tv.tv_sec = ts >> 32;
                 tv.tv_usec = ((ts&0xFFFFFFFF)*1000000)>>32;
-                if (tv.tv_usec >= 1000000) {
-                        tv.tv_usec -= 1000000;
-                        tv.tv_sec += 1;
-                }
+                if (tv.tv_usec >= 1000000) {
+                        tv.tv_usec -= 1000000;
+                        tv.tv_sec += 1;
+                }
         } else if (packet->trace->format->get_timespec) {
                 struct timespec ts = packet->trace->format->get_timespec(packet);
@@ -968 +971 @@
                 ts.tv_sec = erfts >> 32;
                 ts.tv_nsec = ((erfts&0xFFFFFFFF)*1000000000)>>32;
-                if (ts.tv_nsec >= 1000000000) {
-                        ts.tv_nsec -= 1000000000;
-                        ts.tv_sec += 1;
-                }
+                if (ts.tv_nsec >= 1000000000) {
+                        ts.tv_nsec -= 1000000000;
+                        ts.tv_sec += 1;
+                }
                 return ts;
         } else if (packet->trace->format->get_timeval) {
@@ -995 +998 @@
 
 /* Get the current time in floating point seconds
- * @param packet        a pointer to a libtrace_packet structure
+ * @param packet        a pointer to a libtrace_packet structure
  * @returns time that this packet was seen in 64bit floating point seconds
- */ 
+ */
 DLLEXPORT double trace_get_seconds(const libtrace_packet_t *packet) {
         double seconds = 0.0;
@@ -1024 +1027 @@
 }
 
-DLLEXPORT size_t trace_get_capture_length(const libtrace_packet_t *packet) 
+DLLEXPORT size_t trace_get_capture_length(const libtrace_packet_t *packet)
 {
         /* Cache the capture length */
@@ -1031 +1034 @@
                         return ~0U;
                 /* Cast away constness because this is "just" a cache */
-                ((libtrace_packet_t*)packet)->capture_length = 
+                ((libtrace_packet_t*)packet)->capture_length =
                         packet->trace->format->get_capture_length(packet);
         }
@@ -1039 +1042 @@
         return packet->capture_length;
 }
-        
+
 /* Get the size of the packet as it was seen on the wire.
  * @param packet        a pointer to a libtrace_packet structure
@@ -1046 +1049 @@
  * @note Due to the trace being a header capture, or anonymisation this may
  * not be the same as the Capture Len.
- */ 
+ */
 DLLEXPORT size_t trace_get_wire_length(const libtrace_packet_t *packet){
-        
+
         if (packet->wire_length == -1) {
-                if (!packet->trace->format->get_wire_length) 
+                if (!packet->trace->format->get_wire_length)
                         return ~0U;
-                ((libtrace_packet_t *)packet)->wire_length = 
+                ((libtrace_packet_t *)packet)->wire_length =
                         packet->trace->format->get_wire_length(packet);
         }
@@ -1062 +1065 @@
 
 /* Get the length of the capture framing headers.
- * @param packet        the packet opaque pointer
+ * @param packet        the packet opaque pointer
  * @returns the size of the packet as it was on the wire.
- * @note this length corresponds to the difference between the size of a 
+ * @note this length corresponds to the difference between the size of a
  * captured packet in memory, and the captured length of the packet
- */ 
+ */
 DLLEXPORT SIMPLE_FUNCTION
 size_t trace_get_framing_length(const libtrace_packet_t *packet) {
@@ -1077 +1080 @@
 
 /* Get the type of the link layer
- * @param packet        a pointer to a libtrace_packet structure
+ * @param packet        a pointer to a libtrace_packet structure
  * @returns libtrace_linktype_t
  */
@@ -1103 +1106 @@
  * which in turn is stored inside the new packet object...
  */
-DLLEXPORT libtrace_eventobj_t trace_event(libtrace_t *trace, 
+DLLEXPORT libtrace_eventobj_t trace_event(libtrace_t *trace,
                 libtrace_packet_t *packet) {
         libtrace_eventobj_t event = {TRACE_EVENT_IOWAIT,0,0.0,0};
@@ -1115 +1118 @@
         /* Clear the packet cache */
         trace_clear_cache(packet);
-        
+
         /* Store the trace we are reading from into the packet opaque
          * structure */
@@ -1122 +1125 @@
         if (packet->trace->format->trace_event) {
                 /* Note: incrementing accepted, filtered etc. packet
-                 * counters is handled by the format-specific 
+                 * counters is handled by the format-specific
                  * function so don't increment them here.
                  */
@@ -1149 +1152 @@
         filter->filter.bf_insns = (struct bpf_insn *)
                 malloc(sizeof(struct bpf_insn) * bf_len);
-        
+
         memcpy(filter->filter.bf_insns, bf_insns,
                         bf_len * sizeof(struct bpf_insn));
-        
+
         filter->filter.bf_len = bf_len;
         filter->filterstring = NULL;
         filter->jitfilter = NULL;
         /* "flag" indicates that the filter member is valid */
-        filter->flag = 1; 
-        
+        filter->flag = 1;
+
         return filter;
 #endif
@@ -1188 +1191 @@
                 pcap_freecode(&filter->filter);
 #ifdef HAVE_LLVM
-        if (filter->jitfilter) 
+        if (filter->jitfilter)
                 destroy_program(filter->jitfilter);
 #endif
@@ -1206 +1209 @@
 static int trace_bpf_compile(libtrace_filter_t *filter,
                 const libtrace_packet_t *packet,
-                void *linkptr, 
+                void *linkptr,
                 libtrace_linktype_t linktype    ) {
 #ifdef HAVE_BPF_FILTER
@@ -1217 +1220 @@
                 return -1;
         }
-        
+
         if (filter->filterstring && ! filter->flag) {
                 pcap_t *pcap = NULL;
@@ -1236 +1239 @@
                 /* build filter */
                 assert(pcap);
-                if (pcap_compile( pcap, &filter->filter, filter->filterstring, 
+                if (pcap_compile( pcap, &filter->filter, filter->filterstring,
                                         1, 0)) {
                         trace_set_err(packet->trace,TRACE_ERR_BAD_FILTER,
-                                        "Unable to compile the filter \"%s\": %s", 
+                                        "Unable to compile the filter \"%s\": %s",
                                         filter->filterstring,
                                         pcap_geterr(pcap));
@@ -1275 +1278 @@
 
         if (linktype == TRACE_TYPE_NONDATA)
-                return 1;       
+                return 1;
 
         if (libtrace_to_pcap_dlt(linktype)==TRACE_DLT_ERROR) {
-                
+
                 /* If we cannot get a suitable DLT for the packet, it may
                  * be because the packet is encapsulated in a link type that
@@ -1284 +1287 @@
                  * popping off headers until we either can find a suitable
                  * link type or we can't do any more sensible decapsulation. */
-                
+
                 /* Copy the packet, as we don't want to trash the one we
                  * were passed in */
@@ -1292 +1295 @@
                 while (libtrace_to_pcap_dlt(linktype) == TRACE_DLT_ERROR) {
                         if (!demote_packet(packet_copy)) {
-                                trace_set_err(packet->trace, 
+                                trace_set_err(packet->trace,
                                                 TRACE_ERR_NO_CONVERSION,
                                                 "pcap does not support this format");
@@ -1304 +1307 @@
 
         }
-        
+
         linkptr = trace_get_packet_buffer(packet_copy,NULL,&clen);
         if (!linkptr) {
@@ -1313 +1316 @@
         }
 
-        /* We need to compile the filter now, because before we didn't know 
+        /* We need to compile the filter now, because before we didn't know
          * what the link type was
          */
@@ -1354 +1357 @@
  * @returns a signed value containing the direction flag, or -1 if this is not supported
  */
-DLLEXPORT libtrace_direction_t trace_set_direction(libtrace_packet_t *packet, 
-                libtrace_direction_t direction) 
+DLLEXPORT libtrace_direction_t trace_set_direction(libtrace_packet_t *packet,
+                libtrace_direction_t direction)
 {
         assert(packet);
@@ -1372 +1375 @@
  * for a special trace.
  */
-DLLEXPORT libtrace_direction_t trace_get_direction(const libtrace_packet_t *packet) 
+DLLEXPORT libtrace_direction_t trace_get_direction(const libtrace_packet_t *packet)
 {
         assert(packet);
@@ -1387 +1390 @@
 #define DYNAMIC(x) ((49152 < (x)) && ((x) < 65535))
 #define SERVER(x) ROOT_SERVER(x) || NONROOT_SERVER(x)
-#define CLIENT(x) ROOT_CLIENT(x) || NONROOT_CLIENT(x) 
+#define CLIENT(x) ROOT_CLIENT(x) || NONROOT_CLIENT(x)
 
 /* Attempt to deduce the 'server' port
@@ -1395 +1398 @@
  * @returns a hint as to which port is the server port
  */
-DLLEXPORT int8_t trace_get_server_port(UNUSED uint8_t protocol, 
-                uint16_t source, uint16_t dest) 
+DLLEXPORT int8_t trace_get_server_port(UNUSED uint8_t protocol,
+                uint16_t source, uint16_t dest)
 {
         /*
@@ -1409 +1412 @@
          * * flip a coin.
          */
-        
+
         /* equal */
         if (source == dest)
@@ -1455 +1458 @@
                 return USE_SOURCE;
         }
-        
+
         /* nonroot client */
         if (NONROOT_CLIENT(source) && NONROOT_CLIENT(dest)) {
-                if (source < dest) 
+                if (source < dest)
                         return USE_SOURCE;
                 return USE_DEST;
@@ -1478 +1481 @@
                 return USE_SOURCE;
         /*
-        if (SERVER(source) && CLIENT(dest)) 
+        if (SERVER(source) && CLIENT(dest))
                 return USE_SOURCE;
-        
-        if (SERVER(dest) && CLIENT(source)) 
+
+        if (SERVER(dest) && CLIENT(source))
                 return USE_DEST;
-        if (ROOT_SERVER(source) && !ROOT_SERVER(dest)) 
+        if (ROOT_SERVER(source) && !ROOT_SERVER(dest))
                 return USE_SOURCE;
-        if (ROOT_SERVER(dest) && !ROOT_SERVER(source)) 
+        if (ROOT_SERVER(dest) && !ROOT_SERVER(source))
                 return USE_DEST;
         */
@@ -1491 +1494 @@
         if (source < dest) {
                 return USE_SOURCE;
-        } 
+        }
         return USE_DEST;
-        
+
 }
 
@@ -1522 +1525 @@
  *
  * Returns a pointer to the URI data, but updates the format parameter to
- * point to a copy of the format component. 
+ * point to a copy of the format component.
  */
 
 DLLEXPORT const char * trace_parse_uri(const char *uri, char **format) {
         const char *uridata = 0;
-        
+
         if((uridata = strchr(uri,':')) == NULL) {
                 /* Badly formed URI - needs a : */
@@ -1544 +1547 @@
         /* Push uridata past the delimiter */
         uridata++;
-        
+
         return uridata;
 }
 
-enum base_format_t trace_get_format(libtrace_packet_t *packet) 
+enum base_format_t trace_get_format(libtrace_packet_t *packet)
 {
         assert(packet);
@@ -1554 +1557 @@
         return packet->trace->format->type;
 }
-        
+
 DLLEXPORT libtrace_err_t trace_get_err(libtrace_t *trace)
 {
@@ -1652 +1655 @@
                 }
                 if (trace->format->seek_seconds) {
-                        double seconds =  
+                        double seconds =
                                 (ts>>32) + ((ts & UINT_MAX)*1.0 / UINT_MAX);
                         return trace->format->seek_seconds(trace,seconds);
@@ -1676 +1679 @@
                 }
                 if (trace->format->seek_erf) {
-                        uint64_t timestamp = 
+                        uint64_t timestamp =
                                 ((uint64_t)((uint32_t)seconds) << 32) + \
                             (uint64_t)(( seconds - (uint32_t)seconds   ) * UINT_MAX);
@@ -1739 +1742 @@
 
 
-/* Creates a libtrace packet from scratch using the contents of the provided 
+/* Creates a libtrace packet from scratch using the contents of the provided
  * buffer as the packet payload.
  *
  * Unlike trace_prepare_packet(), the buffer should not contain any capture
- * format headers; instead this function will add the PCAP header to the 
+ * format headers; instead this function will add the PCAP header to the
  * packet record. This also means only PCAP packets can be constructed using
  * this function.
@@ -1765 +1768 @@
         /* We need a trace to attach the constructed packet to (and it needs
          * to be PCAP) */
-        if (NULL == deadtrace) 
+        if (NULL == deadtrace)
                 deadtrace=trace_create_dead("pcapfile");
 
@@ -1783 +1786 @@
 
         /* Now fill in the libtrace packet itself */
+        assert(deadtrace);
         packet->trace=deadtrace;
         size=len+sizeof(hdr);
+        if (size < LIBTRACE_PACKET_BUFSIZE)
+            size = LIBTRACE_PACKET_BUFSIZE;
         if (packet->buf_control==TRACE_CTRL_PACKET) {
-                packet->buffer=realloc(packet->buffer,size);
+            packet->buffer = realloc(packet->buffer, size);
         }
         else {
-                packet->buffer=malloc(size);
+                packet->buffer = malloc(size);
         }
         packet->buf_control=TRACE_CTRL_PACKET;
         packet->header=packet->buffer;
         packet->payload=(void*)((char*)packet->buffer+sizeof(hdr));
-        
-        /* Ugh, memcpy - sadly necessary */
-        memcpy(packet->header,&hdr,sizeof(hdr));
-        memcpy(packet->payload,data,(size_t)len);
+
+        /* Ugh, memmove - sadly necessary, also beware that we might be
+         * moving data around within this packet, so ordering is important.
+         */
+        memmove(packet->payload, data, (size_t)len);
+        memmove(packet->header, &hdr, sizeof(hdr));
         packet->type=pcap_linktype_to_rt(libtrace_to_pcap_linktype(linktype));
 
@@ -1865 +1873 @@
 
         /* Now, verify that the format has at least the minimum functionality.
-         * 
+         *
          * This #if can be changed to a 1 to output warnings about inconsistent
          * functions being provided by format modules.  This generally is very
          * noisy, as almost all modules don't implement one or more functions
-         * for various reasons.  This is very useful when checking a new 
+         * for various reasons.  This is very useful when checking a new
          * format module is sane.
-         */ 
+         */
 #if 0
         if (f->init_input) {
@@ -1885 +1893 @@
                 REQUIRE(get_framing_length);
                 REQUIRE(trace_event);
-                if (!f->get_erf_timestamp 
+                if (!f->get_erf_timestamp
                         && !f->get_seconds
                         && !f->get_timeval) {

tools/tracesplit/tracesplit.1

@@ -5 +5 @@
 .B tracesplit
 [ \fB-f \fRbpf | \fB--filter=\fRbpf]
+[ \fB-j \fRnumhdrs | \fB--jump=\fRnumhdrs]
 [ \fB-c \fRcount | \fB--count=\fRcount]
 [ \fB-b \fRbytes | \fB--bytes=\fRbytes]
@@ -20 +21 @@
 \fB\-f\fR bpf filter
 output only packets that match tcpdump style bpf filter
+
+.TP
+\fB\-j\fR numhdrs
+Strip headers before the numhdrs layer 3 header.  For example, \-j1 will strip
+off all the layer 2 headers, \-j2 will strip off all the l2 headers, the first
+l3 header, any transport headers, and return a trace that starts at the next
+l3 header.
 
 .TP

tools/tracesplit/tracesplit.c

@@ -25 +25 @@
 uint64_t filescreated = 0;
 uint16_t snaplen = 0;
+int jump=0;
 int verbose=0;
 int compress_level=-1;
@@ -56 +57 @@
         "-e --endtime=time      End at time\n"
         "-m --maxfiles=n        Create a maximum of n trace files\n"
+        "-j --jump=n            Jump to the nth IP header\n"
         "-H --libtrace-help     Print libtrace runtime documentation\n"
         "-S --snaplen           Snap packets at the specified length\n"
@@ -75 +77 @@
 
 
+static libtrace_packet_t *perform_jump(libtrace_packet_t *packet, int jump)
+{
+    uint16_t ethertype;
+    uint32_t remaining;
+    uint8_t ipproto;
+    void *offset = trace_get_layer3(packet, &ethertype, &remaining);
+    while(jump > 0 && offset) {
+        --jump;
+        switch (ethertype) {
+            case TRACE_ETHERTYPE_IP:
+                if (jump <= 0) {
+                    void *newpacket = trace_create_packet();
+                    trace_construct_packet(newpacket,
+                            TRACE_TYPE_NONE,
+                            offset,
+                            remaining);
+                    return newpacket;
+                }
+                offset = trace_get_payload_from_ip(offset,
+                        &ipproto,
+                        &remaining);
+                if (!offset)
+                    return NULL;
+                break;
+            case TRACE_ETHERTYPE_IPV6:
+                if (jump <= 0) {
+                    void *newpacket = trace_create_packet();
+                    trace_construct_packet(newpacket,
+                            TRACE_TYPE_NONE,
+                            offset,
+                            remaining);
+                    return newpacket;
+                }
+                offset = trace_get_payload_from_ip6(offset,
+                        &ipproto,
+                        &remaining);
+                if (!offset)
+                    return NULL;
+                break;
+            /* TODO: vlan, etc */
+            default:
+                return NULL;
+        }
+        if (!offset)
+            return false;
+        switch (ipproto) {
+            case TRACE_IPPROTO_IPV6:
+                ethertype = TRACE_ETHERTYPE_IPV6;
+                continue;
+            case TRACE_IPPROTO_IPIP:
+                ethertype = TRACE_ETHERTYPE_IP;
+                continue;
+            case TRACE_IPPROTO_GRE:
+                if (remaining < sizeof(libtrace_gre_t)) {
+                    return NULL;
+                }
+                ethertype = ((libtrace_gre_t *)offset)->ethertype;
+                offset = trace_get_payload_from_gre(offset, &remaining);
+                continue;
+            case TRACE_IPPROTO_UDP:
+                offset = trace_get_vxlan_from_udp(offset, &remaining);
+                if (!offset)
+                    return NULL;
+                offset = trace_get_payload_from_vxlan(offset, &remaining);
+                if (!offset)
+                    return NULL;
+                offset = trace_get_payload_from_layer2(offset,
+                        TRACE_TYPE_ETH,
+                        &ethertype,
+                        &remaining);
+                if (!offset)
+                    return NULL;
+                continue;
+        }
+        return NULL;
+    }
+    return NULL;
+}
+
+
 /* Return values:
  *  1 = continue reading packets
@@ -80 +162 @@
  *  -1 = stop reading packets, we've got an error
  */
-static int per_packet(libtrace_packet_t *packet) {
-
-        if (trace_get_link_type(packet) == ~0U) {
+static int per_packet(libtrace_packet_t **packet) {
+
+        if (trace_get_link_type(*packet) == -1) {
                 fprintf(stderr, "Halted due to being unable to determine linktype - input trace may be corrupt.\n");
                 return -1;
@@ -88 +170 @@
 
         if (snaplen>0) {
-                trace_set_capture_length(packet,snaplen);
-        }
-
-        if (trace_get_seconds(packet)<starttime) {
+                trace_set_capture_length(*packet,snaplen);
+        }
+
+        if (trace_get_seconds(*packet)<starttime) {
                 return 1;
         }
 
-        if (trace_get_seconds(packet)>endtime) {
+        if (trace_get_seconds(*packet)>endtime) {
                 return 0;
         }
 
         if (firsttime==0) {
-                time_t now = trace_get_seconds(packet);
+                time_t now = trace_get_seconds(*packet);
                 if (starttime != 0) {
                         firsttime=now-((now - starttime)%interval);
@@ -109 +191 @@
         }
 
-        if (output && trace_get_seconds(packet)>firsttime+interval) {
+        if (output && trace_get_seconds(*packet)>firsttime+interval) {
                 trace_destroy_output(output);
                 output=NULL;
@@ -121 +203 @@
 
         pktcount++;
-        totbytes+=trace_get_capture_length(packet);
+        totbytes+=trace_get_capture_length(*packet);
         if (output && totbytes-totbyteslast>=bytes) {
                 trace_destroy_output(output);
@@ -198 +280 @@
         }
 
-        /* Some traces we have are padded (usually with 0x00), so 
+        /* Some traces we have are padded (usually with 0x00), so
          * lets sort that out now and truncate them properly
          */
 
-        if (trace_get_capture_length(packet) 
-                        > trace_get_wire_length(packet)) {
-                trace_set_capture_length(packet,trace_get_wire_length(packet));
-        }
-
-        if (trace_write_packet(output,packet)==-1) {
+        if (trace_get_capture_length(*packet)
+                        > trace_get_wire_length(*packet)) {
+                trace_set_capture_length(*packet,
+                        trace_get_wire_length(*packet));
+        }
+
+        /* Support "jump"ping to the nth IP header. */
+        if (jump) {
+            /* Skip headers */
+            void *newpacket = perform_jump(*packet, jump);
+            if (newpacket) {
+                trace_destroy_packet(*packet);
+                *packet = newpacket;
+            }
+            else /* Skip packet */
+                return 1;
+        }
+
+        if (trace_write_packet(output, *packet)==-1) {
                 trace_perror_output(output,"write_packet");
                 return -1;
@@ -223 +318 @@
         struct libtrace_packet_t *packet = trace_create_packet();
         struct sigaction sigact;
-        int i;  
-        
+        int i;
+
         if (argc<2) {
                 usage(argv[0]);
@@ -240 +335 @@
                         { "endtime",       1, 0, 'e' },
                         { "interval",      1, 0, 'i' },
+                        { "jump",          1, 0, 'j' },
                         { "libtrace-help", 0, 0, 'H' },
                         { "maxfiles",      1, 0, 'm' },
@@ -249 +345 @@
                 };
 
-                int c=getopt_long(argc, argv, "f:c:b:s:e:i:m:S:Hvz:Z:",
+                int c=getopt_long(argc, argv, "j:f:c:b:s:e:i:m:S:Hvz:Z:",
                                 long_options, &option_index);
 
@@ -268 +364 @@
                         case 'i': interval=atoi(optarg);
                                   break;
+                        case 'j': jump=atoi(optarg);
+                                  break;
                         case 'm': maxfiles=atoi(optarg);
                                   break;
@@ -283 +381 @@
                                   if (compress_level<0 || compress_level>9) {
                                         usage(argv[0]);
-                                        exit(1);
+                                        exit(1);
                                   }
                                   break;
                         case 'Z':
                                   compress_type_str=optarg;
-                                  break;        
+                                  break;
                         default:
                                 fprintf(stderr,"Unknown option: %c\n",c);
@@ -321 +419 @@
                 compress_type = TRACE_OPTION_COMPRESSTYPE_NONE;
         } else {
-                fprintf(stderr, "Unknown compression type: %s\n", 
+                fprintf(stderr, "Unknown compression type: %s\n",
                         compress_type_str);
                 return 1;
@@ -348 +446 @@
 
 
-                input = trace_create(argv[i]);  
-                
+                input = trace_create(argv[i]);
+
                 if (trace_is_err(input)) {
                         trace_perror(input,"%s",argv[i]);
@@ -356 +454 @@
 
                 if (filter && trace_config(input, TRACE_OPTION_FILTER, filter) == 1) {
-                        trace_perror(input, "Configuring filter for %s", 
+                        trace_perror(input, "Configuring filter for %s",
                                         argv[i]);
                         return 1;
@@ -367 +465 @@
 
                 while (trace_read_packet(input,packet)>0) {
-                        if (per_packet(packet) < 1)
+                        if (per_packet(&packet) < 1)
                                 done = 1;
                         if (done)