Changeset 1f7f10c


Ignore:
Timestamp:
05/18/18 11:44:17 (2 years ago)
Author:
Shane Alcock <salcock@…>
Branches:
cachetimestamps, develop, master, rc-4.0.4, ringdecrementfix, ringperformance
Children:
a7e4f40
Parents:
6d17620 (diff), 9845e97 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Merge branch 'etsilive' into develop

Files:
5 added
18 edited

Legend:

Unmodified
Added
Removed
  • configure.in

    r62ee4ec rb663d33  
    215215        AC_CHECK_HEADER(openssl/evp.h, cryptofound=1, cryptofound=0)
    216216fi
     217AC_CHECK_LIB(tasn1, asn1_array2tree, asn1found=1, asn1found=0)
    217218
    218219# Check for libpcap
     
    458459AC_CHECK_LIB(numa, numa_node_to_cpus, have_numa=1, have_numa=0)
    459460
     461# Need libwandder for ETSI live decoding
     462AC_CHECK_LIB(wandder, init_wandder_decoder, have_wandder=1, have_wandder=0)
     463
    460464# Checks for various "optional" libraries
    461465AC_CHECK_LIB(pthread, pthread_create, have_pthread=1, have_pthread=0)
     
    487491        AC_DEFINE(HAVE_LIBNUMA, 0, [Set to 1 if libnuma is supported])
    488492        with_numa=no
     493fi
     494
     495if test "$have_wandder" = 1; then
     496        LIBTRACE_LIBS="$LIBTRACE_LIBS -lwandder"
     497        AC_DEFINE(HAVE_WANDDER, 1, [Set to 1 if libwandder is available])
     498        wandder_avail=yes
     499else
     500        AC_DEFINE(HAVE_WANDDER, 0, [Set to 1 if libwandder is available])
     501        wandder_avail=no
    489502fi
    490503
     
    614627AM_CONDITIONAL([HAVE_DAG], [test "$libtrace_dag" = true])
    615628AM_CONDITIONAL([HAVE_DPDK], [test "$libtrace_dpdk" = true])
     629AM_CONDITIONAL([HAVE_WANDDER], [test "x$wandder_avail" = "xyes"])
    616630AM_CONDITIONAL([DAG2_4], [test "$libtrace_dag_version" = 24])
    617631AM_CONDITIONAL([DAG2_5], [test "$libtrace_dag_version" = 25])
     
    695709fi
    696710reportopt "Compiled with LLVM BPF JIT support" $JIT
     711reportopt "Compiled with live ETSI LI support (requires libwandder)" $wandder_avail
    697712reportopt "Building man pages/documentation" $libtrace_doxygen
    698713reportopt "Building tracetop (requires libncurses)" $with_ncurses
  • lib/Makefile.am

    r49969f2 rb663d33  
    77        data-struct/deque.h data-struct/linked_list.h \
    88        data-struct/buckets.h data-struct/sliding_window.h \
    9         data-struct/message_queue.h hash_toeplitz.h
     9        data-struct/message_queue.h hash_toeplitz.h \
     10        data-struct/simple_circular_buffer.h
    1011
    1112AM_CFLAGS=@LIBCFLAGS@ @CFLAG_VISIBILITY@ -pthread
     
    5152endif
    5253
     54if HAVE_WANDDER
     55ETSISOURCES=format_etsilive.c
     56else
     57ETSISOURCES=
     58endif
     59
    5360libtrace_la_SOURCES = trace.c trace_parallel.c common.h \
    5461                format_erf.c format_pcap.c format_legacy.c \
     
    6370                protocols_application.c \
    6471                $(DAGSOURCE) format_erf.h format_ndag.c format_ndag.h \
    65                 $(BPFJITSOURCE) \
     72                $(BPFJITSOURCE) $(ETSISOURCES) \
    6673                libtrace_arphrd.h \
    6774                data-struct/ring_buffer.c data-struct/vector.c \
     
    6976                data-struct/sliding_window.c data-struct/object_cache.c \
    7077                data-struct/linked_list.c hash_toeplitz.c combiner_ordered.c \
    71                 data-struct/buckets.c \
     78                data-struct/buckets.c data-struct/simple_circular_buffer.c \
    7279                combiner_sorted.c combiner_unordered.c \
    7380                pthread_spinlock.c pthread_spinlock.h
  • lib/format_dag25.c

    ra857389 rcc9c9de  
    608608                return -1;
    609609        case TRACE_OPTION_EVENT_REALTIME:
     610        case TRACE_OPTION_REPLAY_SPEEDUP:
    610611                /* Live capture is always going to be realtime */
    611612                return -1;
  • lib/format_helper.c

    r633339d rde060a8  
    127127        double ts;
    128128        double now;
     129        double sincebeginnow = 0;
     130        double sincebegintrace = 0;
     131
    129132#ifdef WIN32
    130133        struct __timeb64 tstruct;
     
    182185
    183186       
    184         if (fabs(trace->event.tdelta)>1e-9) {
    185                 /* Subtract the tdelta from the walltime to get a suitable
     187        if (fabs(trace->event.first_ts)>1e-9) {
     188                /* Subtract the tdelta from the starting times to get a suitable
    186189                 * "relative" time */
    187                 now -= trace->event.tdelta;
     190                sincebeginnow = (now - trace->event.first_now);
     191                sincebegintrace = (ts - trace->event.first_ts);
    188192
    189193                /* If the trace timestamp is still in the future, return a
    190194                 * SLEEP event, otherwise return the packet */
    191                 if (ts > now) {
    192                         event.seconds = ts -
    193                                 trace->event.trace_last_ts;
    194                         trace->event.trace_last_ts = ts;
     195                if (sincebeginnow <= sincebegintrace / (double)trace->replayspeedup) {
     196                        event.seconds = ((sincebegintrace / (double)trace->replayspeedup) - sincebeginnow);
    195197                        event.type = TRACE_EVENT_SLEEP;
    196198                        trace->event.waiting = true;
     
    204206                 * trace file.
    205207                 */
    206                 trace->event.tdelta = now - ts;
     208                trace->event.first_now = (double)now;
     209                trace->event.first_ts = (double)ts;
    207210        }
    208211
     
    219222        event.type = TRACE_EVENT_PACKET;
    220223
    221         trace->event.trace_last_ts = ts;
    222224        trace->event.waiting = false;
    223225
  • lib/format_linux_common.c

    rd47ca18 rcc9c9de  
    164164                        /* Live captures are always going to be in trace time */
    165165                        break;
     166                case TRACE_OPTION_REPLAY_SPEEDUP:
     167                        break;
    166168                /* Avoid default: so that future options will cause a warning
    167169                 * here to remind us to implement it, or flag it as
  • lib/format_linux_ring.c

    rdb84bb2 r1060b6d  
    461461        struct pollfd pollset[2];
    462462
    463         ring_release_frame(libtrace, packet);
    464        
    465463        packet->buf_control = TRACE_CTRL_EXTERNAL;
    466464        packet->type = TRACE_RT_DATA_LINUX_RING;
  • lib/format_pcapfile.c

    r5a70a80 rcc9c9de  
    257257                case TRACE_OPTION_FILTER:
    258258                case TRACE_OPTION_HASHER:
     259                case TRACE_OPTION_REPLAY_SPEEDUP:
    259260                        /* All these are either unsupported or handled
    260261                         * by trace_config */
  • lib/format_pcapng.c

    r2c457ec r9845e97  
    260260                case TRACE_OPTION_FILTER:
    261261                case TRACE_OPTION_HASHER:
     262                case TRACE_OPTION_REPLAY_SPEEDUP:
    262263                        break;
    263264        }
     
    11941195
    11951196        ts.tv_sec = (timestamp / interface->tsresol);
    1196         ts.tv_nsec = (uint64_t)(timestamp - (ts.tv_sec * interface->tsresol)) / interface->tsresol * 1000000000;
     1197        ts.tv_nsec = (uint64_t)(timestamp - (ts.tv_sec * interface->tsresol))
     1198                        / ((double)interface->tsresol) * 1000000000;
    11971199
    11981200        return ts;
  • lib/libtrace.h.in

    rc1205bd rcc9c9de  
    370370       TRACE_TYPE_NONDATA = 19,         /**< Not a data packet */
    371371       TRACE_TYPE_OPENBSD_LOOP = 20,    /**< OpenBSD loopback */
    372        TRACE_TYPE_ERF_META = 21 /**< ERF Provenance metadata record */
     372       TRACE_TYPE_ERF_META = 21, /**< ERF Provenance metadata record */
     373       TRACE_TYPE_ETSILI = 22   /**< ETSI Lawful Intercept */
    373374} libtrace_linktype_t;
    374375
     
    398399        TRACE_FORMAT_NDAG       =19,    /**< DAG multicast over a network */
    399400        TRACE_FORMAT_DPDK_NDAG       =20,    /**< DAG multicast over a network, received via DPDK */
     401        TRACE_FORMAT_ETSILIVE     =21,  /**< ETSI LI over a network */
    400402};
    401403
     
    452454    /** RT is encapsulating a Intel DPDK capture record */
    453455        TRACE_RT_DATA_DPDK=TRACE_RT_DATA_SIMPLE+TRACE_FORMAT_DPDK,
     456
     457        TRACE_RT_DATA_ETSILI = TRACE_RT_DATA_SIMPLE + TRACE_FORMAT_ETSILIVE,
    454458
    455459        /** As PCAP does not store the linktype with the packet, we need to
     
    13161320        /** The hasher function for a parallel libtrace. It is recommended to
    13171321         * access this option via trace_set_hasher(). */
    1318         TRACE_OPTION_HASHER
     1322        TRACE_OPTION_HASHER,
     1323
     1324        /** Speed up trace file replays (via trace_event()) by this factor */
     1325        TRACE_OPTION_REPLAY_SPEEDUP,
    13191326} trace_option_t;
    13201327
  • lib/libtrace_int.h

    re375e0f rcc9c9de  
    152152//#define RP_BUFSIZE 65536U
    153153
     154#define LIBTRACE_MAX_REPLAY_SPEEDUP 1000
     155
    154156/** Data about the most recent event from a trace file */
    155157struct libtrace_event_status_t {
    156158        /** A libtrace packet to store the packet when a PACKET event occurs */
    157159        libtrace_packet_t *packet;
    158         /** Time between the timestamp for the current packet and the current
    159          * walltime */
    160         double tdelta;
    161         /** The timestamp of the previous PACKET event */
    162         double trace_last_ts;
     160
     161        /* The walltime when we processed the first packet from the trace */
     162        double first_now;
     163
     164        /* The tracetime of the first packet in the trace */
     165        double first_ts;
     166
    163167        /** The size of the current PACKET event */
    164168        int psize;
     
    313317         * used only if the capture format does not support snapping natively */
    314318        size_t snaplen;                 
     319        /** Speed up the packet rate when using trace_event() to process trace
     320         * files by this factor. */
     321        int replayspeedup;
    315322        /** Count of the number of packets returned to the libtrace user */
    316323        uint64_t accepted_packets;
     
    12421249/** Constructor for the network DAG format module */
    12431250void ndag_constructor(void);
     1251/** Constructor for the live ETSI over TCP format module */
     1252void etsilive_constructor(void);
    12441253#ifdef HAVE_BPF
    12451254/** Constructor for the BPF format module */
  • lib/linktypes.c

    ra857389 rb663d33  
    109109                case TRACE_TYPE_METADATA:
    110110                case TRACE_TYPE_NONDATA:
     111                case TRACE_TYPE_ETSILI:
    111112                        break;
    112113                case TRACE_TYPE_UNKNOWN:
     
    211212                case TRACE_TYPE_NONDATA:
    212213                case TRACE_TYPE_OPENBSD_LOOP:
     214                case TRACE_TYPE_ETSILI:
    213215                case TRACE_TYPE_UNKNOWN:
    214216                        break;
  • lib/protocols_l2.c

    ra857389 rfa555ec  
    483483                case TRACE_TYPE_PFLOG:
    484484                case TRACE_TYPE_ERF_META:
     485                case TRACE_TYPE_ETSILI:
    485486                        break;
    486487                case TRACE_TYPE_UNKNOWN:
     
    519520                                case TRACE_TYPE_PFLOG:
    520521                                case TRACE_TYPE_ERF_META:
     522                                case TRACE_TYPE_ETSILI:
    521523                                        break;
    522524                                case TRACE_TYPE_UNKNOWN:
     
    574576                case TRACE_TYPE_PFLOG:
    575577                case TRACE_TYPE_LINUX_SLL:
     578                case TRACE_TYPE_ETSILI:
    576579                        return NULL;
    577580
     
    696699                case TRACE_TYPE_80211_PRISM:
    697700                case TRACE_TYPE_80211_RADIO:
     701                case TRACE_TYPE_ETSILI:
    698702                        assert(!"Metadata headers should already be skipped");
    699703                        break;
     
    746750                case TRACE_TYPE_80211_PRISM:
    747751                case TRACE_TYPE_80211_RADIO:
     752                case TRACE_TYPE_ETSILI:
    748753                        assert(!"Metadata headers should already be skipped");
    749754                        break;
  • lib/protocols_pktmeta.c

    ra857389 r691b182  
    2828#include "protocols.h"
    2929#include <assert.h>
     30
     31#ifdef HAVE_WANDDER
     32#include <libwandder_etsili.h>
     33#endif
    3034
    3135/* This file contains all the protocol decoding functions for the meta-data
     
    121125}
    122126
     127static void *trace_get_payload_from_etsili(const void *link,
     128                libtrace_linktype_t *type, uint32_t *remaining) {
     129
     130#ifdef HAVE_WANDDER
     131        wandder_etsispec_t *dec;
     132        uint8_t *ccptr;
     133
     134        /* XXX Bit annoying to be creating and freeing this every time */
     135        dec = wandder_create_etsili_decoder();
     136        wandder_attach_etsili_buffer(dec, (uint8_t *)link, *remaining, false);
     137        ccptr = wandder_etsili_get_cc_contents(dec, remaining, NULL, 0);
     138        /* Assuming all CCs are IP for now */
     139        *type = TRACE_TYPE_NONE;
     140        wandder_free_etsili_decoder(dec);
     141        return ccptr;
     142
     143#else
     144        *remaining = NULL;
     145        return NULL;
     146#endif
     147
     148}
     149
    123150DLLEXPORT void *trace_get_packet_meta(const libtrace_packet_t *packet,
    124151                libtrace_linktype_t *linktype,
     
    139166                case TRACE_TYPE_80211_PRISM:
    140167                case TRACE_TYPE_ERF_META:
     168                case TRACE_TYPE_ETSILI:
    141169                        return pktbuf;
    142170                /* Non metadata packets */
     
    199227                                        linktype, remaining);
    200228                        return nexthdr;
     229                case TRACE_TYPE_ETSILI:
     230                        nexthdr = trace_get_payload_from_etsili(meta,
     231                                        linktype, remaining);
     232                        return nexthdr;
     233
    201234                case TRACE_TYPE_HDLC_POS:
    202235                case TRACE_TYPE_ETH:
  • lib/trace.c

    rdc27de3 rcc9c9de  
    148148                rt_constructor();
    149149                ndag_constructor();
     150#ifdef HAVE_WANDDER
     151                etsilive_constructor();
     152#endif
    150153#ifdef HAVE_DAG
    151154                dag_constructor();
     
    248251        libtrace->format=NULL;
    249252
    250         libtrace->event.tdelta = 0.0;
    251253        libtrace->event.packet = NULL;
    252254        libtrace->event.psize = 0;
    253         libtrace->event.trace_last_ts = 0.0;
     255        libtrace->event.first_ts = 0.0;
     256        libtrace->event.first_now = 0.0;
    254257        libtrace->event.waiting = false;
    255258        libtrace->filter = NULL;
    256259        libtrace->snaplen = 0;
     260        libtrace->replayspeedup = 1;
    257261        libtrace->started=false;
    258262        libtrace->uridata = NULL;
     
    375379        libtrace->format=NULL;
    376380
    377         libtrace->event.tdelta = 0.0;
    378381        libtrace->event.packet = NULL;
    379382        libtrace->event.psize = 0;
    380         libtrace->event.trace_last_ts = 0.0;
     383        libtrace->event.first_ts = 0;
     384        libtrace->event.first_now = 0;
    381385        libtrace->filter = NULL;
    382386        libtrace->snaplen = 0;
     
    592596         * deal with some options itself, so give that a go */
    593597        switch(option) {
     598                case TRACE_OPTION_REPLAY_SPEEDUP:
     599                        /* Clear the error if there was one */
     600                        if (trace_is_err(libtrace)) {
     601                                trace_get_err(libtrace);
     602                        }
     603                        if (*(int*)value<1
     604                                || *(int*)value>LIBTRACE_MAX_REPLAY_SPEEDUP) {
     605                                trace_set_err(libtrace,TRACE_ERR_BAD_STATE,
     606                                        "Invalid replay speed");
     607                        }
     608                        libtrace->replayspeedup=*(int*)value;
     609                        return 0;
     610
    594611                case TRACE_OPTION_SNAPLEN:
    595612                        /* Clear the error if there was one */
     
    15431560                                trace_set_err(packet->trace,
    15441561                                                TRACE_ERR_NO_CONVERSION,
    1545                                                 "pcap does not support this format");
     1562                                                "pcap does not support this linktype so cannot apply BPF filters");
    15461563                                if (free_packet_needed) {
    15471564                                        trace_destroy_packet(packet_copy);
  • libpacketdump/Makefile.am

    r3a333e2 ra82a9e8  
    6060# 17: PPP
    6161TXT_PROTOCOLS+=link_17.protocol
     62
     63# 22: ETSI LI
     64BIN_PROTOCOLS+=link_22.la
    6265
    6366# Decoders for various ethertypes (in decimal)
     
    135138link_11_la_LDFLAGS=$(modflags)
    136139link_15_la_LDFLAGS=$(modflags)
     140link_22_la_LDFLAGS=$(modflags)
    137141eth_0_la_LDFLAGS=$(modflags)
    138142eth_2048_la_LDFLAGS=$(modflags)
  • tools/tracepktdump/tracepktdump.cc

    ree6e802 rf35a853  
    113113                                /* Ignore RT messages */
    114114                                continue;
     115                        if (trace_is_err(trace)) {
     116                                break;
     117                        }
    115118                        trace_dump_packet(packet);
    116119
  • tools/tracereplay/tracereplay.1

    rd6dc0f6 r1e0a804  
    44.SH SYNOPSIS
    55.B tracereplay
    6 [\-b | \-\^\-broadcast] [-s \-\^\-snaplength [ snaplength] ] 
    7 [\-f | \-\^\-filter [ filter string ] ]
     6[\-b | \-\^\-broadcast] [-s \-\^\-snaplength [ snaplength] ]
     7[\-f | \-\^\-filter [ filter string ] ] [\-X | \-\^\-speedup [ factor] ]
    88inputuri outputuri
    99.SH DESCRPTION
     
    2323.TP
    2424.PD 0
    25 .BI \-s [snaplength]
     25.BI \-s " snaplength"
    2626.TP
    2727.PD
    28 .BI \-\^\-snaplength [snaplength]
     28.BI \-\^\-snaplength " snaplength"
    2929Truncate the packets to the length specified. This only makes a difference
    3030if the inputuri is a capture device. The value should be > 0. Inappropriate
     
    3333.TP
    3434.PD 0
    35 .BI \-f [ filter string ]
     35.BI \-f " filterstring"
    3636.TP
    3737.PD
    38 .BI \-\^\-filter [ filter ]
     38.BI \-\^\-filter " filterstring"
    3939Apply a filter to the inputuri.
     40
     41.TP
     42.PD 0
     43.BI \-X " factor"
     44.TP
     45.PD
     46.BI \-\^\-speedup " factor"
     47Decrease the gaps between packets by the specified factor. This will accelerate
     48the rate at which the replay is performed. By default, the factor is 1 (i.e.
     49no acceleration).
    4050
    4151.SH LINKS
  • tools/tracereplay/tracereplay.c

    r8e11beb r1e0a804  
    9898        if (IS_LIBTRACE_META_PACKET(packet))
    9999                return NULL;
     100        if (trace_get_wire_length(packet) == 0)
     101                return NULL;
    100102
    101103        pkt_buffer = trace_get_packet_buffer(packet,&linktype,&remaining);
     
    194196        fprintf(stderr, " --broadcast\n");
    195197        fprintf(stderr, "\t\tSend ethernet frames to broadcast address\n");
     198        fprintf(stderr, " -X\n");
     199        fprintf(stderr, " --speedup\n");
     200        fprintf(stderr, "\t\tSpeed up replay by a factor of <speedup>\n");
    196201
    197202}
     
    207212        libtrace_packet_t * new;
    208213        int snaplen = 0;
     214        int speedup = 1;
    209215
    210216
     
    216222                        { "snaplen",    1, 0, 's'},
    217223                        { "broadcast",  0, 0, 'b'},
     224                        { "speedup",    1, 0, 'X'},
    218225                        { NULL,         0, 0, 0}
    219226                };
    220227
    221                 int c = getopt_long(argc, argv, "bhs:f:",
     228                int c = getopt_long(argc, argv, "bhs:f:X:",
    222229                                long_options, &option_index);
    223230
     
    232239                                snaplen = atoi(optarg);
    233240                                break;
    234 
     241                        case 'X':
     242                                speedup = atoi(optarg);
     243                                break;
    235244                        case 'b':
    236245                                broadcast = 1;
     
    257266        }
    258267
     268        if (speedup < 1) {
     269                speedup = 1;
     270        }
     271
    259272        uri = strdup(argv[optind]);
    260273
     
    279292                }
    280293        }
     294
     295        if (trace_config(trace, TRACE_OPTION_REPLAY_SPEEDUP, &speedup)) {
     296                trace_perror(trace, "error setting replay speedup factor");
     297                return 1;
     298        }
    281299
    282300        /* Starting the trace */
Note: See TracChangeset for help on using the changeset viewer.