Changeset 41148f2


Ignore:
Timestamp:
07/22/14 13:00:46 (6 years ago)
Author:
Richard Sanger <rsangerarj@…>
Branches:
4.0.1-hotfixes, cachetimestamps, develop, dpdk-ndag, etsilive, libtrace4, master, ndag_format, pfring, rc-4.0.1, rc-4.0.2, rc-4.0.3, rc-4.0.4, ringdecrementfix, ringperformance, ringtimestampfixes
Children:
a49a9eb
Parents:
c99b1e5 (diff), 17c5749 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

Merge branch 'FixingDPDK' into develop

Files:
11 added
32 edited

Legend:

Unmodified
Added
Removed
  • README

    rf1015ad r17c5749  
    44Libtrace functions might not function correctly breaking the supplied tools.
    55
    6 libtrace 3.0.19
     6libtrace 3.0.20
    77
    88---------------------------------------------------------------------------
  • configure.in

    rabf01b6 r457bf45  
    44# and in the README
    55
    6 AC_INIT([libtrace],[3.0.19],[contact@wand.net.nz],[libtrace])
     6AC_INIT([libtrace],[3.0.20],[contact@wand.net.nz],[libtrace])
    77
    88LIBTRACE_MAJOR=3
    99LIBTRACE_MID=0
    10 LIBTRACE_MINOR=19
     10LIBTRACE_MINOR=20
    1111
    1212# OpenSolaris hides libraries like libncurses in /usr/gnu/lib, which is not
     
    580580)
    581581
     582AC_ARG_WITH([lzma],
     583        AC_HELP_STRING([--with-lzma], [build with support for lzma compressed files]))
     584
     585AS_IF([test "x$with_lzma" != "xno"],
     586        [
     587        AC_CHECK_HEADER(lzma.h, have_lzma=yes, have_lzma=no)
     588        ], [have_lzma=no])
     589
     590AS_IF([test "x$have_lzma" = "xyes"], [
     591        if test "$ac_cv_lib_lzma_code" != "none required"; then
     592                LIBWANDIO_LIBS="$LIBWANDIO_LIBS -llzma"
     593        fi
     594        AC_DEFINE(HAVE_LIBLZMA, 1, "Compiled with lzma support")
     595        with_lzma=yes],
     596
     597       
     598        [AS_IF([test "x$with_lzma" = "xyes"],
     599                [AC_MSG_ERROR([lzma requested but not found])])
     600        AC_DEFINE(HAVE_LIBLZMA, 0, "Compiled with lzma support")
     601        with_lzma=no]
     602)
    582603
    583604# Define automake conditionals for use in our Makefile.am files
     
    594615AM_CONDITIONAL([HAVE_ZLIB], [test "x$with_zlib" != "xno"])
    595616AM_CONDITIONAL([HAVE_LZO], [ test "x$with_lzo" != "xno"])
     617AM_CONDITIONAL([HAVE_LZMA], [ test "x$with_lzma" != "xno"])
    596618
    597619# Check for miscellaneous programs
     
    648670reportopt "Compiled with compressed trace (bz2) support" $with_bzip2
    649671reportopt "Compiled with compressed trace (lzo write only) support" $with_lzo
     672reportopt "Compiled with compressed trace (lzma) support" $with_lzma
    650673if test x"$libtrace_dag" = xtrue; then
    651674        if test "$libtrace_dag_version" = 24; then
  • lib/Makefile.am

    re3a639a rdafe86a  
    3333export RTE_SDK=@RTE_SDK@
    3434export RTE_TARGET=@RTE_TARGET@
     35export SAVED_CFLAGS:=$(CFLAGS)
     36export SAVED_CXXFLAGS:=$(CXXFLAGS)
    3537include $(RTE_SDK)/mk/rte.vars.mk
    3638# We need to add -Wl before the linker otherwise this breaks our build
     
    3840export DPDK_LIBTRACE_MK=dpdk_libtrace.mk
    3941include $(DPDK_LIBTRACE_MK)
     42export CFLAGS += $(SAVED_CFLAGS)
     43export CXXFLAGS += $(SAVED_CXXFLAGS)
    4044endif
    4145
  • lib/format_dpdk.c

    rb13b939 r50ce607  
    4141 */
    4242
     43#define _GNU_SOURCE
     44
    4345#include "config.h"
    4446#include "libtrace.h"
     
    7678#include <rte_lcore.h>
    7779#include <rte_per_lcore.h>
     80#include <pthread.h>
    7881
    7982/* The default size of memory buffers to use - This is the max size of standard
     
    111114#define MBUF_PKTDATA(x) ((char *) x + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
    112115#define FORMAT(x) ((struct dpdk_format_data_t*)(x->format_data))
     116#define PERPKT_FORMAT(x) ((struct dpdk_per_lcore_t*)(x->format_data))
     117
    113118#define TV_TO_NS(tv) ((uint64_t) tv.tv_sec*1000000000ull + \
    114119                        (uint64_t) tv.tv_usec*1000ull)
     
    182187};
    183188
    184 struct per_lcore_t
     189struct dpdk_per_lcore_t
    185190{
    186191        // TODO move time stamp stuff here
    187192        uint16_t queue_id;
    188193        uint8_t port;
    189         uint8_t enabled;
    190194};
    191195
     
    213217    uint32_t wrap_count; /* Number of times the NIC clock has wrapped around completely */
    214218#endif
    215         // DPDK normally seems to have a limit of
    216         struct per_lcore_t per_lcore[RTE_MAX_LCORE];
     219        // DPDK normally seems to have a limit of 8 queues for a given card
     220        struct dpdk_per_lcore_t per_lcore[RTE_MAX_LCORE];
    217221};
    218222
     
    351355    if (global_config != NULL) {
    352356        int i;
    353         printf("Intel DPDK setup\n"
     357        fprintf(stderr, "Intel DPDK setup\n"
    354358               "---Version      : %"PRIu32"\n"
    355359               "---Magic        : %"PRIu32"\n"
     
    360364       
    361365        for (i = 0 ; i < nb_cpu; i++) {
    362             printf("   ---Core %d : %s\n", i,
     366            fprintf(stderr, "   ---Core %d : %s\n", i,
    363367                   global_config->lcore_role[i] == ROLE_RTE ? "on" : "off");
    364368        }
     
    381385                proc_type = "something worse than invalid!!";
    382386        }
    383         printf("---Process Type : %s\n", proc_type);
    384     }
    385    
    386 }
    387 #endif
     387        fprintf(stderr, "---Process Type : %s\n", proc_type);
     388    }
     389   
     390}
     391#endif
     392
     393/**
     394 * Expects to be called from the master lcore and moves it to the given dpdk id
     395 * @param core (zero indexed) If core is on the physical system affinity is bound otherwise
     396 *               affinity is set to all cores. Must be less than RTE_MAX_LCORE
     397 *               and not already in use.
     398 * @return 0 is successful otherwise -1 on error.
     399 */
     400static inline int dpdk_move_master_lcore(size_t core) {
     401    struct rte_config *cfg = rte_eal_get_configuration();
     402    cpu_set_t cpuset;
     403    int i;
     404
     405    assert (core < RTE_MAX_LCORE);
     406    assert (rte_get_master_lcore() == rte_lcore_id());
     407
     408    if (core == rte_lcore_id())
     409        return 0;
     410
     411    // Make sure we are not overwriting someone else
     412    assert(!rte_lcore_is_enabled(core));
     413
     414    // Move the core
     415    cfg->lcore_role[rte_lcore_id()] = ROLE_OFF;
     416    cfg->lcore_role[core] = ROLE_RTE;
     417    lcore_config[core].thread_id = lcore_config[rte_lcore_id()].thread_id;
     418    rte_eal_get_configuration()->master_lcore = core;
     419    RTE_PER_LCORE(_lcore_id) = core;
     420
     421    // Now change the affinity
     422    CPU_ZERO(&cpuset);
     423
     424    if (lcore_config[core].detected) {
     425        CPU_SET(core, &cpuset);
     426    } else {
     427        for (i = 0; i < RTE_MAX_LCORE; ++i) {
     428            if (lcore_config[i].detected)
     429                CPU_SET(i, &cpuset);
     430        }
     431    }
     432
     433    i = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
     434    if (i != 0) {
     435        // TODO proper libtrace style error here!!
     436        fprintf(stderr, "pthread_setaffinity_np failed\n");
     437        return -1;
     438    }
     439    return 0;
     440}
     441
    388442
    389443static inline int dpdk_init_enviroment(char * uridata, struct dpdk_format_data_t * format_data,
     
    392446    struct rte_pci_addr use_addr; /* The only address that we don't blacklist */   
    393447    char cpu_number[10] = {0}; /* The CPU mask we want to bind to */
     448    char mem_map[20] = {0}; /* The memory name */
    394449    long nb_cpu; /* The number of CPUs in the system */
    395450    long my_cpu; /* The CPU number we want to bind to */
     451    int i;
     452    struct rte_config *cfg = rte_eal_get_configuration();
    396453   
    397454#if DEBUG
     
    401458#endif
    402459    /* Using proc-type auto allows this to be either primary or secondary
    403      * Secondary allows two intances of libtrace to be used on different
     460     * Secondary allows two instances of libtrace to be used on different
    404461     * ports. However current version of DPDK doesn't support this on the
    405      * same card (My understanding is this should work with two seperate
     462     * same card (My understanding is this should work with two separate
    406463     * cards).
     464     *
     465     * Using unique file prefixes mean separate memory is used, unlinking
     466     * the two processes. However be careful we still cannot access a
     467     * port that already in use.
    407468     */
    408     char* argv[] = {"libtrace", "-c", NULL, "-n", "1", "--proc-type", "auto", NULL};
     469    char* argv[] = {"libtrace", "-c", cpu_number, "-n", "1", "--proc-type", "auto",
     470                "--file-prefix", mem_map, "-m", "256", NULL};
    409471    int argc = sizeof(argv) / sizeof(argv[0]) - 1;
    410472   
    411     /* This initilises the Enviroment Abstraction Layer (EAL)
     473    /* This initialises the Environment Abstraction Layer (EAL)
    412474     * If we had slave workers these are put into WAITING state
    413475     *
     
    451513    }
    452514
    453     /* Make our mask */ //  0x1 << (my_cpu - 1)
    454     snprintf(cpu_number, sizeof(cpu_number), "%x", 0x3);
    455     argv[2] = cpu_number;
    456 
     515    /* Make our mask with all cores turned on this is so that DPDK to gets CPU
     516       info older versions */
     517    snprintf(cpu_number, sizeof(cpu_number), "%x", ~(UINT32_MAX<<MIN(31, nb_cpu)));
     518    //snprintf(cpu_number, sizeof(cpu_number), "%x", 0x1 << (my_cpu - 1));
     519
     520        /* Give this a name */
     521        snprintf(mem_map, sizeof(mem_map), "libtrace-%d", (int) getpid());
    457522    /* rte_eal_init it makes a call to getopt so we need to reset the
    458523     * global optind variable of getopt otherwise this fails */
     
    463528        return -1;
    464529    }
     530
     531    // These are still running but will never do anything with DPDK v1.7 we
     532    // should remove this XXX in the future
     533    for(i = 0; i < RTE_MAX_LCORE; ++i) {
     534        if (rte_lcore_is_enabled(i) && i != rte_get_master_lcore()) {
     535            cfg->lcore_role[i] = ROLE_OFF;
     536            cfg->lcore_count--;
     537        }
     538    }
     539    // Only the master should be running
     540    assert(cfg->lcore_count == 1);
     541
     542    dpdk_move_master_lcore(my_cpu-1);
     543
    465544#if DEBUG
    466545    dump_configuration();
     
    505584    char err[500];
    506585    err[0] = 0;
    507     int i;
    508586   
    509587    libtrace->format_data = (struct dpdk_format_data_t *)
     
    525603    FORMAT(libtrace)->wrap_count = 0;
    526604#endif
    527         for (i = 0;i < RTE_MAX_LCORE; i++) {
    528                 // Disabled by default
    529                 FORMAT(libtrace)->per_lcore[i].enabled = 0;
    530         }
    531605       
    532606    if (dpdk_init_enviroment(libtrace->uridata, FORMAT(libtrace), err, sizeof(err)) != 0) {
     
    591665                                        return -1;
    592666                        }
     667        break;
    593668        }
    594669        return -1;
     
    688763static const struct rte_eth_txconf tx_conf = {
    689764        .tx_thresh = {
    690                 .pthresh = 36,/* TX_PTHRESH prefetch */
    691                 .hthresh = 0,/* TX_HTHRESH host */
    692                 .wthresh = 4,/* TX_WTHRESH writeback */
     765        /**
     766         * TX_PTHRESH prefetch
     767         * Set on the NIC, if the number of unprocessed descriptors to queued on
     768         * the card fall below this try grab at least hthresh more unprocessed
     769         * descriptors.
     770         */
     771                .pthresh = 36,
     772
     773        /* TX_HTHRESH host
     774         * Set on the NIC, the batch size to prefetch unprocessed tx descriptors.
     775         */
     776                .hthresh = 0,
     777       
     778        /* TX_WTHRESH writeback
     779         * Set on the NIC, the number of sent descriptors before writing back
     780         * status to confirm the transmission. This is done more efficiently as
     781         * a bulk DMA-transfer rather than writing one at a time.
     782         * Similar to tx_free_thresh however this is applied to the NIC, where
     783         * as tx_free_thresh is when DPDK will check these. This is extended
     784         * upon by tx_rs_thresh (10Gbit cards) which doesn't write all
     785         * descriptors rather only every n'th item, reducing DMA memory bandwidth.
     786         */
     787                .wthresh = 4,
    693788        },
    694         .tx_free_thresh = 0, /* Use PMD default values */
    695         .tx_rs_thresh = 0, /* Use PMD default values */
     789
     790    /* Used internally by DPDK rather than passed to the NIC. The number of
     791     * packet descriptors to send before checking for any responses written
     792     * back (to confirm the transmission). Default = 32 if set to 0)
     793     */
     794        .tx_free_thresh = 0,
     795
     796    /* This is the Report Status threshold, used by 10Gbit cards,
     797     * This signals the card to only write back status (such as
     798     * transmission successful) after this minimum number of transmit
     799     * descriptors are seen. The default is 32 (if set to 0) however if set
     800     * to greater than 1 TX wthresh must be set to zero, because this is kindof
     801     * a replacement. See the dpdk programmers guide for more restrictions.
     802     */
     803        .tx_rs_thresh = 1,
    696804};
    697805
     
    737845         */
    738846#if DEBUG
    739     printf("Creating mempool named %s\n", format_data->mempool_name);
     847    fprintf(stderr, "Creating mempool named %s\n", format_data->mempool_name);
    740848#endif
    741849        format_data->pktmbuf_pool =
     
    776884        return -1;
    777885    }
    778     /* Initilise the TX queue a minimum value if using this port for
     886    /* Initialise the TX queue a minimum value if using this port for
    779887     * receiving. Otherwise a larger size if writing packets.
    780888     */
     
    787895        return -1;
    788896    }
    789     /* Initilise the RX queue with some packets from memory */
     897    /* Initialise the RX queue with some packets from memory */
    790898    ret = rte_eth_rx_queue_setup(format_data->port, format_data->queue_id,
    791899                            format_data->nb_rx_buf, SOCKET_ID_ANY,
     
    818926    rte_eth_link_get(format_data->port, &link_info);
    819927#if DEBUG
    820     printf("Link status is %d %d %d\n", (int) link_info.link_status,
     928    fprintf(stderr, "Link status is %d %d %d\n", (int) link_info.link_status,
    821929            (int) link_info.link_duplex, (int) link_info.link_speed);
    822930#endif
    823931
    824     /* We have now successfully started/unpased */
     932    /* We have now successfully started/unpaused */
    825933    format_data->paused = DPDK_RUNNING;
    826934   
     
    828936}
    829937
    830 /* Attach memory to the port and start the port or restart the ports.
    831  */
    832 static int dpdk_start_port_queues (libtrace_t *libtrace, struct dpdk_format_data_t * format_data, char *err, int errlen, uint16_t rx_queues){
     938/* Attach memory to the port and start (or restart) the port/s.
     939 */
     940static int dpdk_start_port_queues (libtrace_t *libtrace, struct dpdk_format_data_t *format_data, char *err, int errlen, uint16_t rx_queues){
    833941    int ret, i; /* Check return values for errors */
    834942    struct rte_eth_link link_info; /* Wait for link */
     
    839947
    840948    /* First time started we need to alloc our memory, doing this here
    841      * rather than in enviroment setup because we don't have snaplen then */
     949     * rather than in environment setup because we don't have snaplen then */
    842950    if (format_data->paused == DPDK_NEVER_STARTED) {
    843951        if (format_data->snaplen == 0) {
     
    860968
    861969        /* Create the mbuf pool, which is the place our packets are allocated
    862          * from - TODO figure out if there is is a free function (I cannot see one) 
     970         * from - TODO figure out if there is is a free function (I cannot see one)
    863971         * NOTE: RX queue requires nb_packets + 1 otherwise it fails to
    864972         * allocate however that extra 1 packet is not used.
     
    9081016    printf("Doing dev configure\n");
    9091017#endif
    910     /* Initilise the TX queue a minimum value if using this port for
     1018    /* Initialise the TX queue a minimum value if using this port for
    9111019     * receiving. Otherwise a larger size if writing packets.
    9121020     */
     
    9241032    printf("Doing queue configure\n");
    9251033#endif 
    926                 /* Initilise the RX queue with some packets from memory */
     1034                /* Initialise the RX queue with some packets from memory */
    9271035                ret = rte_eth_rx_queue_setup(format_data->port, i,
    9281036                                                                format_data->nb_rx_buf, SOCKET_ID_ANY,
    9291037                                                                &rx_conf, format_data->pktmbuf_pool);
     1038        /* Init per_thread data structures */
     1039        format_data->per_lcore[i].port = format_data->port;
     1040        format_data->per_lcore[i].queue_id = i;
     1041
    9301042                if (ret < 0) {
    9311043                        snprintf(err, errlen, "Intel DPDK - Cannot configure RX queue on port"
     
    9371049   
    9381050#if DEBUG
    939     printf("Doing start device\n");
     1051    fprintf(stderr, "Doing start device\n");
    9401052#endif 
    9411053    /* Start device */
    9421054    ret = rte_eth_dev_start(format_data->port);
    9431055#if DEBUG
    944     printf("Done start device\n");
     1056    fprintf(stderr, "Done start device\n");
    9451057#endif 
    9461058    if (ret < 0) {
     
    9721084    rte_eth_link_get(format_data->port, &link_info);
    9731085#if DEBUG
    974     printf("Link status is %d %d %d\n", (int) link_info.link_status,
     1086    fprintf(stderr, "Link status is %d %d %d\n", (int) link_info.link_status,
    9751087            (int) link_info.link_duplex, (int) link_info.link_speed);
    9761088#endif
     
    9921104}
    9931105
     1106static inline size_t dpdk_get_max_rx_queues (uint8_t port_id) {
     1107    struct rte_eth_dev_info dev_info;
     1108    rte_eth_dev_info_get(port_id, &dev_info);
     1109    return dev_info.max_rx_queues;
     1110}
     1111
     1112static inline size_t dpdk_processor_count () {
     1113    long nb_cpu = sysconf(_SC_NPROCESSORS_ONLN);
     1114    if (nb_cpu <= 0)
     1115        return 1;
     1116    else
     1117        return (size_t) nb_cpu;
     1118}
     1119
    9941120static int dpdk_pstart_input (libtrace_t *libtrace) {
    9951121    char err[500];
    996     int enabled_lcore_count = 0, i=0;
     1122    int i=0, phys_cores=0;
    9971123    int tot = libtrace->perpkt_thread_count;
    9981124    err[0] = 0;
    999        
    1000         libtrace->perpkt_thread_count;
    1001        
    1002         for (i = 0; i < RTE_MAX_LCORE; i++)
    1003         {
    1004                 if (rte_lcore_is_enabled(i))
    1005                         enabled_lcore_count++;
    1006         }
    1007        
    1008         tot = MIN(libtrace->perpkt_thread_count, enabled_lcore_count);
    1009         tot = MIN(tot, 8);
    1010         printf("Running pstart DPDK %d %d %d %d\n", tot, libtrace->perpkt_thread_count, enabled_lcore_count, rte_lcore_count());
     1125
     1126    if (rte_lcore_id() != rte_get_master_lcore())
     1127        fprintf(stderr, "Warning dpdk_pstart_input should be called from the master DPDK thread!\n");
     1128
     1129    // If the master is not on the last thread we move it there
     1130    if (rte_get_master_lcore() != RTE_MAX_LCORE - 1) {
     1131        // Consider error handling here
     1132        dpdk_move_master_lcore(RTE_MAX_LCORE - 1) == -1;
     1133    }
     1134
     1135    // Don't exceed the number of cores in the system/detected by dpdk
     1136    // We don't have to force this but performance wont be good if we don't
     1137    for (i = 0; i < RTE_MAX_LCORE; ++i) {
     1138        if (lcore_config[i].detected) {
     1139            if (rte_lcore_is_enabled(i))
     1140                fprintf(stderr, "Found core %d already in use!\n", i);
     1141            else
     1142                phys_cores++;
     1143        }
     1144    }
     1145
     1146        tot = MIN(libtrace->perpkt_thread_count, dpdk_get_max_rx_queues(FORMAT(libtrace)->port));
     1147    tot = MIN(tot, phys_cores);
     1148
     1149        fprintf(stderr, "Running pstart DPDK tot=%d req=%d phys=%d\n", tot, libtrace->perpkt_thread_count, phys_cores);
    10111150       
    10121151    if (dpdk_start_port_queues(libtrace, FORMAT(libtrace), err, sizeof(err), tot) != 0) {
     
    10161155        return -1;
    10171156    }
    1018    
     1157
     1158    // Make sure we only start the number that we should
     1159    libtrace->perpkt_thread_count = tot;
    10191160    return 0;
    1020     return tot;
     1161}
     1162
     1163
     1164/**
     1165 * Register a thread with the DPDK system,
     1166 * When we start DPDK in parallel libtrace we move the 'main thread' to the
     1167 * MAXIMUM CPU core slot (32) and remove any affinity restrictions DPDK
     1168 * gives it.
     1169 *
     1170 * We then allow a mapper thread to be started on every real core as DPDK would
     1171 * we also bind these to the corresponding CPU cores.
     1172 *
     1173 * @param libtrace A pointer to the trace
     1174 * @param reading True if the thread will be used to read packets, i.e. will
     1175 *                call pread_packet(), false if thread used to process packet
     1176 *                in any other manner including statistics functions.
     1177 */
     1178static int dpdk_pregister_thread(libtrace_t *libtrace, libtrace_thread_t *t, bool reading)
     1179{
     1180    struct rte_config *cfg = rte_eal_get_configuration();
     1181    int i;
     1182    int new_id = -1;
     1183
     1184    // If 'reading packets' fill in cores from 0 up and bind affinity
     1185    // otherwise start from the MAX core (which is also the master) and work backwards
     1186    // in this case physical cores on the system will not exist so we don't bind
     1187    // these to any particular physical core
     1188    if (reading) {
     1189        for (i = 0; i < RTE_MAX_LCORE; ++i) {
     1190            if (!rte_lcore_is_enabled(i)) {
     1191                new_id = i;
     1192                if (!lcore_config[i].detected)
     1193                    fprintf(stderr, "Warning the number of 'reading' threads exceed cores on machine!!\n");
     1194                break;
     1195            }
     1196        }
     1197    } else {
     1198        for (i = RTE_MAX_LCORE-1; i >= 0; --i) {
     1199            if (!rte_lcore_is_enabled(i)) {
     1200                new_id = i;
     1201                break;
     1202            }
     1203        }
     1204    }
     1205
     1206    if (new_id == -1) {
     1207        assert(cfg->lcore_count == RTE_MAX_LCORE);
     1208        // TODO proper libtrace style error here!!
     1209        fprintf(stderr, "Too many threads for DPDK!!\n");
     1210        return -1;
     1211    }
     1212
     1213    // Enable the core in global DPDK structs
     1214    cfg->lcore_role[new_id] = ROLE_RTE;
     1215    cfg->lcore_count++;
     1216    // Set TLS to reflect our new number
     1217    assert(rte_lcore_id() == 0); // I think new threads are going get a default thread number of 0
     1218    fprintf(stderr, "original id%d", rte_lcore_id());
     1219    RTE_PER_LCORE(_lcore_id) = new_id;
     1220    fprintf(stderr, " new id%d\n", rte_lcore_id());
     1221
     1222    if (reading) {
     1223        // Set affinity bind to corresponding core
     1224        cpu_set_t cpuset;
     1225        CPU_ZERO(&cpuset);
     1226        CPU_SET(rte_lcore_id(), &cpuset);
     1227        i = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
     1228        if (i != 0) {
     1229            fprintf(stderr, "Warning pthread_setaffinity_np failed\n");
     1230            return -1;
     1231        }
     1232    }
     1233
     1234    // Map our TLS to the thread data
     1235    if (reading) {
     1236        if(t->type == THREAD_PERPKT) {
     1237            t->format_data = &FORMAT(libtrace)->per_lcore[t->perpkt_num];
     1238        } else {
     1239            t->format_data = &FORMAT(libtrace)->per_lcore[0];
     1240        }
     1241    }
     1242}
     1243
     1244
     1245/**
     1246 * Unregister a thread with the DPDK system.
     1247 *
     1248 * Only previously registered threads should be calling this just before
     1249 * they are destroyed.
     1250 */
     1251static int dpdk_punregister_thread(libtrace_t libtrace, libtrace_thread_t *t UNUSED)
     1252{
     1253    struct rte_config *cfg = rte_eal_get_configuration();
     1254
     1255    assert(rte_lcore_id() >= 0 && rte_lcore_id() < RTE_MAX_LCORE);
     1256
     1257    // Skip if master!!
     1258    if (rte_lcore_id() == rte_get_master_lcore()) {
     1259        fprintf(stderr, "INFO: we are skipping unregistering the master lcore\n");
     1260        return 0;
     1261    }
     1262
     1263    // Disable this core in global DPDK structs
     1264    cfg->lcore_role[rte_lcore_id()] = ROLE_OFF;
     1265    cfg->lcore_count--;
     1266    RTE_PER_LCORE(_lcore_id) = -1; // Might make the world burn if used again
     1267    assert(cfg->lcore_count >= 1); // We cannot unregister the master LCORE!!
     1268    return 0;
    10211269}
    10221270
     
    10391287    if (FORMAT(libtrace)->paused == DPDK_RUNNING) {
    10401288#if DEBUG     
    1041         printf("Pausing port\n");
     1289        fprintf(stderr, "Pausing port\n");
    10421290#endif
    10431291        rte_eth_dev_stop(FORMAT(libtrace)->port);
     
    13961644    return -1;
    13971645}
    1398 libtrace_thread_t * get_thread_table(libtrace_t *libtrace);
    1399 static int dpdk_pread_packet (libtrace_t *libtrace, libtrace_packet_t *packet) {
     1646
     1647static int dpdk_pread_packet (libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t *packet) {
    14001648    int nb_rx; /* Number of rx packets we've recevied */
    14011649    struct rte_mbuf* pkts_burst[1]; /* Array of 1 pointer(s) */
     
    14041652    if (packet->buffer != NULL) {
    14051653        /* Buffer is owned by DPDK */
    1406         if ( packet->buf_control == TRACE_CTRL_EXTERNAL ) {
     1654        if ( packet->buf_control == TRACE_CTRL_EXTERNAL) {
    14071655            rte_pktmbuf_free(packet->buffer);
    14081656            packet->buffer = NULL;
     
    14211669    while (1) {
    14221670        /* Poll for a single packet */
    1423         nb_rx = rte_eth_rx_burst(FORMAT(libtrace)->port,
    1424                             get_thread_table_num(libtrace), pkts_burst, 1);
     1671        nb_rx = rte_eth_rx_burst(PERPKT_FORMAT(t)->port,
     1672                            PERPKT_FORMAT(t)->queue_id, pkts_burst, 1);
    14251673        if (nb_rx > 0) { /* Got a packet - otherwise we keep spining */
    1426                         printf("Doing P READ PACKET port=%d q=%d\n", (int) FORMAT(libtrace)->port, (int) get_thread_table_num(libtrace));
     1674                        //fprintf(stderr, "Doing P READ PACKET port=%d q=%d\n", (int) FORMAT(libtrace)->port, (int) get_thread_table_num(libtrace));
    14271675            return dpdk_ready_pkt(libtrace, packet, pkts_burst[0]);
    14281676        }
    14291677        // Check the message queue this could be (Well it shouldn't but anyway) be less than 0
    1430         if (libtrace_message_queue_count(&(get_thread_table(libtrace)->messages)) > 0) {
     1678        if (libtrace_message_queue_count(&t->messages) > 0) {
    14311679                        printf("Extra message yay");
    14321680                        return -2;
     
    16411889        dpdk_pause_input, /* ppause */
    16421890        dpdk_fin_input, /* p_fin */
    1643         dpdk_pconfig_input /* pconfig_input */
     1891        dpdk_pconfig_input, /* pconfig_input */
     1892    dpdk_pregister_thread, /* pregister_thread */
     1893    dpdk_punregister_thread /* unpregister_thread */
    16441894};
    16451895
  • lib/format_linux.c

    re3a639a r17c5749  
    214214        // The flag layout should be the same for all (I Hope)
    215215        // max_order
    216 };
     216} ALIGN_STRUCT(CACHE_LINE_SIZE);
    217217
    218218struct linux_format_data_t {
     
    305305
    306306#define FORMAT(x) ((struct linux_format_data_t*)(x))
     307#define PERPKT_FORMAT(x) ((struct linux_per_thread_t*)(x->format_data))
    307308#define DATAOUT(x) ((struct linux_output_format_data_t*)((x)->format_data))
    308309
     
    696697       
    697698        if (!FORMAT(libtrace->format_data)->per_thread) {
    698                 per_thread = calloc(tot, sizeof(struct linux_per_thread_t));
     699                //per_thread = calloc(tot, sizeof(struct linux_per_thread_t));
     700                posix_memalign((void **)&per_thread, CACHE_LINE_SIZE, tot*sizeof(struct linux_per_thread_t));
    699701                FORMAT(libtrace->format_data)->per_thread = per_thread;
    700702        } else {
     
    750752}
    751753
     754static int linux_pregister_thread(libtrace_t *libtrace, libtrace_thread_t *t, bool reading) {
     755        fprintf(stderr, "registering thread %d!!\n", t->perpkt_num);
     756    if (reading) {
     757        if(t->type == THREAD_PERPKT) {
     758            t->format_data = &FORMAT(libtrace->format_data)->per_thread[t->perpkt_num];
     759        } else {
     760            t->format_data = &FORMAT(libtrace->format_data)->per_thread[0];
     761        }
     762    }
     763    return 0;
     764}
     765
    752766static int linuxnative_start_output(libtrace_out_t *libtrace)
    753767{
     
    10951109        if (check_queue) {
    10961110                // Check for a packet - TODO only Linux has MSG_DONTWAIT should use fctl O_NONBLOCK
    1097                 hdr->wirelen = recvmsg(fd, &msghdr, MSG_DONTWAIT);
    1098                 if ((unsigned) hdr->wirelen == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
     1111                hdr->wirelen = recvmsg(fd, &msghdr, MSG_DONTWAIT | MSG_TRUNC);
     1112                if ((int) hdr->wirelen == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
    10991113                        // Do message queue check or select
    11001114                        int ret;
     
    11221136                }
    11231137        } else {
    1124                 hdr->wirelen = recvmsg(fd, &msghdr, 0);
     1138                hdr->wirelen = recvmsg(fd, &msghdr, MSG_TRUNC);
    11251139        }
    11261140       
     
    11991213}
    12001214
    1201 static int linuxnative_pread_packet(libtrace_t *libtrace, libtrace_packet_t *packet)
    1202 {
    1203         int fd = FORMAT(libtrace->format_data)->per_thread[get_thread_table_num(libtrace)].fd;
    1204         //printf("Thread number is #%d point %p\n", get_thread_table_num(libtrace), FORMAT(libtrace->format_data)->per_thread);
     1215static int linuxnative_pread_packet(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t *packet)
     1216{
     1217        int fd = PERPKT_FORMAT(t)->fd;
     1218        //fprintf(stderr, "Thread number is #%d fd=%d\n", t->perpkt_num, PERPKT_FORMAT(t)->fd);
    12051219        return linuxnative_read_packet_fd(libtrace, packet, fd, 1);
    12061220}
     
    12421256static void linuxring_fin_packet(libtrace_packet_t *packet)
    12431257{
     1258
     1259        if (packet->buffer == NULL)
     1260                return;
    12441261        assert(packet->trace);
    12451262       
     
    13421359}
    13431360
    1344 static int linuxring_pread_packet(libtrace_t *libtrace, libtrace_packet_t *packet) {
    1345         int tnum = get_thread_table_num(libtrace);
    1346         int fd = FORMAT(libtrace->format_data)->per_thread[tnum].fd;
    1347         int *rxring_offset = &FORMAT(libtrace->format_data)->per_thread[tnum].rxring_offset;
    1348         char *rx_ring = FORMAT(libtrace->format_data)->per_thread[tnum].rx_ring;
    1349         printf("Thread number is #%d point %p\n", get_thread_table_num(libtrace), FORMAT(libtrace->format_data)->per_thread);
     1361static int linuxring_pread_packet(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t *packet) {
     1362        fprintf(stderr, "Thread number is #%d\n", t->perpkt_num);
     1363        int fd = PERPKT_FORMAT(t)->fd;
     1364        int *rxring_offset = &PERPKT_FORMAT(t)->rxring_offset;
     1365        char *rx_ring = PERPKT_FORMAT(t)->rx_ring;
    13501366        return linuxring_read_packet_fd(libtrace, packet, fd, rxring_offset, rx_ring, 1);
    13511367}
     
    14061422
    14071423        return ret;
    1408 
    1409 }
     1424}
     1425
    14101426static int linuxring_write_packet(libtrace_out_t *trace,
    14111427                libtrace_packet_t *packet)
     
    18441860        linuxnative_ppause_input,                       /* ppause */
    18451861        linuxnative_fin_input,                          /* p_fin */
    1846         linuxnative_pconfig_input                       /* pconfig input */
     1862        linuxnative_pconfig_input,                      /* pconfig input */
     1863        linux_pregister_thread,
     1864        NULL
    18471865};
    18481866
     
    18931911        linuxnative_ppause_input,                       /* ppause */
    18941912        linuxnative_fin_input,                          /* p_fin */
    1895         linuxnative_pconfig_input
    1896        
     1913        linuxnative_pconfig_input,
     1914        linux_pregister_thread,
     1915        NULL
    18971916};
    18981917#else
  • lib/format_pcap.c

    rb13b939 r17c5749  
    578578static libtrace_direction_t pcap_set_direction(libtrace_packet_t *packet,
    579579                libtrace_direction_t dir) {
    580        
     580
     581        /* We only support tagging with IN or OUT return error for any others */
     582        if(!(dir == TRACE_DIR_OUTGOING || dir == TRACE_DIR_INCOMING))
     583                return -1;
     584
    581585        /* PCAP doesn't have a direction field in the header, so we need to
    582586         * promote to Linux SLL to tag it properly */
  • lib/libtrace.h.in

    r9594cf9 r17c5749  
    194194#  define PRINTF(formatpos, argpos)
    195195#endif
     196
     197// Used to fight against false sharing
     198#define CACHE_LINE_SIZE 64
     199#define ALIGN_STRUCT(x) __attribute__((aligned(x)))
    196200
    197201#ifdef _MSC_VER
     
    879883        uint8_t ospf_v;         /**< OSPF Version, should be 2 */
    880884        uint8_t type;           /**< OSPF Packet Type */
    881         uint16_t len;           /**< Packet length, including OSPF header */
     885        uint16_t ospf_len;      /**< Packet length, including OSPF header */
    882886        struct in_addr router;  /**< Router ID of the packet source */
    883887        struct in_addr area;    /**< Area the packet belongs to */
     
    12511255        TRACE_OPTION_COMPRESSTYPE_BZ2  = 2, /**< BZip2 Compression */
    12521256        TRACE_OPTION_COMPRESSTYPE_LZO  = 3,  /**< LZO Compression */
     1257        TRACE_OPTION_COMPRESSTYPE_LZMA  = 4,  /**< LZO Compression */
    12531258        TRACE_OPTION_COMPRESSTYPE_LAST
    12541259} trace_option_compresstype_t;
     
    17911796DLLEXPORT uint16_t *trace_checksum_transport(libtrace_packet_t *packet,
    17921797                uint16_t *csum);
     1798
     1799/** Calculates the fragment offset in bytes for an IP packet
     1800 * @param packet        The libtrace packet to calculate the offset for
     1801 * @param[out] more     A boolean flag to indicate whether there are more
     1802 *                      fragments after the current packet.
     1803 * @return The fragment offset for the packet in bytes. If the packet is not
     1804 * an IP packet or the fragment offset is not present in packet, the return
     1805 * value will be 0.
     1806 *
     1807 * @note The value returned is in bytes, not 8-octet units as it is stored
     1808 * in the fragment offset field in the headers. In other words, libtrace
     1809 * automatically does the multiplication for you.
     1810 *
     1811 * The value passed in for 'more' does not matter; it will be overwritten
     1812 * with the value of the More Fragments flag from the IP header.
     1813 *
     1814 * New in libtrace 3.0.20
     1815 */
     1816DLLEXPORT uint16_t trace_get_fragment_offset(const libtrace_packet_t *packet,
     1817                uint8_t *more);
    17931818
    17941819/** Gets a pointer to the transport layer header (if any)
     
    31633188
    31643189DLLEXPORT int trace_pstart(libtrace_t *libtrace, void* global_blob, fn_per_pkt per_pkt, fn_reducer reducer);
    3165 DLLEXPORT int trace_pread_packet(libtrace_t *libtrace, libtrace_packet_t **packet);
     3190DLLEXPORT int trace_pread_packet(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t **packet);
    31663191DLLEXPORT int trace_ppause(libtrace_t *libtrace);
    31673192DLLEXPORT int trace_pstop(libtrace_t *libtrace);
  • lib/libtrace_int.h

    r3296252 r50ce607  
    208208        enum thread_states state;
    209209        void* user_data; // TLS for the user to use
     210        void* format_data; // TLS for the format to use
    210211        pthread_t tid;
    211212        int perpkt_num; // A number from 0-X that represents this perpkt threads number
     
    912913         * @return same as read_packet, with the addition of return -2 to represent
    913914         * interrupted due to message waiting. */
    914         int (*pread_packet)(libtrace_t *trace, libtrace_packet_t *packet);
     915        int (*pread_packet)(libtrace_t *trace, libtrace_thread_t *t, libtrace_packet_t *packet);
    915916       
    916917        /** Pause a parallel trace
     
    936937         */
    937938        int (*pconfig_input)(libtrace_t *libtrace,trace_parallel_option_t option,void *value);
    938                
     939
     940        /**
     941         * Register a thread for use with the format or using the packets produced
     942         * by it. This is NOT only used for threads reading packets infact all
     943         * threads use this.
     944         *
     945         * Some use cases include setting up any thread local storage required for
     946         * to read packets and free packets. For DPDK we require any thread that
     947         * may release or read a packet to have have an internal number associated
     948         * with it.
     949         *
     950         * The thread type can be used to see if this thread is going to be used
     951         * to read packets or otherwise.
     952         *
     953         * @return 0 if successful, -1 if the option is unsupported or an error
     954         * occurs (such as a maximum of threads being reached)
     955         */
     956        int (*pregister_thread)(libtrace_t *libtrace, libtrace_thread_t *t, bool reader);
     957
     958        /**
     959         * If needed any memory allocated with pregister_thread can be released
     960         * in this function. The thread will be destroyed directly after this
     961         * function is called.
     962         */
     963        void (*punregister_thread)(libtrace_t *libtrace, libtrace_thread_t *t);
    939964};
    940965
     
    946971NULL,                   /* ppause_input */ \
    947972NULL,                   /* pfin_input */ \
    948 NULL,                   /* pconfig_input */
     973NULL,                   /* pconfig_input */ \
     974NULL,                   /* pregister_thread */ \
     975NULL                    /* punregister_thread */
    949976
    950977/** The list of registered capture formats */
  • lib/protocols_l3.c

    r9ca1fce r7baa948  
    724724}
    725725
     726DLLEXPORT uint16_t trace_get_fragment_offset(const libtrace_packet_t *packet,
     727                uint8_t *more) {
     728
     729        void *l3;
     730        uint16_t ethertype;
     731        uint32_t remaining;
     732
     733        *more = 0;
     734
     735        l3 = trace_get_layer3(packet, &ethertype, &remaining);
     736        if (l3 == NULL)
     737                return 0;
     738
     739        if (ethertype == TRACE_ETHERTYPE_IP) {
     740                libtrace_ip_t *ip = (libtrace_ip_t *)l3;
     741                uint16_t offset = 0;
     742
     743                /* Fragment offset appears in 7th and 8th bytes */
     744                if (remaining < 8)
     745                        return 0;
     746                 
     747                offset = ntohs(ip->ip_off);
     748
     749                if ((offset & 0x2000) != 0)
     750                        *more = 1;
     751                return (offset & 0x1FFF) * 8;
     752        }
     753
     754        if (ethertype == TRACE_ETHERTYPE_IPV6) {
     755                libtrace_ip6_t *ip6 = (libtrace_ip6_t *)l3;
     756                void *payload = ip6++;
     757                uint8_t nxt = ip6->nxt;
     758                uint16_t len;
     759               
     760                /* First task, find a Fragment header if present */
     761                if (remaining < sizeof(libtrace_ip6_t))
     762                        return 0;
     763                remaining -= sizeof(libtrace_ip6_t);
     764
     765                /* Adapted from trace_get_payload_from_ip6 */
     766                while (1) {
     767                        switch (nxt) {
     768                        case 0:
     769                        case TRACE_IPPROTO_ROUTING:
     770                        case TRACE_IPPROTO_AH:
     771                        case TRACE_IPPROTO_DSTOPTS:
     772                        {
     773
     774                                /* Length does not include the first 8 bytes */
     775                                len=((libtrace_ip6_ext_t*)payload)->len * 8;
     776                                len += 8;
     777
     778                                if (remaining < len) {
     779                                        /* Snap too short */
     780                                        return 0;
     781                                }
     782                                remaining-=len;
     783
     784                                nxt=((libtrace_ip6_ext_t*)payload)->nxt;
     785                                continue;
     786                        }
     787                        case TRACE_IPPROTO_FRAGMENT:
     788                        {
     789                                libtrace_ip6_frag_t *frag = (libtrace_ip6_frag_t *)payload;
     790                                uint16_t offset;
     791                                len = sizeof(libtrace_ip6_frag_t);
     792                                if (remaining < len) {
     793                                        /* Snap too short */
     794                                        return 0;
     795                                }
     796                                remaining-=len;
     797
     798                                offset = ntohs(frag->frag_off);
     799                                if ((offset & 0x0001) != 0)
     800                                        *more = 1;
     801
     802                                return ((offset & 0xFFF8) >> 3) * 8;
     803                         }
     804                         default:
     805                                return 0;
     806                         }
     807                }
     808
     809        }
     810        return 0;
     811}
  • lib/protocols_transport.c

    r10f924c re0bea4e5  
    382382        uint32_t remaining;
    383383        uint8_t proto;
    384         const struct ports_t *port =
    385                 (const struct ports_t*)trace_get_transport((libtrace_packet_t*)packet,
     384        struct ports_t *port;
     385        uint16_t fragoff;
     386        uint8_t more;
     387
     388        fragoff = trace_get_fragment_offset(packet, &more);
     389
     390        /* If we're not the first fragment, we're unlikely to be able
     391         * to get any useful port numbers from this packet.
     392         */
     393        if (fragoff != 0)
     394                return 0;
     395       
     396       
     397        port = (struct ports_t*)trace_get_transport(
     398                        (libtrace_packet_t*)packet,
    386399                        &proto, &remaining);
    387400
     
    391404
    392405        /* ICMP *technically* doesn't have ports */
    393         if (proto == TRACE_IPPROTO_ICMP)
     406        if (proto == TRACE_IPPROTO_ICMP || proto == TRACE_IPPROTO_ICMPV6)
    394407                return 0;
    395408
     
    405418        uint32_t remaining;
    406419        uint8_t proto;
    407         struct ports_t *port =
    408                 (struct ports_t*)trace_get_transport((libtrace_packet_t*)packet,
     420        struct ports_t *port;
     421        uint16_t fragoff;
     422        uint8_t more;
     423
     424        fragoff = trace_get_fragment_offset(packet, &more);
     425
     426        /* If we're not the first fragment, we're unlikely to be able
     427         * to get any useful port numbers from this packet.
     428         */
     429        if (fragoff != 0)
     430                return 0;
     431       
     432       
     433        port = (struct ports_t*)trace_get_transport(
     434                        (libtrace_packet_t*)packet,
    409435                        &proto, &remaining);
    410436        /* Snapped too early */
     
    413439       
    414440        /* ICMP *technically* doesn't have ports */
    415         if (proto == TRACE_IPPROTO_ICMP)
     441        if (proto == TRACE_IPPROTO_ICMP || proto == TRACE_IPPROTO_ICMPV6)
    416442                return 0;
    417443
  • lib/trace_parallel.c

    r3296252 r049a700  
    317317        assert(t);
    318318        //printf("Yay Started perpkt thread #%d\n", (int) get_thread_table_num(trace));
     319        if (trace->format->pregister_thread) {
     320                trace->format->pregister_thread(trace, t, !trace_has_dedicated_hasher(trace));
     321        }
    319322        assert(pthread_mutex_unlock(&trace->libtrace_lock) == 0);
    320323
     
    343346                                                // The hasher has stopped by this point, so the queue shouldn't be filling
    344347                                                while(!libtrace_ringbuffer_is_empty(&t->rbuffer)) {
    345                                                         psize = trace_pread_packet(trace, &packet);
     348                                                        psize = trace_pread_packet(trace, t, &packet);
    346349                                                        if (psize > 0) {
    347350                                                                packet = (*trace->per_pkt)(trace, packet, NULL, t);
     
    375378                        }
    376379                } else {
    377                         psize = trace_pread_packet(trace, &packet);
     380                        psize = trace_pread_packet(trace, t, &packet);
    378381                }
    379382
     
    413416        trace_send_message_to_reducer(trace, &message);
    414417
     418        assert(pthread_mutex_lock(&trace->libtrace_lock) == 0);
     419        if (trace->format->punregister_thread) {
     420                trace->format->punregister_thread(trace, t);
     421        }
     422        assert(pthread_mutex_unlock(&trace->libtrace_lock) == 0);
     423
    415424        pthread_exit(NULL);
    416425};
     
    434443        assert(t->type == THREAD_HASHER && pthread_equal(pthread_self(), t->tid));
    435444        printf("Hasher Thread started\n");
     445        if (trace->format->pregister_thread) {
     446                trace->format->pregister_thread(trace, t, true);
     447        }
    436448        assert(pthread_mutex_unlock(&trace->libtrace_lock) == 0);
    437449        int pkt_skipped = 0;
     
    514526        message.additional.uint64 = 0;
    515527        trace_send_message_to_reducer(trace, &message);
     528        assert(pthread_mutex_lock(&trace->libtrace_lock) == 0);
     529        if (trace->format->punregister_thread) {
     530                trace->format->punregister_thread(trace, t);
     531        }
     532        assert(pthread_mutex_unlock(&trace->libtrace_lock) == 0);
    516533
    517534        // TODO remove from TTABLE t sometime
     
    539556 * lock to read a packet from the underlying trace.
    540557 */
    541 inline static int trace_pread_packet_first_in_first_served(libtrace_t *libtrace, libtrace_packet_t **packet)
     558inline static int trace_pread_packet_first_in_first_served(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t **packet)
    542559{
    543560        // We need this to fill the 'first' packet table
    544         libtrace_thread_t *t = get_thread_table(libtrace);
    545561        if (!*packet) {
    546562                if (!libtrace_ringbuffer_try_sread_bl(&libtrace->packet_freelist, (void **) packet))
     
    565581 * 2. Move that into the packet provided (packet)
    566582 */
    567 inline static int trace_pread_packet_hasher_thread(libtrace_t *libtrace, libtrace_packet_t **packet)
    568 {
    569         int this_thread = get_thread_table_num(libtrace); // Could be worth caching ... ?
    570         libtrace_thread_t* t = &libtrace->perpkt_threads[this_thread];
    571 
     583inline static int trace_pread_packet_hasher_thread(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t **packet)
     584{
    572585        if (*packet) // Recycle the old get the new
    573586                if (!libtrace_ringbuffer_try_swrite_bl(&libtrace->packet_freelist, (void *) *packet))
     
    675688 * 2. Move that into the packet provided (packet)
    676689 */
    677 inline static int trace_pread_packet_hash_locked(libtrace_t *libtrace, libtrace_packet_t **packet)
    678 {
    679         int this_thread = get_thread_table_num(libtrace); // Could be worth caching ... ?
    680         libtrace_thread_t * t = &libtrace->perpkt_threads[this_thread];
     690inline static int trace_pread_packet_hash_locked(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t **packet)
     691{
    681692        int thread, ret/*, psize*/;
    682693
     
    696707                // Another thread cannot write a packet because a queue has filled up. Is it ours?
    697708                if (libtrace->perpkt_queue_full) {
    698                         contention_stats[this_thread].wait_for_fill_complete_hits++;
     709                        contention_stats[t->perpkt_num].wait_for_fill_complete_hits++;
    699710                        assert(pthread_mutex_unlock(&libtrace->libtrace_lock) == 0);
    700711                        continue;
     
    718729                trace_packet_set_hash(*packet, (*libtrace->hasher)(*packet, libtrace->hasher_data));
    719730                thread = trace_packet_get_hash(*packet) % libtrace->perpkt_thread_count;
    720                 if (thread == this_thread) {
     731                if (thread == t->perpkt_num) {
    721732                        // If it's this thread we must be in order because we checked the buffer once we got the lock
    722733                        assert(pthread_mutex_unlock(&libtrace->libtrace_lock) == 0);
     
    728739                                libtrace->perpkt_queue_full = true;
    729740                                assert(pthread_mutex_unlock(&libtrace->libtrace_lock) == 0);
    730                                 contention_stats[this_thread].full_queue_hits++;
     741                                contention_stats[t->perpkt_num].full_queue_hits++;
    731742                                assert(pthread_mutex_lock(&libtrace->libtrace_lock) == 0);
    732743                        }
     
    754765 * 2. Move that into the packet provided (packet)
    755766 */
    756 inline static int trace_pread_packet_sliding_window(libtrace_t *libtrace, libtrace_packet_t **packet)
    757 {
    758         int this_thread = get_thread_table_num(libtrace); // Could be worth caching ... ?
    759         libtrace_thread_t * t = &libtrace->perpkt_threads[this_thread];
     767inline static int trace_pread_packet_sliding_window(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t **packet)
     768{
    760769        int ret, i, thread/*, psize*/;
    761770
     
    785794                        assert(pthread_mutex_unlock(&libtrace->libtrace_lock) == 0);
    786795                        assert(sem_post(&libtrace->sem) == 0);
    787                         contention_stats[this_thread].wait_for_fill_complete_hits++;
     796                        contention_stats[t->perpkt_num].wait_for_fill_complete_hits++;
    788797                        continue;
    789798                }
     
    833842                                if (libtrace->perpkt_threads[thread].state != THREAD_FINISHED) {
    834843                                        while (!libtrace_ringbuffer_try_swrite_bl(&libtrace->perpkt_threads[thread].rbuffer, *packet)) {
    835                                                 if (this_thread == thread)
     844                                                if (t->perpkt_num == thread)
    836845                                                {
    837846                                                        // TODO think about this case more because we have to stop early if this were to happen on the last read
     
    855864                                                        assert(sem_post(&libtrace->sem) == 0);
    856865
    857                                                 contention_stats[this_thread].full_queue_hits++;
     866                                                contention_stats[t->perpkt_num].full_queue_hits++;
    858867                                                assert(pthread_rwlock_wrlock(&libtrace->window_lock) == 0);
    859868                                                // Grab these back
     
    10481057 *
    10491058 */
    1050 static inline int trace_pread_packet_wrapper(libtrace_t *libtrace, libtrace_packet_t *packet) {
     1059static inline int trace_pread_packet_wrapper(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t *packet) {
    10511060
    10521061        assert(libtrace && "You called trace_read_packet() with a NULL libtrace parameter!\n");
     
    10731082                         * structure */
    10741083                        packet->trace = libtrace;
    1075                         ret=libtrace->format->pread_packet(libtrace,packet);
     1084                        ret=libtrace->format->pread_packet(libtrace, t, packet);
    10761085                        if (ret==(size_t)-1 || ret==(size_t)-2 || ret==0) {
    10771086                                return ret;
     
    11031112 * Read a packet from the parallel trace
    11041113 */
    1105 DLLEXPORT int trace_pread_packet(libtrace_t *libtrace, libtrace_packet_t **packet)
     1114DLLEXPORT int trace_pread_packet(libtrace_t *libtrace, libtrace_thread_t *t, libtrace_packet_t **packet)
    11061115{
    11071116        int ret;
    1108         libtrace_thread_t *t = get_thread_table(libtrace);
    11091117
    11101118        // Cleanup the packet passed back
     
    11151123                if (!*packet)
    11161124                        *packet = trace_create_packet();
    1117                 ret = trace_pread_packet_wrapper(libtrace, *packet);
     1125                ret = trace_pread_packet_wrapper(libtrace, t, *packet);
    11181126        } else if (trace_has_dedicated_hasher(libtrace)) {
    1119                 ret = trace_pread_packet_hasher_thread(libtrace, packet);
     1127                ret = trace_pread_packet_hasher_thread(libtrace, t, packet);
    11201128        } else if (!trace_has_dedicated_hasher(libtrace)) {
    11211129                /* We don't care about which core a packet goes to */
    1122                 ret = trace_pread_packet_first_in_first_served(libtrace, packet);
     1130                ret = trace_pread_packet_first_in_first_served(libtrace, t, packet);
    11231131        } /* else {
    11241132                ret = trace_pread_packet_hash_locked(libtrace, packet);
     
    11651173                return -1;
    11661174        }
     1175       
    11671176        // NOTE: Until the trace is started we wont have a libtrace_lock initialised
    11681177        if (libtrace->state != STATE_NEW) {
  • libpacketdump/eth_2048.c

    r66ad025 r387d299  
    1010#include <netdb.h>
    1111
    12 #define DISPLAY_EXP(x,fmt,exp) \
    13         if ((unsigned int)len>=((char*)&ip->x-(char*)ip+sizeof(ip->x))) \
    14                 printf(fmt,exp); \
    15         else \
    16                 return;
    17 
    18 #define DISPLAY(x,fmt) DISPLAY_EXP(x,fmt,ip->x)
    19 
    20 #define DISPLAYS(x,fmt) DISPLAY_EXP(x,fmt,htons(ip->x))
    21 #define DISPLAYIP(x,fmt) DISPLAY_EXP(x,fmt,inet_ntoa(*(struct in_addr*)&ip->x))
    22 
    2312DLLEXPORT void decode(int link_type UNUSED,const char *packet,unsigned len)
    2413{
     
    2918        }
    3019        //DISPLAY(ip_tos," TOS %02x")
    31         DISPLAY_EXP(ip_tos," DSCP %02x",ip->ip_tos >> 2)
    32         DISPLAY_EXP(ip_tos," ECN %x",ip->ip_tos & 0x2)
    33         DISPLAYS(ip_len," Total Length %i")
     20        DISPLAY_EXP(ip, ip_tos," DSCP %02x",ip->ip_tos >> 2);
     21        DISPLAY_EXP(ip, ip_tos," ECN %x",ip->ip_tos & 0x2);
     22        DISPLAYS(ip, ip_len," Total Length %i");
    3423        printf("\n IP:");
    35         DISPLAYS(ip_id," Id %u");
     24        DISPLAYS(ip, ip_id," Id %u");
    3625       
    3726        if ((unsigned int)len >= ((char *)&ip->ip_ttl - (char *)ip - 2)) {
     
    4231        }
    4332        //printf("\n IP:");
    44         DISPLAY(ip_ttl,"\n IP: TTL %i");
     33        DISPLAY(ip, ip_ttl,"\n IP: TTL %i");
    4534        if ((unsigned int)len>=((char*)&ip->ip_p-(char*)ip+sizeof(ip->ip_p))) {
    4635                struct protoent *ent=getprotobynumber(ip->ip_p);
     
    5544                return;
    5645        }
    57         DISPLAYS(ip_sum," Checksum %i\n");
    58         DISPLAYIP(ip_src," IP: Source %s ");
    59         DISPLAYIP(ip_dst,"Destination %s\n");
     46        DISPLAYS(ip, ip_sum," Checksum %i\n");
     47        DISPLAYIP(ip, ip_src," IP: Source %s ");
     48        DISPLAYIP(ip, ip_dst,"Destination %s\n");
    6049        decode_next(packet+ip->ip_hl*4,len-ip->ip_hl*4,"ip",ip->ip_p);
    6150        return;
  • libpacketdump/ip_1.c

    rec0b927 r387d299  
    33#include <dlfcn.h>
    44#include "libpacketdump.h"
    5 
    6 #define STRUCT icmp
    7 
    8 #define SAFE(x) \
    9         ((unsigned int)len>=((char*)&STRUCT->x-(char*)STRUCT+sizeof(STRUCT->x)))
    10 #define DISPLAY_EXP(x,fmt,exp) \
    11         if (SAFE(x)) \
    12                 printf(fmt,exp); \
    13         else \
    14                 return;
    15 
    16 #define DISPLAY(x,fmt) DISPLAY_EXP(x,fmt,STRUCT->x)
    17 
    18 #define DISPLAYS(x,fmt) DISPLAY_EXP(x,fmt,htons(STRUCT->x))
    19 #define DISPLAYL(x,fmt) DISPLAY_EXP(x,fmt,htonl(STRUCT->x))
    20 #define DISPLAYIP(x,fmt) DISPLAY_EXP(x,fmt,inet_ntoa(*(struct in_addr*)&STRUCT->x))
    215
    226static char *unreach_types[]={
  • libpacketdump/ip_17.c

    rc7062df r387d299  
    88#include <netdb.h>
    99
    10 #define STRUCT udp
    11 
    12 #define SAFE(x) \
    13         ((unsigned int)len>=((char*)&STRUCT->x-(char*)STRUCT+sizeof(STRUCT->x)))
    14 #define DISPLAY_EXP(x,fmt,exp) \
    15         if (SAFE(x)) \
    16                 printf(fmt,exp); \
    17         else \
    18                 return;
    19 
    20 #define DISPLAY(x,fmt) DISPLAY_EXP(x,fmt,STRUCT->x)
    21 
    22 #define DISPLAYS(x,fmt) DISPLAY_EXP(x,fmt,htons(STRUCT->x))
    23 #define DISPLAYL(x,fmt) DISPLAY_EXP(x,fmt,htonl(STRUCT->x))
    24 #define DISPLAYIP(x,fmt) DISPLAY_EXP(x,fmt,inet_ntoa(*(struct in_addr*)&STRUCT->x))
    25 
    2610
    2711DLLEXPORT void decode(int link_type UNUSED,const char *packet,unsigned len)
     
    2913        struct libtrace_udp *udp = (struct libtrace_udp*)packet;
    3014        printf(" UDP:");
    31         if (SAFE(source)) {
     15        if (SAFE(udp, source)) {
    3216                struct servent *ent=getservbyport(udp->source,"udp");
    3317                if(ent) {
     
    4125                return;
    4226        }
    43         if (SAFE(dest)) {
     27        if (SAFE(udp, dest)) {
    4428                struct servent *ent=getservbyport(udp->dest,"udp");
    4529                if(ent) {
     
    5438        }
    5539        printf("\n UDP:");
    56         DISPLAYS(len," Len %u");
    57         DISPLAYS(check," Checksum %u");
     40        DISPLAYS(udp, len," Len %u");
     41        DISPLAYS(udp, check," Checksum %u");
    5842        printf("\n");
    5943        if (htons(udp->source) < htons(udp->dest))
  • libpacketdump/ip_33.c

    rc7062df r387d299  
    66#include <netinet/tcp.h>
    77#include <netinet/in.h>
    8 
    9 #define STRUCT dccp
    10 
    11 #define SAFE(x) \
    12         ((unsigned int)len>=((char*)&STRUCT->x-(char*)STRUCT+sizeof(STRUCT->x)))
    13 #define DISPLAY_EXP(x,fmt,exp) \
    14         if (SAFE(x)) \
    15                 printf(fmt,exp); \
    16         else \
    17                 return;
    18 
    19 #define DISPLAY(x,fmt) DISPLAY_EXP(x,fmt,STRUCT->x)
    20 
    21 #define DISPLAYS(x,fmt) DISPLAY_EXP(x,fmt,htons(STRUCT->x))
    22 #define DISPLAYL(x,fmt) DISPLAY_EXP(x,fmt,htonl(STRUCT->x))
    23 #define DISPLAYIP(x,fmt) DISPLAY_EXP(x,fmt,inet_ntoa(*(struct in_addr*)&STRUCT->x))
    248
    259struct dccphdr {
     
    5034{
    5135        struct dccphdr *dccp = (struct dccphdr*)packet;
    52         DISPLAYS(source," DCCP: Source %i");
    53         DISPLAYS(dest," Dest %i");
     36        DISPLAYS(dccp, source," DCCP: Source %i");
     37        DISPLAYS(dccp, dest," Dest %i");
    5438        if (len>4) {
    5539                printf("\n DCCP: Type %i",dccp->type);
     
    6953        else
    7054                return;
    71         DISPLAY(doff," DCCP: Dataoff: %i\n");
     55        DISPLAY(dccp, doff," DCCP: Dataoff: %i\n");
    7256        if (len>9)
    7357                printf(" DCCP: NDP %i CsLen: %i\n",dccp->ndp,dccp->cslen);
     
    7559                return;
    7660        }
    77         DISPLAY(check," DCCP: Checksum: %i\n");
     61        /* Should this be byteswapped??? */
     62        DISPLAY(dccp, check," DCCP: Checksum: %i\n");
    7863        if (htons(dccp->source) < htons(dccp->dest))
    7964                decode_next(packet+dccp->doff*4,len-dccp->doff*4,"dccp",htons(dccp->source));
  • libpacketdump/ip_6.c

    r9ca0b29 r387d299  
    55#include <assert.h>
    66#include <netdb.h>
    7 
    8 #define SAFE(x) \
    9         ((unsigned int)len>=((char*)&tcp->x-(char*)tcp+sizeof(tcp->x)))
    10 #define DISPLAY_EXP(x,fmt,exp) \
    11         if (SAFE(x)) \
    12                 printf(fmt,exp); \
    13         else \
    14                 return;
    15 
    16 #define DISPLAY(x,fmt) DISPLAY_EXP(x,fmt,tcp->x)
    17 
    18 #define DISPLAYS(x,fmt) DISPLAY_EXP(x,fmt,htons(tcp->x))
    19 #define DISPLAYL(x,fmt) DISPLAY_EXP(x,fmt,htonl(tcp->x))
    20 #define DISPLAYIP(x,fmt) DISPLAY_EXP(x,fmt,inet_ntoa(*(struct in_addr*)&tcp->x))
    217
    228DLLEXPORT void decode(int link_type UNUSED,const char *packet,unsigned len)
     
    2713        libtrace_tcp_t *tcp = (libtrace_tcp_t *)packet;
    2814        printf(" TCP:");
    29         if (SAFE(source)) {
     15        if (SAFE(tcp, source)) {
    3016                struct servent *ent=getservbyport(tcp->source,"tcp");
    3117                if(ent) {
     
    3925                return;
    4026        }
    41         if (SAFE(dest)) {
     27        if (SAFE(tcp, dest)) {
    4228                struct servent *ent=getservbyport(tcp->dest,"tcp");
    4329                if(ent) {
     
    5238        }
    5339        printf("\n TCP:");
    54         DISPLAYL(seq," Seq %u");
     40        DISPLAYL(tcp, seq," Seq %u");
    5541        printf("\n TCP:");
    56         DISPLAYL(ack_seq," Ack %u");
     42        DISPLAYL(tcp, ack_seq," Ack %u");
    5743        if ((char*)&tcp->window-(char *)tcp>len) {
    5844                printf("\n");
     
    7157        if (tcp->ack) printf(" ACK");
    7258        if (tcp->urg) printf(" URG");
    73         DISPLAYS(window," Window %i");
     59        DISPLAYS(tcp, window," Window %i");
    7460        printf("\n TCP:");
    75         DISPLAYS(check," Checksum %i");
    76         DISPLAYS(urg_ptr," Urgent %i");
     61        DISPLAYS(tcp, check," Checksum %i");
     62        DISPLAYS(tcp, urg_ptr," Urgent %i");
    7763        pkt = (unsigned char*)packet+sizeof(*tcp);
    7864        plen = (len-sizeof *tcp) < (tcp->doff*4-sizeof(*tcp))?(len-sizeof(*tcp)):(tcp->doff*4-sizeof *tcp);
  • libpacketdump/ip_89.c

    re224862 r387d299  
    1010static void dump_ospf_v2_header(libtrace_ospf_v2_t *hdr, unsigned len) {
    1111
    12         printf(" OSPF Header: Version %u Type %u ",
    13                         hdr->ospf_v, hdr->type);
     12        DISPLAY(hdr, ospf_v, " OSPF Header: Version %u");
     13        DISPLAY(hdr, type, " Type %u ");
    1414        switch(hdr->type) {
    1515                case TRACE_OSPF_HELLO:
     
    2929                        break;
    3030        }
     31        printf("\n");
    3132
    32         printf("\n OSPF Header: Length %u \n", ntohs(hdr->len));
    33         printf(" OSPF Header: Router Id %s ", inet_ntoa(hdr->router));
    34         printf("Area Id %s\n", inet_ntoa(hdr->area));
    35         printf(" OSPF Header: Checksum %u Auth Type %u\n", ntohs(hdr->sum),
    36                         ntohs(hdr->au_type));
    37 
    38         printf(" OSPF Header: Auth Key ID %u Auth Data Len %u\n",
    39                         hdr->au_key_id, hdr->au_data_len);
    40         printf(" OSPF Header: Auth Crypto Seq %u\n", ntohl(hdr->au_seq_num));
    41 
     33        DISPLAYS(hdr, ospf_len, "OSPF Header: Length %u \n");
     34        DISPLAYIP(hdr, router, " OSPF Header: Router Id %s ");
     35        DISPLAYIP(hdr, area, "Area Id %s\n");
     36        DISPLAYS(hdr, sum, " OSPF Header: Checksum %u ");
     37        DISPLAYS(hdr, au_type, "Auth Type %u\n");
     38        DISPLAY(hdr, au_key_id, " OSPF Header: Auth Key ID %u ");
     39        DISPLAY(hdr, au_data_len, "Auth Data Len %u\n");
     40        DISPLAYL(hdr, au_seq_num, " OSPF Header: Auth Crypto Seq %u\n");
    4241
    4342}
  • libpacketdump/libpacketdump.h

    r66ad025 r387d299  
    66extern "C" {
    77#endif
     8
     9#define SAFE(hdr,x) \
     10        ((unsigned int)len>=((char*)&hdr->x-(char*)hdr+sizeof(hdr->x)))
     11
     12#define DISPLAY_EXP(hdr,x,fmt,exp) \
     13        if (SAFE(hdr, x)) \
     14                printf(fmt,exp); \
     15        else {\
     16                printf("(Truncated)\n"); \
     17                return; \
     18        }
     19
     20#define DISPLAY(hdr,x,fmt) DISPLAY_EXP(hdr,x,fmt,hdr->x)
     21
     22#define DISPLAYS(hdr,x,fmt) DISPLAY_EXP(hdr,x,fmt,htons(hdr->x))
     23#define DISPLAYL(hdr,x,fmt) DISPLAY_EXP(hdr,x,fmt,htonl(hdr->x))
     24#define DISPLAYIP(hdr,x,fmt) DISPLAY_EXP(hdr,x,fmt,inet_ntoa(*(struct in_addr*)&hdr->x))
     25
    826
    927void trace_hexdump_packet(libtrace_packet_t *packet);
  • libwandio/Makefile.am

    rbd119b3 r3b6e0cf  
    2424endif
    2525
     26if HAVE_LZMA
     27LIBTRACEIO_LZMA=ior-lzma.c iow-lzma.c
     28else
     29LIBTRACEIO_LZMA=
     30endif
     31
    2632libwandio_la_SOURCES=wandio.c ior-peek.c ior-stdio.c ior-thread.c \
    2733                iow-stdio.c iow-thread.c wandio.h wandio_internal.h \
    28                 $(LIBTRACEIO_ZLIB) $(LIBTRACEIO_BZLIB) $(LIBTRACEIO_LZO)
     34                $(LIBTRACEIO_ZLIB) $(LIBTRACEIO_BZLIB) $(LIBTRACEIO_LZO) \
     35                $(LIBTRACEIO_LZMA)
    2936
    3037AM_CPPFLAGS = @ADD_INCLS@
    3138libwandio_la_LIBADD = @LIBWANDIO_LIBS@
    32 libwandio_la_LDFLAGS=-version-info 1:0:0 @ADD_LDFLAGS@
     39libwandio_la_LDFLAGS=-version-info 1:1:0 @ADD_LDFLAGS@
    3340
     41bin_PROGRAMS = wandiocat
     42wandiocat_SOURCES = wcat.c
     43wandiocat_CFLAGS = -I"$(top_srcdir)/libwandio"
     44wandiocat_CXXFLAGS = -I"$(top_srcdir)/libwandio"
     45wandiocat_LDFLAGS = -L"$(top_srcdir)/libwandio" -lwandio
  • libwandio/iow-lzo.c

    r2a7047c rfa7faf3  
    274274}
    275275
    276 iow_t *lzo_wopen(iow_t *child, int compress_level UNUSED)
     276iow_t *lzo_wopen(iow_t *child, int compress_level)
    277277{
    278278        const int opt_filter = 0;
     
    290290                return NULL;
    291291        }
     292
     293        /* Compress level is useless for LZO, but getting UNUSED into here
     294         * is more trouble than it is worth so this check will at least
     295         * stop us from getting warnings about it.
     296         */
     297        if (compress_level < 0)
     298                return NULL;
    292299
    293300        iow = malloc(sizeof(iow_t));
  • libwandio/wandio.c

    r10f924c r4b0cd2f  
    4646 */
    4747
    48 struct compression_type compression_type[]  = {
    49         { "GZ",         "gz",   WANDIO_COMPRESS_ZLIB    },
    50         { "BZ2",        "bz2",  WANDIO_COMPRESS_BZ2     },
    51         { "LZO",        "lzo",  WANDIO_COMPRESS_LZO     },
     48struct wandio_compression_type compression_type[]  = {
     49        { "gzip",       "gz",   WANDIO_COMPRESS_ZLIB    },
     50        { "bzip2",      "bz2",  WANDIO_COMPRESS_BZ2     },
     51        { "lzo",        "lzo",  WANDIO_COMPRESS_LZO     },
     52        { "lzma",       "xz",  WANDIO_COMPRESS_LZMA     },
    5253        { "NONE",       "",     WANDIO_COMPRESS_NONE    }
    5354};
     
    141142        DEBUG_PIPELINE("peek");
    142143        io_t *io = peek_open(stdio_open(filename));
    143         char buffer[1024];
     144        unsigned char buffer[1024];
    144145        int len;
    145146        if (!io)
     
    151152
    152153        if (autodetect) {
    153                 if (len>=3 && buffer[0] == '\037' && buffer[1] == '\213' &&
     154                if (len>=3 && buffer[0] == 0x1f && buffer[1] == 0x8b &&
    154155                                buffer[2] == 0x08) {
    155156#if HAVE_LIBZ
     
    162163                }
    163164                /* Auto detect compress(1) compressed data (gzip can read this) */
    164                 if (len>=2 && buffer[0] == '\037' && buffer[1] == '\235') {
     165                if (len>=2 && buffer[0] == 0x1f && buffer[1] == 0x9d) {
    165166#if HAVE_LIBZ
    166167                        DEBUG_PIPELINE("zlib");
     
    182183#endif
    183184                }
     185
     186                if (len >=5 && buffer[0] == 0xfd && buffer[1] == '7' &&
     187                                buffer[2] == 'z' && buffer[3] == 'X' &&
     188                                buffer[4] == 'Z') {
     189#if HAVE_LIBLZMA
     190                        DEBUG_PIPELINE("lzma");
     191                        io = lzma_open(io);
     192#else
     193                        fprintf(stderr, "File %s is lzma compressed but libtrace has not been built with lzma support!\n", filename);
     194                        return NULL;
     195#endif
     196                }
    184197        }       
    185198        /* Now open a threaded, peekable reader using the appropriate module
     
    193206        DEBUG_PIPELINE("peek");
    194207        return peek_open(io);
     208}
     209
     210DLLEXPORT struct wandio_compression_type *wandio_lookup_compression_type(
     211        const char *name) {
     212
     213        struct wandio_compression_type *wct = compression_type;
     214
     215        while (strcmp(wct->name, "NONE") != 0) {
     216                if (strcmp(wct->name, name) == 0)
     217                        return wct;
     218                wct++;
     219        }
     220
     221        return NULL;
    195222}
    196223
     
    291318        }
    292319#endif
     320#if HAVE_LIBLZMA
     321        else if (compression_level != 0 &&
     322            compress_type == WANDIO_COMPRESS_LZMA) {
     323                iow = lzma_wopen(iow,compression_level);
     324        }
     325#endif
    293326        /* Open a threaded writer */
    294327        if (use_threads)
  • libwandio/wandio.h

    rc7021d9 r4b0cd2f  
    6666
    6767/** Structure defining a supported compression method */
    68 struct compression_type {
     68struct wandio_compression_type {
    6969        /** Name of the compression method */
    7070        const char *name;
     
    7777
    7878/** The list of supported compression methods */
    79 extern struct compression_type compression_type[];
     79extern struct wandio_compression_type compression_type[];
    8080
    8181/** Structure defining a libtrace IO reader module */
     
    178178        /** LZO compression */
    179179        WANDIO_COMPRESS_LZO     = 3,
     180        /** LZMA compression */
     181        WANDIO_COMPRESS_LZMA    = 4,
    180182        /** All supported methods - used as a bitmask */
    181183        WANDIO_COMPRESS_MASK    = 7
     
    193195io_t *zlib_open(io_t *parent);
    194196io_t *thread_open(io_t *parent);
     197io_t *lzma_open(io_t *parent);
    195198io_t *peek_open(io_t *parent);
    196199io_t *stdio_open(const char *filename);
     
    199202iow_t *bz_wopen(iow_t *child, int compress_level);
    200203iow_t *lzo_wopen(iow_t *child, int compress_level);
     204iow_t *lzma_wopen(iow_t *child, int compress_level);
    201205iow_t *thread_wopen(iow_t *child);
    202206iow_t *stdio_wopen(const char *filename, int fileflags);
     
    211215 *
    212216 * @{ */
     217
     218/** Given a string describing the compression method, finds the internal
     219  * data structure representing that method. This is mostly useful for
     220  * nicely mapping a method name to the internal libwandio compression
     221  * method enum when configuring an output file.
     222  *
     223  * @param name          The compression method name as a string, e.g. "gzip",
     224  *                      "bzip2", "lzo" or "lzma".
     225  * @return A pointer to the compression_type structure representing the
     226  * compression method or NULL if no match can be found.
     227  *
     228  */
     229struct wandio_compression_type *wandio_lookup_compression_type(const char *name);
    213230
    214231/** Creates a new libtrace IO reader and opens the provided file for reading.
  • test/Makefile

    r59ef093 r17c5749  
    1818
    1919BINS = test-pcap-bpf test-event test-time test-dir test-wireless test-errors \
    20         test-plen test-autodetect $(BINS_DATASTRUCT) $(BINS_PARALLEL)
     20        test-plen test-autodetect test-ports test-fragment test-live test-live-snaplen \
     21        $(BINS_DATASTRUCT) $(BINS_PARALLEL)
    2122
    2223.PHONY: all clean distclean install depend test
  • test/do-tests-datastruct.sh

    • Property mode changed from 100644 to 100755
  • test/do-tests-parallel.sh

    • Property mode changed from 100644 to 100755
  • test/do-tests.sh

    r5692bc4 r95b6218  
    7272echo \* Testing payload length
    7373do_test ./test-plen
     74
     75echo \* Testing port numbers
     76do_test ./test-ports
     77
     78echo \* Testing fragment parsing
     79do_test ./test-fragment
    7480
    7581echo \* Testing event framework
     
    221227echo " * format autodetection - bzip2"
    222228do_test ./test-autodetect traces/5_packets.erf.bz2
     229echo " * format autodetection - lzma"
     230do_test ./test-autodetect traces/5_packets.erf.xz
    223231
    224232echo
  • tools/traceanon/traceanon.1

    rd6dc0f6 r17f954f  
    8686.BI \-\^\-compress-type=method
    8787compress the output trace using the compression algorithm "method". Possible
    88 algorithms are "gzip", "bzip2", "lzo" and "none". Default is "none".
     88algorithms are "gzip", "bzip2", "lzo", "xz" and "none". Default is "none".
    8989
    9090.SH EXAMPLES
  • tools/traceanon/traceanon.c

    rf5b5cca r17f954f  
    219219        } else if (strncmp(compress_type_str, "lzo", 3) == 0) {
    220220                compress_type = TRACE_OPTION_COMPRESSTYPE_LZO;
     221        } else if (strncmp(compress_type_str, "xz", 2) == 0) {
     222                compress_type = TRACE_OPTION_COMPRESSTYPE_LZMA;
    221223        } else if (strncmp(compress_type_str, "no", 2) == 0) {
    222224                compress_type = TRACE_OPTION_COMPRESSTYPE_NONE;
  • tools/tracemerge/tracemerge.1

    rd6dc0f6 r17f954f  
    5353.BI \-\^\-compress-type method
    5454Describes the compression algorithm to be used when writing the output trace.
    55 Possible methods are "gzip", "bzip2", "lzo" and "none". Defaults to "none".
     55Possible methods are "gzip", "bzip2", "lzo", "xz" and "none". Defaults to
     56"none".
    5657
    5758
  • tools/tracemerge/tracemerge.c

    rc0ccccd r17f954f  
    120120        } else if (strncmp(compress_type_str, "lzo", 3) == 0) {
    121121                compress_type = TRACE_OPTION_COMPRESSTYPE_LZO;
     122        } else if (strncmp(compress_type_str, "xz", 2) == 0) {
     123                compress_type = TRACE_OPTION_COMPRESSTYPE_LZMA;
    122124        } else if (strncmp(compress_type_str, "no", 2) == 0) {
    123125                compress_type = TRACE_OPTION_COMPRESSTYPE_NONE;
  • tools/tracesplit/tracesplit.1

    rd6dc0f6 r17f954f  
    6161\fB-Z\fR compression-method
    6262Compress the data using the specified compression algorithm. Accepted methods
    63 are "gzip", "bzip2", "lzo" or "none". Default value is none unless a
     63are "gzip", "bzip2", "lzo", "xz" or "none". Default value is none unless a
    6464compression level is specified, in which case gzip will be used.
    6565
  • tools/tracesplit/tracesplit.c

    r755855a r17f954f  
    316316        } else if (strncmp(compress_type_str, "lzo", 3) == 0) {
    317317                compress_type = TRACE_OPTION_COMPRESSTYPE_LZO;
     318        } else if (strncmp(compress_type_str, "xz", 2) == 0) {
     319                compress_type = TRACE_OPTION_COMPRESSTYPE_LZMA;
    318320        } else if (strncmp(compress_type_str, "no", 2) == 0) {
    319321                compress_type = TRACE_OPTION_COMPRESSTYPE_NONE;
Note: See TracChangeset for help on using the changeset viewer.