Changeset 2697eab

Timestamp:

06/30/14 13:23:39 (6 years ago)

Author:

Shane Alcock <salcock@…>

Branches:

4.0.1-hotfixes, cachetimestamps, develop, dpdk-ndag, etsilive, libtrace4, master, ndag_format, pfring, rc-4.0.1, rc-4.0.2, rc-4.0.3, rc-4.0.4, ringdecrementfix, ringperformance, ringtimestampfixes

Children:

b4b8e55

Parents:

d1cb537 (diff), d19ff50 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

Merge remote branch 'origin/develop' into develop

Files:

• configure.in (modified) (3 diffs)
• libwandio/Makefile.am (modified) (2 diffs)
• libwandio/wandio.c (modified) (7 diffs)
• libwandio/wandio.h (modified) (6 diffs)
• libwandio/wcat.c (modified) (1 diff)

configure.in

@@ -580 +580 @@
 )
 
+AC_ARG_WITH([lzma],
+        AC_HELP_STRING([--with-lzma], [build with support for lzma compressed files]))
+
+AS_IF([test "x$with_lzma" != "xno"],
+        [
+        AC_CHECK_LIB(lzma, lzma_code, have_lzma=yes, have_lzma=no)
+        ], [have_lzma=no])
+
+AS_IF([test "x$have_lzma" = "xyes"], [
+        if test "$ac_cv_lib_lzma_code" != "none required"; then
+                LIBWANDIO_LIBS="$LIBWANDIO_LIBS -llzma"
+        fi
+        AC_DEFINE(HAVE_LIBLZMA, 1, "Compiled with lzma support")
+        with_lzma=yes],
+
+        
+        [AS_IF([test "x$with_lzma" = "xyes"],
+                [AC_MSG_ERROR([lzma requested but not found])])
+        AC_DEFINE(HAVE_LIBLZMA, 0, "Compiled with lzma support")
+        with_lzma=no]
+)
 
 # Define automake conditionals for use in our Makefile.am files
@@ -594 +615 @@
 AM_CONDITIONAL([HAVE_ZLIB], [test "x$with_zlib" != "xno"])
 AM_CONDITIONAL([HAVE_LZO], [ test "x$with_lzo" != "xno"])
+AM_CONDITIONAL([HAVE_LZMA], [ test "x$with_lzma" != "xno"])
 
 # Check for miscellaneous programs
@@ -648 +670 @@
 reportopt "Compiled with compressed trace (bz2) support" $with_bzip2
 reportopt "Compiled with compressed trace (lzo write only) support" $with_lzo
+reportopt "Compiled with compressed trace (lzma) support" $with_lzma
 if test x"$libtrace_dag" = xtrue; then
         if test "$libtrace_dag_version" = 24; then

libwandio/Makefile.am

@@ -24 +24 @@
 endif
 
+if HAVE_LZMA
+LIBTRACEIO_LZMA=ior-lzma.c iow-lzma.c
+else
+LIBTRACEIO_LZMA=
+endif
+
 libwandio_la_SOURCES=wandio.c ior-peek.c ior-stdio.c ior-thread.c \
                 iow-stdio.c iow-thread.c wandio.h wandio_internal.h \
-                $(LIBTRACEIO_ZLIB) $(LIBTRACEIO_BZLIB) $(LIBTRACEIO_LZO)
+                $(LIBTRACEIO_ZLIB) $(LIBTRACEIO_BZLIB) $(LIBTRACEIO_LZO) \
+                $(LIBTRACEIO_LZMA)
 
 AM_CPPFLAGS = @ADD_INCLS@
@@ -32 +39 @@
 libwandio_la_LDFLAGS=-version-info 1:0:0 @ADD_LDFLAGS@
 
+bin_PROGRAMS = wandiocat
+wandiocat_SOURCES = wcat.c
+wandiocat_CFLAGS = -I"$(top_srcdir)/libwandio"
+wandiocat_CXXFLAGS = -I"$(top_srcdir)/libwandio"
+wandiocat_LDFLAGS = -L"$(top_srcdir)/libwandio" -lwandio

libwandio/wandio.c

@@ -46 +46 @@
  */
 
-struct compression_type compression_type[]  = {
-        { "GZ",         "gz",   WANDIO_COMPRESS_ZLIB    },
-        { "BZ2",        "bz2",  WANDIO_COMPRESS_BZ2     },
-        { "LZO",        "lzo",  WANDIO_COMPRESS_LZO     },
+struct wandio_compression_type compression_type[]  = {
+        { "gzip",       "gz",   WANDIO_COMPRESS_ZLIB    },
+        { "bzip2",      "bz2",  WANDIO_COMPRESS_BZ2     },
+        { "lzo",        "lzo",  WANDIO_COMPRESS_LZO     },
+        { "lzma",       "xz",  WANDIO_COMPRESS_LZMA     },
         { "NONE",       "",     WANDIO_COMPRESS_NONE    }
 };
@@ -141 +142 @@
         DEBUG_PIPELINE("peek");
         io_t *io = peek_open(stdio_open(filename));
-        char buffer[1024];
+        unsigned char buffer[1024];
         int len;
         if (!io)
@@ -151 +152 @@
 
         if (autodetect) {
-                if (len>=3 && buffer[0] == '\037' && buffer[1] == '\213' &&
+                if (len>=3 && buffer[0] == 0x1f && buffer[1] == 0x8b &&
                                 buffer[2] == 0x08) { 
 #if HAVE_LIBZ
@@ -162 +163 @@
                 }
                 /* Auto detect compress(1) compressed data (gzip can read this) */
-                if (len>=2 && buffer[0] == '\037' && buffer[1] == '\235') {
+                if (len>=2 && buffer[0] == 0x1f && buffer[1] == 0x9d) {
 #if HAVE_LIBZ
                         DEBUG_PIPELINE("zlib");
@@ -182 +183 @@
 #endif
                 }
+
+                if (len >=5 && buffer[0] == 0xfd && buffer[1] == '7' && 
+                                buffer[2] == 'z' && buffer[3] == 'X' &&
+                                buffer[4] == 'Z') {
+#if HAVE_LIBLZMA
+                        DEBUG_PIPELINE("lzma");
+                        io = lzma_open(io);
+#else
+                        fprintf(stderr, "File %s is lzma compressed but libtrace has not been built with lzma support!\n", filename);
+                        return NULL;
+#endif
+                }
         }       
         /* Now open a threaded, peekable reader using the appropriate module
@@ -193 +206 @@
         DEBUG_PIPELINE("peek");
         return peek_open(io);
+}
+
+DLLEXPORT struct wandio_compression_type *wandio_lookup_compression_type(
+        const char *name) {
+
+        struct wandio_compression_type *wct = compression_type;
+
+        while (strcmp(wct->name, "NONE") != 0) {
+                if (strcmp(wct->name, name) == 0)
+                        return wct;
+                wct++;
+        }
+
+        return NULL;
 }
 
@@ -291 +318 @@
         }
 #endif
+#if HAVE_LIBLZMA
+        else if (compression_level != 0 && 
+            compress_type == WANDIO_COMPRESS_LZMA) {
+                iow = lzma_wopen(iow,compression_level);
+        }
+#endif
         /* Open a threaded writer */
         if (use_threads)

libwandio/wandio.h

@@ -66 +66 @@
 
 /** Structure defining a supported compression method */
-struct compression_type {
+struct wandio_compression_type {
         /** Name of the compression method */
         const char *name;
@@ -77 +77 @@
 
 /** The list of supported compression methods */
-extern struct compression_type compression_type[];
+extern struct wandio_compression_type compression_type[];
 
 /** Structure defining a libtrace IO reader module */
@@ -178 +178 @@
         /** LZO compression */
         WANDIO_COMPRESS_LZO     = 3,
+        /** LZMA compression */
+        WANDIO_COMPRESS_LZMA    = 4,
         /** All supported methods - used as a bitmask */
         WANDIO_COMPRESS_MASK    = 7
@@ -193 +195 @@
 io_t *zlib_open(io_t *parent);
 io_t *thread_open(io_t *parent);
+io_t *lzma_open(io_t *parent);
 io_t *peek_open(io_t *parent);
 io_t *stdio_open(const char *filename);
@@ -199 +202 @@
 iow_t *bz_wopen(iow_t *child, int compress_level);
 iow_t *lzo_wopen(iow_t *child, int compress_level);
+iow_t *lzma_wopen(iow_t *child, int compress_level);
 iow_t *thread_wopen(iow_t *child);
 iow_t *stdio_wopen(const char *filename, int fileflags);
@@ -211 +215 @@
  *
  * @{ */
+
+/** Given a string describing the compression method, finds the internal
+  * data structure representing that method. This is mostly useful for
+  * nicely mapping a method name to the internal libwandio compression
+  * method enum when configuring an output file.
+  *
+  * @param name          The compression method name as a string, e.g. "gzip",
+  *                      "bzip2", "lzo" or "lzma".
+  * @return A pointer to the compression_type structure representing the
+  * compression method or NULL if no match can be found.
+  *
+  */
+struct wandio_compression_type *wandio_lookup_compression_type(const char *name);
 
 /** Creates a new libtrace IO reader and opens the provided file for reading.

libwandio/wcat.c

@@ -6 +6 @@
 #include <string.h>
 
+static void printhelp() {
+        printf("wandiocat: concatenate files into a single compressed file\n");
+        printf("\n");
+        printf("Available options:\n\n");
+        printf(" -z <level>\n");
+        printf("    Sets a compression level for the output file, must be \n");
+        printf("    between 0 (uncompressed) and 9 (max compression)\n"); 
+        printf("    Default is 0.\n");
+        printf(" -Z <method>\n");
+        printf("    Set the compression method. Must be one of 'gzip', \n");
+        printf("    'bzip2', 'lzo' or 'lzma'. If not specified, no\n");
+        printf("    compression is performed.\n");
+        printf(" -o <file>\n");
+        printf("    The name of the output file. If not specified, output\n");
+        printf("    is written to standard output.\n");
+}
+
+
 int main(int argc, char *argv[])
 {
-    int compress_level = 0;
-    int compress_type = WANDIO_COMPRESS_NONE;
-    char *output = "-";
-    char c;
-    while ((c = getopt (argc, argv, "Z:z:o:")) != -1) {
-        switch (c)
-        {
-            case 'Z':
-                compress_type = wandio_lookup_compression_type(optarg)
-                        ->compress_type;
-                break;
-            case 'z':
-                compress_level = atoi(optarg);
-                break;
-            case 'o':
-                output = optarg;
-                 break;
-           case '?':
-             if (optopt == 'Z' || optopt == 'z' || optopt == 'o')
-               fprintf (stderr, "Option -%c requires an argument.\n", optopt);
-             else if (isprint (optopt))
-               fprintf (stderr, "Unknown option `-%c'.\n", optopt);
-             else
-               fprintf (stderr,
-                        "Unknown option character `\\x%x'.\n",
-                        optopt);
-             return 1;
-           default:
-             abort ();
-           }
-    }
-
-    iow_t *iow = wandio_wcreate(output, compress_type, compress_level, 0);
-                /* stdout */
-    int i;
-    for(i=optind; i<argc; ++i) {
-        char buffer[1024*1024];
-        io_t *ior = wandio_create(argv[i]);
-        if (!ior) {
-            fprintf(stderr, "Failed to open %s\n", argv[i]);
-            continue;
+        int compress_level = 0;
+        int compress_type = WANDIO_COMPRESS_NONE;
+        char *output = "-";
+        char c;
+        while ((c = getopt (argc, argv, "Z:z:o:h")) != -1) {
+                switch (c)
+                {
+                case 'Z':
+                        compress_type = wandio_lookup_compression_type(optarg)
+                                        ->compress_type;
+                        break;
+                case 'z':
+                        compress_level = atoi(optarg);
+                        break;
+                case 'o':
+                        output = optarg;
+                        break;
+                case 'h':
+                        printhelp();
+                        return 0;
+                case '?':
+                        if (optopt == 'Z' || optopt == 'z' || optopt == 'o')
+                                fprintf (stderr, "Option -%c requires an argument.\n", optopt);
+                        else if (isprint (optopt))
+                                fprintf (stderr, "Unknown option `-%c'.\n", optopt);
+                        else
+                                fprintf (stderr, "Unknown option character `\\x%x'.\n", optopt);
+                        return 1;
+                default:
+                        abort ();
+                }
         }
 
-        off_t len;
-        do {
-            len = wandio_read(ior, buffer, sizeof(buffer));
-            if (len > 0)
-                wandio_wwrite(iow, buffer, len);
-        } while(len > 0);
+        iow_t *iow = wandio_wcreate(output, compress_type, compress_level, 0);
+        /* stdout */
+        int i;
+        for(i=optind; i<argc; ++i) {
+                char buffer[1024*1024];
+                io_t *ior = wandio_create(argv[i]);
+                if (!ior) {
+                        fprintf(stderr, "Failed to open %s\n", argv[i]);
+                        continue;
+                }
 
-        wandio_destroy(ior);
-    }
-    wandio_wdestroy(iow);
-    return 0;
+                off_t len;
+                do {
+                        len = wandio_read(ior, buffer, sizeof(buffer));
+                        if (len > 0)
+                                wandio_wwrite(iow, buffer, len);
+                } while(len > 0);
+
+                wandio_destroy(ior);
+        }
+        wandio_wdestroy(iow);
+        return 0;
 }