Opened 8 years ago

#351 new defect

libtrace/libwandio tries to auto-detect compression on erf files, detects compression on non-compressed files, fails

Reported by: yuri@… Owned by: salcock
Priority: major Milestone:
Component: libtrace-library Version: 3.0
Keywords: ERF compression auto-detect Cc:

Description

First, thank you very much for a great library.

Now the bug: Since ERF format does not have a header and the first few bytes are timestamp, detecting compression reliably is problematic. libtrace (noticed first using tracesplit to convert to pcap) attempts to auto-detect compression. This mostly works, but not always. Case in point was one ERF file I tried to convert to PCAP. tracesplit failed (quietly, this is another bug). I traced the problem to the auto-detection of compression. In particular, the first two bytes of the buffer satisfied the test (buffer[0] == '\037' && buffer[1] == '\235'). Compressing the file first before passing to tracesplit fixed the issue. Here's a small proposed patch (I don't know if this is the right place to put it, but there does not seem to be another option:

diff -aur libtrace-3.0.17/lib/format_erf.c libtrace-3.0.17.new/lib/format_erf.c --- libtrace-3.0.17/lib/format_erf.c 2013-03-05 16:28:51.000000000 -0800 +++ libtrace-3.0.17.new/lib/format_erf.c 2013-05-13 14:14:03.462456692 -0700 @@ -307,7 +307,7 @@

if (DATA(libtrace)->seek.exists==INDEX_UNKNOWN) {

char buffer[PATH_MAX]; snprintf(buffer,sizeof(buffer),"%s.idx",libtrace->uridata);

  • DATA(libtrace)->seek.index=wandio_create(buffer);

+ DATA(libtrace)->seek.index=wandio_create(buffer, 0);

if (DATA(libtrace)->seek.index) {

DATA(libtrace)->seek.exists=INDEX_EXISTS;

}

diff -aur libtrace-3.0.17/lib/format_helper.c libtrace-3.0.17.new/lib/format_helper.c --- libtrace-3.0.17/lib/format_helper.c 2013-03-05 16:28:51.000000000 -0800 +++ libtrace-3.0.17.new/lib/format_helper.c 2013-05-13 14:10:10.339441279 -0700 @@ -246,7 +246,8 @@

/* Open a file for reading using the new Libtrace IO system */ io_t *trace_open_file(libtrace_t *trace) {

  • io_t *io=wandio_create(trace->uridata);

+ int autodetect = (trace->format->type != TRACE_FORMAT_ERF); + io_t *io=wandio_create(trace->uridata, autodetect);

if (!io) {

if (errno != 0) {

trace_set_err(trace,errno,"Unable to open %s",trace->uridata);

diff -aur libtrace-3.0.17/lib/trace.c libtrace-3.0.17.new/lib/trace.c --- libtrace-3.0.17/lib/trace.c 2013-03-05 16:28:51.000000000 -0800 +++ libtrace-3.0.17.new/lib/trace.c 2013-05-13 14:13:06.868724525 -0700 @@ -181,7 +181,8 @@

}

}

  • libtrace->io = wandio_create(filename);

+ int autodetect = 1; + libtrace->io = wandio_create(filename, autodetect);

if (!libtrace->io)

return;

diff -aur libtrace-3.0.17/libwandio/wandio.c libtrace-3.0.17.new/libwandio/wandio.c --- libtrace-3.0.17/libwandio/wandio.c 2012-05-27 19:07:07.000000000 -0700 +++ libtrace-3.0.17.new/libwandio/wandio.c 2013-05-13 14:04:54.814362447 -0700 @@ -124,7 +124,7 @@

#define DEBUG_PIPELINE(x) #endif

-DLLEXPORT io_t *wandio_create(const char *filename) +DLLEXPORT io_t *wandio_create(const char *filename, int autodetect)

{

parse_env();

@@ -140,38 +140,39 @@

if (!io)

return NULL;

len = wandio_peek(io, buffer, sizeof(buffer));

  • /* Auto detect gzip compressed data */
  • if (len>=2 && buffer[0] == '\037' && buffer[1] == '\213') {

+ if (autodetect) { + /* Auto detect gzip compressed data */ + if (len>=2 && buffer[0] == '\037' && buffer[1] == '\213') {

#if HAVE_LIBZ

  • DEBUG_PIPELINE("zlib");
  • io = zlib_open(io);

+ DEBUG_PIPELINE("zlib"); + io = zlib_open(io);

#else

  • fprintf(stderr, "File %s is gzip compressed but libtrace has not been built with zlib support!\n", filename);
  • return NULL;

+ fprintf(stderr, "File %s is gzip compressed but libtrace has not been built with zlib support!\n", filename); + return NULL;

#endif

  • }
  • /* Auto detect compress(1) compressed data (gzip can read this) */
  • if (len>=2 && buffer[0] == '\037' && buffer[1] == '\235') {

+ } + /* Auto detect compress(1) compressed data (gzip can read this) */ + if (len>=2 && buffer[0] == '\037' && buffer[1] == '\235') {

#if HAVE_LIBZ

  • DEBUG_PIPELINE("zlib");
  • io = zlib_open(io);

+ DEBUG_PIPELINE("zlib"); + io = zlib_open(io);

#else

  • fprintf(stderr, "File %s is compress(1) compressed but libtrace has not been built with zlib support!\n", filename);
  • return NULL;

+ fprintf(stderr, "File %s is compress(1) compressed but libtrace has not been built with zlib support!\n", filename); + return NULL;

#endif

  • }

+ }

  • /* Auto detect bzip compressed data */
  • if (len>=3 && buffer[0] == 'B' && buffer[1] == 'Z' && buffer[2] == 'h') {

+ /* Auto detect bzip compressed data */ + if (len>=3 && buffer[0] == 'B' && buffer[1] == 'Z' && buffer[2] == 'h') {

#if HAVE_LIBBZ2

  • DEBUG_PIPELINE("bzip");
  • io = bz_open(io);

+ DEBUG_PIPELINE("bzip"); + io = bz_open(io);

#else

  • fprintf(stderr, "File %s is bzip compressed but libtrace has not been built with bzip2 support!\n", filename);
  • return NULL;

+ fprintf(stderr, "File %s is bzip compressed but libtrace has not been built with bzip2 support!\n", filename); + return NULL;

#endif

+ }

}

-

/* Now open a threaded, peekable reader using the appropriate module

  • to read the data */

diff -aur libtrace-3.0.17/libwandio/wandio.h libtrace-3.0.17.new/libwandio/wandio.h --- libtrace-3.0.17/libwandio/wandio.h 2012-03-13 19:00:22.000000000 -0700 +++ libtrace-3.0.17.new/libwandio/wandio.h 2013-05-13 14:12:02.611893323 -0700 @@ -236,7 +236,7 @@

  • given file (if any), provided that libtrace was built with the appropriate
  • libraries. */

-io_t *wandio_create(const char *filename); +io_t *wandio_create(const char *filename, int autodetect);

/ Returns the current offset of the read pointer for a libtrace IO reader.

*

Change History (0)

Note: See TracTickets for help on using tickets.