source: libwandio/wandio.c @ 4b0cd2f

4.0.1-hotfixescachetimestampsdevelopdpdk-ndagetsilivelibtrace4ndag_formatpfringrc-4.0.1rc-4.0.2rc-4.0.3rc-4.0.4ringdecrementfixringperformanceringtimestampfixes
Last change on this file since 4b0cd2f was 4b0cd2f, checked in by Shane Alcock <salcock@…>, 6 years ago

Integrate lzma into libwandio properly

Update configure to detect lzma properly.
Add missing get_compression_type API function to wandio code.
Add code to detect if input file is LZMA.

  • Property mode set to 100644
File size: 8.9 KB
Line 
1/*
2 * This file is part of libtrace
3 *
4 * Copyright (c) 2007,2008,2009,2010 The University of Waikato, Hamilton,
5 * New Zealand.
6 *
7 * Authors: Daniel Lawson
8 *          Perry Lorier
9 *          Shane Alcock
10 *         
11 * All rights reserved.
12 *
13 * This code has been developed by the University of Waikato WAND
14 * research group. For further information please see http://www.wand.net.nz/
15 *
16 * libtrace is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 *
21 * libtrace is distributed in the hope that it will be useful,
22 * but WITHOUT ANY WARRANTY; without even the implied warranty of
23 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24 * GNU General Public License for more details.
25 *
26 * You should have received a copy of the GNU General Public License
27 * along with libtrace; if not, write to the Free Software
28 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29 *
30 * $Id$
31 *
32 */
33
34
35#include "config.h"
36#include "wandio_internal.h"
37#include "wandio.h"
38#include <stdlib.h>
39#include <assert.h>
40#include <errno.h>
41#include <inttypes.h>
42#include <string.h>
43
44/* This file contains the implementation of the libtrace IO API, which format
45 * modules should use to open, read from, write to, seek and close trace files.
46 */
47
48struct wandio_compression_type compression_type[]  = {
49        { "gzip",       "gz",   WANDIO_COMPRESS_ZLIB    },
50        { "bzip2",      "bz2",  WANDIO_COMPRESS_BZ2     },
51        { "lzo",        "lzo",  WANDIO_COMPRESS_LZO     },
52        { "lzma",       "xz",  WANDIO_COMPRESS_LZMA     },
53        { "NONE",       "",     WANDIO_COMPRESS_NONE    }
54};
55
56int keep_stats = 0;
57int force_directio_write = 0;
58int force_directio_read = 0;
59int use_autodetect = 1;
60unsigned int use_threads = -1;
61unsigned int max_buffers = 50;
62
63uint64_t read_waits = 0;
64uint64_t write_waits = 0;
65
66/** Parse an option.
67 * stats -- Show summary stats
68 * directwrite -- bypass the diskcache on write
69 * directread -- bypass the diskcache on read
70 * noautodetect -- disable autodetection of file compression, assume all files
71 *                 are uncompressed
72 * nothreads -- Don't use threads
73 * threads=n -- Use a maximum of 'n' threads for thread farms
74 */
75static void do_option(const char *option)
76{
77        if (*option == '\0') 
78                ;
79        else if (strcmp(option,"stats") == 0)
80                keep_stats = 1;
81        /*
82        else if (strcmp(option,"directwrite") == 0)
83                force_directio_write = 1;
84        else if (strcmp(option,"directread") == 0)
85                force_directio_read  = 1;
86        */
87        else if (strcmp(option,"nothreads") == 0)
88                use_threads = 0;
89        else if (strcmp(option,"noautodetect") == 0)
90                use_autodetect = 0;
91        else if (strncmp(option,"threads=",8) == 0)
92                use_threads = atoi(option+8);
93        else if (strncmp(option,"buffers=",8) == 0)
94                max_buffers = atoi(option+8);
95        else {
96                fprintf(stderr,"Unknown libtraceio debug option '%s'\n", option);
97        }
98}
99
100static void parse_env(void)
101{
102        const char *str = getenv("LIBTRACEIO");
103        char option[1024];
104        const char *ip;
105        char *op;
106
107        if (!str)
108                return;
109
110        for(ip=str, op=option; *ip!='\0' && op < option+sizeof(option); ++ip) {
111                if (*ip == ',') {
112                        *op='\0';
113                        do_option(option);
114                        op=option;
115                }
116                else
117                        *(op++) = *ip;
118        }
119        *op='\0';
120        do_option(option);
121}
122
123
124#define READ_TRACE 0
125#define WRITE_TRACE 0
126#define PIPELINE_TRACE 0
127
128#if PIPELINE_TRACE
129#define DEBUG_PIPELINE(x) fprintf(stderr,"PIPELINE: %s\n",x)
130#else
131#define DEBUG_PIPELINE(x)
132#endif
133
134static io_t *create_io_reader(const char *filename, int autodetect)
135{
136
137        /* Use a peeking reader to look at the start of the trace file and
138         * determine what type of compression may have been used to write
139         * the file */
140
141        DEBUG_PIPELINE("stdio");
142        DEBUG_PIPELINE("peek");
143        io_t *io = peek_open(stdio_open(filename));
144        unsigned char buffer[1024];
145        int len;
146        if (!io)
147                return NULL;
148        len = wandio_peek(io, buffer, sizeof(buffer));
149        /* Auto detect gzip compressed data -- if autodetect is false,
150         * instead we just assume uncompressed.
151         */
152
153        if (autodetect) {
154                if (len>=3 && buffer[0] == 0x1f && buffer[1] == 0x8b &&
155                                buffer[2] == 0x08) { 
156#if HAVE_LIBZ
157                        DEBUG_PIPELINE("zlib");
158                        io = zlib_open(io);
159#else
160                        fprintf(stderr, "File %s is gzip compressed but libtrace has not been built with zlib support!\n", filename);
161                        return NULL;
162#endif
163                }
164                /* Auto detect compress(1) compressed data (gzip can read this) */
165                if (len>=2 && buffer[0] == 0x1f && buffer[1] == 0x9d) {
166#if HAVE_LIBZ
167                        DEBUG_PIPELINE("zlib");
168                        io = zlib_open(io);
169#else
170                        fprintf(stderr, "File %s is compress(1) compressed but libtrace has not been built with zlib support!\n", filename);
171                        return NULL;
172#endif
173                }
174
175                /* Auto detect bzip compressed data */
176                if (len>=3 && buffer[0] == 'B' && buffer[1] == 'Z' && buffer[2] == 'h') { 
177#if HAVE_LIBBZ2
178                        DEBUG_PIPELINE("bzip");
179                        io = bz_open(io);
180#else
181                        fprintf(stderr, "File %s is bzip compressed but libtrace has not been built with bzip2 support!\n", filename);
182                        return NULL;
183#endif
184                }
185
186                if (len >=5 && buffer[0] == 0xfd && buffer[1] == '7' && 
187                                buffer[2] == 'z' && buffer[3] == 'X' &&
188                                buffer[4] == 'Z') {
189#if HAVE_LIBLZMA
190                        DEBUG_PIPELINE("lzma");
191                        io = lzma_open(io);
192#else
193                        fprintf(stderr, "File %s is lzma compressed but libtrace has not been built with lzma support!\n", filename);
194                        return NULL;
195#endif
196                }
197        }       
198        /* Now open a threaded, peekable reader using the appropriate module
199         * to read the data */
200
201        if (use_threads) {
202                DEBUG_PIPELINE("thread");
203                io = thread_open(io);
204        }
205       
206        DEBUG_PIPELINE("peek");
207        return peek_open(io);
208}
209
210DLLEXPORT struct wandio_compression_type *wandio_lookup_compression_type(
211        const char *name) {
212
213        struct wandio_compression_type *wct = compression_type;
214
215        while (strcmp(wct->name, "NONE") != 0) {
216                if (strcmp(wct->name, name) == 0)
217                        return wct;
218                wct++;
219        }
220
221        return NULL;
222}
223
224DLLEXPORT io_t *wandio_create(const char *filename) {
225        parse_env();
226        return create_io_reader(filename, use_autodetect);
227}
228
229DLLEXPORT io_t *wandio_create_uncompressed(const char *filename) {
230        parse_env();
231        return create_io_reader(filename, 0);
232}
233
234
235DLLEXPORT off_t wandio_tell(io_t *io)
236{
237        if (!io->source->tell) {
238                errno = -ENOSYS;
239                return -1;
240        }
241        return io->source->tell(io);
242}
243
244DLLEXPORT off_t wandio_seek(io_t *io, off_t offset, int whence)
245{
246        if (!io->source->seek) {
247                errno = -ENOSYS;
248                return -1;
249        }
250        return io->source->seek(io,offset,whence);
251}
252
253DLLEXPORT off_t wandio_read(io_t *io, void *buffer, off_t len)
254{ 
255        off_t ret;
256        ret=io->source->read(io,buffer,len); 
257#if READ_TRACE
258        fprintf(stderr,"%p: read(%s): %d bytes = %d\n",io,io->source->name, (int)len,(int)ret);
259#endif
260        return ret;
261}
262
263DLLEXPORT off_t wandio_peek(io_t *io, void *buffer, off_t len)
264{
265        off_t ret;
266        assert(io->source->peek); /* If this fails, it means you're calling
267                                   * peek on something that doesn't support
268                                   * peeking.   Push a peek_open() on the io
269                                   * first.
270                                   */
271        ret=io->source->peek(io, buffer, len);
272#if READ_TRACE
273        fprintf(stderr,"%p: peek(%s): %d bytes = %d\n",io,io->source->name, (int)len, (int)ret);
274#endif
275        return ret;
276}
277
278DLLEXPORT void wandio_destroy(io_t *io)
279{ 
280        if (!io)
281                return;
282       
283        if (keep_stats) 
284                fprintf(stderr,"LIBTRACEIO STATS: %"PRIu64" blocks on read\n", read_waits);
285        io->source->close(io); 
286}
287
288DLLEXPORT iow_t *wandio_wcreate(const char *filename, int compress_type, int compression_level, int flags)
289{
290        iow_t *iow;
291        parse_env();
292
293        assert ( compression_level >= 0 && compression_level <= 9 );
294        assert (compress_type != WANDIO_COMPRESS_MASK);
295
296        iow=stdio_wopen(filename, flags);
297        if (!iow)
298                return NULL;
299
300        /* We prefer zlib if available, otherwise we'll use bzip. If neither
301         * are present, guess we'll just have to write uncompressed */
302#if HAVE_LIBZ
303        if (compression_level != 0 && 
304            compress_type == WANDIO_COMPRESS_ZLIB) {
305                iow = zlib_wopen(iow,compression_level);
306        }
307#endif
308#if HAVE_LIBLZO2
309        else if (compression_level != 0 && 
310            compress_type == WANDIO_COMPRESS_LZO) {
311                iow = lzo_wopen(iow,compression_level);
312        }
313#endif
314#if HAVE_LIBBZ2
315        else if (compression_level != 0 && 
316            compress_type == WANDIO_COMPRESS_BZ2) {
317                iow = bz_wopen(iow,compression_level);
318        }
319#endif
320#if HAVE_LIBLZMA
321        else if (compression_level != 0 && 
322            compress_type == WANDIO_COMPRESS_LZMA) {
323                iow = lzma_wopen(iow,compression_level);
324        }
325#endif
326        /* Open a threaded writer */
327        if (use_threads)
328                return thread_wopen(iow);
329        else
330                return iow;
331}
332
333DLLEXPORT off_t wandio_wwrite(iow_t *iow, const void *buffer, off_t len)
334{
335#if WRITE_TRACE
336        fprintf(stderr,"wwrite(%s): %d bytes\n",iow->source->name, (int)len);
337#endif
338        return iow->source->write(iow,buffer,len);     
339}
340
341DLLEXPORT void wandio_wdestroy(iow_t *iow)
342{
343        iow->source->close(iow);
344        if (keep_stats) 
345                fprintf(stderr,"LIBTRACEIO STATS: %"PRIu64" blocks on write\n", write_waits);
346}
347
Note: See TracBrowser for help on using the repository browser.