source: libwandio/wandio.c @ d57ae6f

4.0.1-hotfixescachetimestampsdevelopdpdk-ndagetsilivegetfragoffhelplibtrace4ndag_formatpfringrc-4.0.1rc-4.0.2rc-4.0.3rc-4.0.4ringdecrementfixringperformanceringtimestampfixes
Last change on this file since d57ae6f was 0acfd1e, checked in by Shane Alcock <salcock@…>, 8 years ago
  • Added a wandio_create_uncompressed function to the libwandio API. It's there for people who are running into problems with the compression autodetection incorrectly thinking their uncompressed files are actually compressed. Hopefully, this should be very very rare, but it can't hurt to have it right?
  • Property mode set to 100644
File size: 7.5 KB
Line 
1/*
2 * This file is part of libtrace
3 *
4 * Copyright (c) 2007,2008,2009,2010 The University of Waikato, Hamilton,
5 * New Zealand.
6 *
7 * Authors: Daniel Lawson
8 *          Perry Lorier
9 *          Shane Alcock
10 *         
11 * All rights reserved.
12 *
13 * This code has been developed by the University of Waikato WAND
14 * research group. For further information please see http://www.wand.net.nz/
15 *
16 * libtrace is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 *
21 * libtrace is distributed in the hope that it will be useful,
22 * but WITHOUT ANY WARRANTY; without even the implied warranty of
23 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24 * GNU General Public License for more details.
25 *
26 * You should have received a copy of the GNU General Public License
27 * along with libtrace; if not, write to the Free Software
28 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29 *
30 * $Id$
31 *
32 */
33
34
35#include "config.h"
36#include "wandio.h"
37#include <stdlib.h>
38#include <assert.h>
39#include <errno.h>
40#include <inttypes.h>
41#include <string.h>
42
43/* This file contains the implementation of the libtrace IO API, which format
44 * modules should use to open, read from, write to, seek and close trace files.
45 */
46
47struct compression_type compression_type[]  = {
48        { "GZ",         "gz",   WANDIO_COMPRESS_ZLIB    },
49        { "BZ2",        "bz2",  WANDIO_COMPRESS_BZ2     },
50        { "LZO",        "lzo",  WANDIO_COMPRESS_LZO     },
51        { "NONE",       "",     WANDIO_COMPRESS_NONE    }
52};
53
54int keep_stats = 0;
55int force_directio_write = 0;
56int force_directio_read = 0;
57unsigned int use_threads = -1;
58unsigned int max_buffers = 50;
59
60uint64_t read_waits = 0;
61uint64_t write_waits = 0;
62
63/** Parse an option.
64 * stats -- Show summary stats
65 * directwrite -- bypass the diskcache on write
66 * directread -- bypass the diskcache on read
67 * nothreads -- Don't use threads
68 * threads=n -- Use a maximum of 'n' threads for thread farms
69 */
70static void do_option(const char *option)
71{
72        if (*option == '\0') 
73                ;
74        else if (strcmp(option,"stats") == 0)
75                keep_stats = 1;
76        /*
77        else if (strcmp(option,"directwrite") == 0)
78                force_directio_write = 1;
79        else if (strcmp(option,"directread") == 0)
80                force_directio_read  = 1;
81        */
82        else if (strcmp(option,"nothreads") == 0)
83                use_threads = 0;
84        else if (strncmp(option,"threads=",8) == 0)
85                use_threads = atoi(option+8);
86        else if (strncmp(option,"buffers=",8) == 0)
87                max_buffers = atoi(option+8);
88        else {
89                fprintf(stderr,"Unknown libtraceio debug option '%s'\n", option);
90        }
91}
92
93static void parse_env(void)
94{
95        const char *str = getenv("LIBTRACEIO");
96        char option[1024];
97        const char *ip;
98        char *op;
99
100        if (!str)
101                return;
102
103        for(ip=str, op=option; *ip!='\0' && op < option+sizeof(option); ++ip) {
104                if (*ip == ',') {
105                        *op='\0';
106                        do_option(option);
107                        op=option;
108                }
109                else
110                        *(op++) = *ip;
111        }
112        *op='\0';
113        do_option(option);
114}
115
116
117#define READ_TRACE 0
118#define WRITE_TRACE 0
119#define PIPELINE_TRACE 0
120
121#if PIPELINE_TRACE
122#define DEBUG_PIPELINE(x) fprintf(stderr,"PIPELINE: %s\n",x)
123#else
124#define DEBUG_PIPELINE(x)
125#endif
126
127static io_t *create_io_reader(const char *filename, int autodetect)
128{
129        parse_env();
130
131        /* Use a peeking reader to look at the start of the trace file and
132         * determine what type of compression may have been used to write
133         * the file */
134
135        DEBUG_PIPELINE("stdio");
136        DEBUG_PIPELINE("peek");
137        io_t *io = peek_open(stdio_open(filename));
138        char buffer[1024];
139        int len;
140        if (!io)
141                return NULL;
142        len = wandio_peek(io, buffer, sizeof(buffer));
143        /* Auto detect gzip compressed data -- if autodetect is false,
144         * instead we just assume uncompressed.
145         */
146
147        if (autodetect) {
148                if (len>=3 && buffer[0] == '\037' && buffer[1] == '\213' &&
149                                buffer[2] == 0x08) { 
150#if HAVE_LIBZ
151                        DEBUG_PIPELINE("zlib");
152                        io = zlib_open(io);
153#else
154                        fprintf(stderr, "File %s is gzip compressed but libtrace has not been built with zlib support!\n", filename);
155                        return NULL;
156#endif
157                }
158                /* Auto detect compress(1) compressed data (gzip can read this) */
159                if (len>=2 && buffer[0] == '\037' && buffer[1] == '\235') {
160#if HAVE_LIBZ
161                        DEBUG_PIPELINE("zlib");
162                        io = zlib_open(io);
163#else
164                        fprintf(stderr, "File %s is compress(1) compressed but libtrace has not been built with zlib support!\n", filename);
165                        return NULL;
166#endif
167                }
168
169                /* Auto detect bzip compressed data */
170                if (len>=3 && buffer[0] == 'B' && buffer[1] == 'Z' && buffer[2] == 'h') { 
171#if HAVE_LIBBZ2
172                        DEBUG_PIPELINE("bzip");
173                        io = bz_open(io);
174#else
175                        fprintf(stderr, "File %s is bzip compressed but libtrace has not been built with bzip2 support!\n", filename);
176                        return NULL;
177#endif
178                }
179        }       
180        /* Now open a threaded, peekable reader using the appropriate module
181         * to read the data */
182
183        if (use_threads) {
184                DEBUG_PIPELINE("thread");
185                io = thread_open(io);
186        }
187       
188        DEBUG_PIPELINE("peek");
189        return peek_open(io);
190}
191
192DLLEXPORT io_t *wandio_create(const char *filename) {
193        return create_io_reader(filename, 1);
194}
195
196DLLEXPORT io_t *wandio_create_uncompressed(const char *filename) {
197        return create_io_reader(filename, 0);
198}
199
200
201DLLEXPORT off_t wandio_tell(io_t *io)
202{
203        if (!io->source->tell) {
204                errno = -ENOSYS;
205                return -1;
206        }
207        return io->source->tell(io);
208}
209
210DLLEXPORT off_t wandio_seek(io_t *io, off_t offset, int whence)
211{
212        if (!io->source->seek) {
213                errno = -ENOSYS;
214                return -1;
215        }
216        return io->source->seek(io,offset,whence);
217}
218
219DLLEXPORT off_t wandio_read(io_t *io, void *buffer, off_t len)
220{ 
221        off_t ret;
222        ret=io->source->read(io,buffer,len); 
223#if READ_TRACE
224        fprintf(stderr,"%p: read(%s): %d bytes = %d\n",io,io->source->name, (int)len,(int)ret);
225#endif
226        return ret;
227}
228
229DLLEXPORT off_t wandio_peek(io_t *io, void *buffer, off_t len)
230{
231        off_t ret;
232        assert(io->source->peek); /* If this fails, it means you're calling
233                                   * peek on something that doesn't support
234                                   * peeking.   Push a peek_open() on the io
235                                   * first.
236                                   */
237        ret=io->source->peek(io, buffer, len);
238#if READ_TRACE
239        fprintf(stderr,"%p: peek(%s): %d bytes = %d\n",io,io->source->name, (int)len, (int)ret);
240#endif
241        return ret;
242}
243
244DLLEXPORT void wandio_destroy(io_t *io)
245{ 
246        if (!io)
247                return;
248       
249        if (keep_stats) 
250                fprintf(stderr,"LIBTRACEIO STATS: %"PRIu64" blocks on read\n", read_waits);
251        io->source->close(io); 
252}
253
254DLLEXPORT iow_t *wandio_wcreate(const char *filename, int compress_type, int compression_level, int flags)
255{
256        iow_t *iow;
257        parse_env();
258
259        assert ( compression_level >= 0 && compression_level <= 9 );
260        assert (compress_type != WANDIO_COMPRESS_MASK);
261
262        iow=stdio_wopen(filename, flags);
263        if (!iow)
264                return NULL;
265
266        /* We prefer zlib if available, otherwise we'll use bzip. If neither
267         * are present, guess we'll just have to write uncompressed */
268#if HAVE_LIBZ
269        if (compression_level != 0 && 
270            compress_type == WANDIO_COMPRESS_ZLIB) {
271                iow = zlib_wopen(iow,compression_level);
272        }
273#endif
274#if HAVE_LIBLZO2
275        else if (compression_level != 0 && 
276            compress_type == WANDIO_COMPRESS_LZO) {
277                iow = lzo_wopen(iow,compression_level);
278        }
279#endif
280#if HAVE_LIBBZ2
281        else if (compression_level != 0 && 
282            compress_type == WANDIO_COMPRESS_BZ2) {
283                iow = bz_wopen(iow,compression_level);
284        }
285#endif
286        /* Open a threaded writer */
287        if (use_threads)
288                return thread_wopen(iow);
289        else
290                return iow;
291}
292
293DLLEXPORT off_t wandio_wwrite(iow_t *iow, const void *buffer, off_t len)
294{
295#if WRITE_TRACE
296        fprintf(stderr,"wwrite(%s): %d bytes\n",iow->source->name, (int)len);
297#endif
298        return iow->source->write(iow,buffer,len);     
299}
300
301DLLEXPORT void wandio_wdestroy(iow_t *iow)
302{
303        iow->source->close(iow);
304        if (keep_stats) 
305                fprintf(stderr,"LIBTRACEIO STATS: %"PRIu64" blocks on write\n", write_waits);
306}
307
Note: See TracBrowser for help on using the repository browser.