source: libwandio/ior-peek.c @ e4eff86

4.0.1-hotfixescachetimestampsdevelopdpdk-ndagetsilivelibtrace4ndag_formatpfringrc-4.0.1rc-4.0.2rc-4.0.3rc-4.0.4ringdecrementfixringperformanceringtimestampfixes
Last change on this file since e4eff86 was e4eff86, checked in by Shane Alcock <salcock@…>, 5 years ago

Ensure all libwandio code includes config.h

This fixes a nasty bug on 32 bit machines where the size of off_t will
change between functions, causing all sorts of havoc. The reason the size
changes is because FILE_OFFSET_BITS is defined to 64 inside config.h so
any source files that include config.h will have 64 bit off_t's whereas
any files that don't include it will end up with a 32 bit off_t (on a 32 bit
machine).

  • Property mode set to 100644
File size: 8.7 KB
Line 
1/*
2 * This file is part of libtrace
3 *
4 * Copyright (c) 2007,2008,2009,2010 The University of Waikato, Hamilton,
5 * New Zealand.
6 *
7 * Authors: Daniel Lawson
8 *          Perry Lorier
9 *          Shane Alcock
10 *         
11 * All rights reserved.
12 *
13 * This code has been developed by the University of Waikato WAND
14 * research group. For further information please see http://www.wand.net.nz/
15 *
16 * libtrace is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License as published by
18 * the Free Software Foundation; either version 2 of the License, or
19 * (at your option) any later version.
20 *
21 * libtrace is distributed in the hope that it will be useful,
22 * but WITHOUT ANY WARRANTY; without even the implied warranty of
23 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
24 * GNU General Public License for more details.
25 *
26 * You should have received a copy of the GNU General Public License
27 * along with libtrace; if not, write to the Free Software
28 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
29 *
30 * $Id$
31 *
32 */
33
34#include "config.h"
35#include "wandio.h"
36#include <sys/types.h>
37#include <sys/stat.h>
38#include <fcntl.h>
39#include <stdlib.h>
40#include <unistd.h>
41#include <string.h>
42#include <assert.h>
43#include <stddef.h>
44
45/* Libtrace IO module implementing a peeking reader.
46 *
47 * Assuming my understanding of Perry's code is correct, this module provides
48 * generic support for "peeking" that can be used in concert with any other
49 * implemented IO reader.
50 *
51 * The other IO reader is a "child" to the peeking reader and is used to read
52 * the data into a buffer managed by the peeking reader. Any actual "peeks"
53 * are serviced from the managed buffer, which means that we do not have to
54 * manipulate the read offsets directly in zlib or bzip, for instance.
55 */
56
57/* for O_DIRECT we have to read in multiples of this */
58#define MIN_READ_SIZE 4096
59/* Round reads for peeks into the buffer up to this size */
60#define PEEK_SIZE (1024*1024)
61
62struct peek_t {
63        io_t *child;
64        char *buffer;
65        off_t length; /* Length of buffer */
66        off_t offset; /* Offset into buffer */
67};
68
69extern io_source_t peek_source;
70
71#define DATA(io) ((struct peek_t *)((io)->data))
72#define MIN(a,b) ((a) < (b) ? (a) : (b))
73
74io_t *peek_open(io_t *child)
75{
76        io_t *io;
77        if (!child)
78                return NULL;
79        io =  malloc(sizeof(io_t));
80        io->data = malloc(sizeof(struct peek_t));
81        io->source = &peek_source;
82
83        /* Wrap the peeking reader around the "child" */
84        DATA(io)->child = child;
85        DATA(io)->buffer = NULL;
86        DATA(io)->length = 0;
87        DATA(io)->offset = 0;   
88
89        return io;
90}
91
92/* Read at least "len" bytes from the child io into the internal buffer, and return how many
93   bytes was actually read.
94 */
95static off_t refill_buffer(io_t *io, off_t len)
96{
97        off_t bytes_read;
98        assert(DATA(io)->length - DATA(io)->offset == 0);
99        /* Select the largest of "len", PEEK_SIZE and the current peek buffer size
100         * and then round up to the nearest multiple of MIN_READ_SIZE
101         */
102        bytes_read = len < PEEK_SIZE ? PEEK_SIZE : len;
103        bytes_read = bytes_read < DATA(io)->length ? DATA(io)->length : bytes_read;
104        bytes_read += MIN_READ_SIZE - (bytes_read % MIN_READ_SIZE);
105        /* Is the current buffer big enough? */
106        if (DATA(io)->length < bytes_read) {
107                int res = 0;
108                void *buf_ptr = (void *)(DATA(io)->buffer);
109
110                if (buf_ptr)
111                        free(buf_ptr);
112                DATA(io)->length = bytes_read;
113                DATA(io)->offset = 0;
114#if _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600
115                /* We need to do this as read() of O_DIRECT might happen into
116                 * this buffer.  The docs suggest 512 bytes is all we need to
117                 * align to, but I'm suspicious. I expect disks with 4k blocks
118                 * will arrive soon, and thus 4k is the minimum I'm willing to
119                 * live with.
120                 */
121                res = posix_memalign(&buf_ptr, 4096, DATA(io)->length);
122                if (res != 0) {
123                        fprintf(stderr, "Error aligning IO buffer: %d\n",
124                                        res);
125                        return res;
126                }
127                DATA(io)->buffer = buf_ptr;
128#else
129                res = 0;        /* << Silly warning */ 
130                DATA(io)->buffer = malloc(DATA(io)->length);
131#endif
132        }
133        else
134                DATA(io)->length = bytes_read;
135
136        assert(DATA(io)->buffer);
137
138        /* Now actually attempt to read that many bytes */
139        bytes_read = DATA(io)->child->source->read(     
140                        DATA(io)->child, DATA(io)->buffer, bytes_read);
141
142        DATA(io)->offset = 0;
143        DATA(io)->length = bytes_read;
144       
145        /* Error? */
146        if (bytes_read < 1)
147                return bytes_read;
148
149        return bytes_read;
150       
151}
152
153static off_t peek_read(io_t *io, void *buffer, off_t len)
154{
155        off_t ret = 0;
156
157        /* Is some of this data in the buffer? */
158        if (DATA(io)->buffer) {
159                ret = MIN(len,DATA(io)->length - DATA(io)->offset);
160
161                /* Copy anything we've got into their buffer, and shift our
162                 * offset so that we don't peek at the data we've read again */
163                memcpy(buffer, 
164                        DATA(io)->buffer + DATA(io)->offset,
165                        ret);
166                buffer += ret;
167                DATA(io)->offset += ret;
168                len -= ret;
169        }
170
171        /* Use the child reader to get the rest of the required data */
172        if (len>0) {
173                /* To get here, the buffer must be empty */
174                assert(DATA(io)->length-DATA(io)->offset == 0);
175                off_t bytes_read;
176                /* If they're reading exactly a block size, just use that, no point in malloc'ing
177                 * and memcpy()ing needlessly.  However, if the buffer isn't aligned, we need to
178                 * pass on an aligning buffer, skip this and do it into our own aligned buffer.
179                 */
180                if ((len % MIN_READ_SIZE  == 0) && ((ptrdiff_t)buffer % 4096)==0) {
181                        assert(((ptrdiff_t)buffer % 4096) == 0);
182                        bytes_read = DATA(io)->child->source->read(
183                                        DATA(io)->child, buffer, len);
184                        /* Error? */
185                        if (bytes_read < 1) {
186                                /* Return if we have managed to get some data ok */
187                                if (ret > 0)
188                                        return ret;
189                                /* Return the error upstream */
190                                return bytes_read;
191                        }
192                }
193                else {
194                        bytes_read = refill_buffer(io, len);
195                        if (bytes_read < 1) {
196                                /* Return if we have managed to get some data ok */
197                                if (ret > 0)
198                                        return ret;
199                                /* Return the error upstream */
200                                return bytes_read;
201                        }
202                        /* Now grab the number of bytes asked for. */
203                        len = len < bytes_read ? len : bytes_read;
204                        memcpy(buffer, DATA(io)->buffer, len);
205
206                        DATA(io)->offset = len;
207                        bytes_read = len;
208                }
209                ret += bytes_read;
210        }
211
212        /* Have we read past the end of the buffer? */
213        if (DATA(io)->buffer && DATA(io)->offset >= DATA(io)->length) {
214                /* If so, free the memory it used */
215                free(DATA(io)->buffer);
216                DATA(io)->buffer = NULL;
217                DATA(io)->offset = 0;
218                DATA(io)->length = 0;
219        }
220
221        return ret;
222}
223
224static void *alignedrealloc(void *old, size_t oldsize, size_t size, int *res)
225{
226#if _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600
227        void *new;
228        /* Shortcut resizing */
229        if (size < oldsize)
230                return old;
231        *res = posix_memalign(&new, 4096, size);
232        if (*res != 0) {
233                fprintf(stderr, "Error aligning IO buffer: %d\n", *res);
234               
235                return NULL;
236        }
237        assert(oldsize<size);
238        memcpy(new,old,oldsize);
239        free(old);
240        return new;
241#else
242        /* These no-ops are to stop the compiler whinging about unused
243         * parameters */
244        (void)oldsize;
245        (void)res;
246        return realloc(old,size);
247#endif
248}
249
250
251static off_t peek_peek(io_t *io, void *buffer, off_t len)
252{
253        off_t ret = 0;
254        int res = 0;
255
256        /* Is there enough data in the buffer to serve this request? */
257        if (DATA(io)->length - DATA(io)->offset < len) {
258                /* No, we need to extend the buffer. */
259                off_t read_amount = len - (DATA(io)->length - DATA(io)->offset);
260                /* Round the read_amount up to the nearest MB */
261                read_amount += PEEK_SIZE - ((DATA(io)->length + read_amount) % PEEK_SIZE);
262                DATA(io)->buffer = alignedrealloc(DATA(io)->buffer, 
263                        DATA(io)->length, 
264                        DATA(io)->length + read_amount, &res);
265
266                if (DATA(io)->buffer == NULL) {
267                        return res;     
268                }
269
270                /* Use the child reader to read more data into our managed
271                 * buffer */
272                read_amount = wandio_read(DATA(io)->child, 
273                        DATA(io)->buffer + DATA(io)->length,
274                        read_amount);
275
276                /* Pass errors up */
277                if (read_amount <0) {
278                        return read_amount;
279                }
280
281                DATA(io)->length += read_amount;
282        }
283
284        /* Right, now return data from the buffer (that now should be large
285         * enough, but might not be if we hit EOF) */
286        ret = MIN(len, DATA(io)->length - DATA(io)->offset);
287        memcpy(buffer, DATA(io)->buffer + DATA(io)->offset, ret);
288        return ret;
289}
290
291static off_t peek_tell(io_t *io)
292{
293        /* We don't actually maintain a read offset as such, so we want to
294         * return the child's read offset */
295        return wandio_tell(DATA(io)->child);
296}
297
298static off_t peek_seek(io_t *io, off_t offset, int whence)
299{
300        /* Again, we don't have a genuine read offset so we need to pass this
301         * one on to the child */
302        return wandio_seek(DATA(io)->child,offset,whence);
303}
304
305static void peek_close(io_t *io)
306{
307        /* Make sure we close the child that is doing the actual reading! */
308        wandio_destroy(DATA(io)->child);
309        if (DATA(io)->buffer)
310                free(DATA(io)->buffer);
311        free(io->data);
312        free(io);
313}
314
315io_source_t peek_source = {
316        "peek",
317        peek_read,
318        peek_peek,
319        peek_tell,
320        peek_seek,
321        peek_close
322};
323
Note: See TracBrowser for help on using the repository browser.