Slim numerical data compression 1.0
bitstream.h
Go to the documentation of this file.
1// -*- mode: c++; -*-
2
5
6// Copyright (C) 2008, 2009 Joseph Fowler
7//
8// This file is part of slim, a compression package for science data.
9//
10// Slim is free software: you can redistribute it and/or modify
11// it under the terms of the GNU General Public License as published by
12// the Free Software Foundation, either version 3 of the License, or
13// (at your option) any later version.
14//
15// Slim is distributed in the hope that it will be useful,
16// but WITHOUT ANY WARRANTY; without even the implied warranty of
17// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18// GNU General Public License for more details.
19//
20// You should have received a copy of the GNU General Public License
21// along with slim. If not, see <http://www.gnu.org/licenses/>.
22
23#ifndef SLIM_BITSTREAM_H
24#define SLIM_BITSTREAM_H
25
26#include <stdint.h>
27#include <cstdio>
28
29#ifdef HAVE_LIBZZIP
30#include <zzip/zzip.h>
31#endif
32
33#ifdef HAVE_LIBLZ4
34#include <lz4frame.h>
35#endif
36
37#include "bit_constants.h"
38
39using namespace std;
40
41class bitstream {
42public:
43 bitstream();
44 bitstream(const char *filename,
45 int buffersize=DEFAULT_IOBUFFER_SIZE);
46 virtual ~bitstream();
47
48 virtual void close() = 0;
49 virtual bool is_open() const = 0;
50 virtual void setupstream();
51 virtual void windup()=0;
52 virtual int get_bytes_used();
53 int get_bitptr();
54 virtual void print() const = 0;
55
56protected:
57 static const int Bits_per_word = 8*sizeof(Word_t);
58
59 size_t bufsize;
60 size_t buf_used;
61 Byte_t *buffer_base;
62 Byte_t *beyondbuffer;
63 union {
64 Byte_t *Bptr;
65 Word_t *Dptr;
67 int bitptr;
70
71public:
72 enum {DEFAULT_IOBUFFER_SIZE=1024*1024};
73 enum {MAX_BITSTREAM_BUFSIZE=16*1024*1024};
74};
75
76
77
78class obitstream : public bitstream {
79private:
80 // No private attributes.
81
82protected:
83 FILE *fp;
84
85public:
86 obitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
87 obitstream(const char *filename,
88 int buffersize=DEFAULT_IOBUFFER_SIZE);
90
91 void writebits(uint32_t data, int nbits);
92 void writestring(const char *str, bool write_trailing_null=false);
93 template <typename T> void writeword(const T data);
94 void write_unary(unsigned int value);
95 virtual void print() const;
96 virtual void close();
97 virtual bool is_open() const;
98 void windup();
99 void flush(bool flush_trailing_bits);
100};
101
102
103
104
105class ibitstream : public bitstream {
106private:
107 // No private attributes (unless debugging).
108#ifdef DEBUG_READBITS
109 int c1,c2, c3, c4, c5, c6;
110 void print_debug();
111#endif
112
113protected:
114#ifdef HAVE_LIBZZIP
115 ZZIP_FILE *zfp;
116#endif
117 FILE *fp;
118#ifdef HAVE_LIBLZ4
119 LZ4F_decompressionContext_t lz4_ctx;
120 Byte_t *lz4_buffer;
121 bool using_lz4;
122#endif
123
124public:
125 ibitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
126#ifdef HAVE_LIBZZIP
127 ibitstream(ZZIP_FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
128#endif
129 ibitstream(const char *filename,
130 int buffersize=DEFAULT_IOBUFFER_SIZE);
131 ibitstream(int fd, int buffersize=DEFAULT_IOBUFFER_SIZE);
132 ~ibitstream();
133
134 virtual void close();
135 virtual bool is_open() const;
136 void setupstream();
137 void windup();
138 virtual void print() const;
139 virtual int get_bytes_used();
140 Word_t readbits(int nbits);
141 int32_t readbits_int(int nbits);
142 Word_t read_unary();
143 int readstring(char *s, int count=-1);
144 //int get_bits_used() { return bitptr + Bits_per_word*buf_used;}
145
146private:
147 void next_word();
148 int fill();
149
150 Word_t partial_word;
151 int partial_word_bitptr;
152};
153
154
155
157// Inline functions
159
166static inline unsigned int bit_size(int32_t i) {
167 if (i < 0)
168 i = (-i)-1; // Convert negative int to non-negatives of same size.
169
170 if (i>lowestNset32bits[15]) {
171 if (i>lowestNset32bits[23]) {
172 if (i>lowestNset32bits[27]) {
173 if (i>lowestNset32bits[29]) {
174 if (i>lowestNset32bits[30]) {
175 return 32;
176 } else {
177 return 31;
178 }
179 } else {
180 if (i>lowestNset32bits[28]) {
181 return 30;
182 } else {
183 return 29;
184 }
185 }
186 } else {
187 if (i>lowestNset32bits[25]) {
188 if (i>lowestNset32bits[26]) {
189 return 28;
190 } else {
191 return 27;
192 }
193 } else {
194 if (i>lowestNset32bits[24]) {
195 return 26;
196 } else {
197 return 25;
198 }
199 }
200 }
201 } else {
202 if (i>lowestNset32bits[19]) {
203 if (i>lowestNset32bits[21]) {
204 if (i>lowestNset32bits[22]) {
205 return 24;
206 } else {
207 return 23;
208 }
209 } else {
210 if (i>lowestNset32bits[20]) {
211 return 22;
212 } else {
213 return 21;
214 }
215 }
216 } else {
217 if (i>lowestNset32bits[17]) {
218 if (i>lowestNset32bits[18]) {
219 return 20;
220 } else {
221 return 19;
222 }
223 } else {
224 if (i>lowestNset32bits[16]) {
225 return 18;
226 } else {
227 return 17;
228 }
229 }
230 }
231 }
232 } else {
233 if (i>lowestNset32bits[7]) {
234 if (i>lowestNset32bits[11]) {
235 if (i>lowestNset32bits[13]) {
236 if (i>lowestNset32bits[14]) {
237 return 16;
238 } else {
239 return 15;
240 }
241 } else {
242 if (i>lowestNset32bits[12]) {
243 return 14;
244 } else {
245 return 13;
246 }
247 }
248 } else {
249 if (i>lowestNset32bits[9]) {
250 if (i>lowestNset32bits[10]) {
251 return 12;
252 } else {
253 return 11;
254 }
255 } else {
256 if (i>lowestNset32bits[8]) {
257 return 10;
258 } else {
259 return 9;
260 }
261 }
262 }
263 } else {
264 if (i>lowestNset32bits[3]) {
265 if (i>lowestNset32bits[5]) {
266 if (i>lowestNset32bits[6]) {
267 return 8;
268 } else {
269 return 7;
270 }
271 } else {
272 if (i>lowestNset32bits[4]) {
273 return 6;
274 } else {
275 return 5;
276 }
277 }
278 } else {
279 if (i>lowestNset32bits[1]) {
280 if (i>lowestNset32bits[2]) {
281 return 4;
282 } else {
283 return 3;
284 }
285 } else {
286 if (i>lowestNset32bits[0]) {
287 return 2;
288 } else {
289 return 1;
290 }
291 }
292 }
293 }
294 }
295}
296
297
298
303static inline unsigned int bit_size(unsigned int u) {
304 for (int bs=1; bs<=32; bs++)
305 if (u == (u&lowestNset32bits[bs]))
306 return bs;
307 throw "Bit size (unsigned int) fails!";
308}
309
310
311#endif // #ifdef SLIM_BITSTREAM_H
static unsigned int bit_size(int32_t i)
Find size (on [0,32]) of the smallest # that can hold the integer i.
Definition bitstream.h:166
Bit stream base class.
Definition bitstream.h:41
static const int Bits_per_word
Bits per buffer word.
Definition bitstream.h:57
size_t bufsize
Size of I/O buffer (bytes)
Definition bitstream.h:59
Byte_t * Bptr
Pointer to the current word (as Byte_t *).
Definition bitstream.h:64
int get_bitptr()
Get the position of the bitptr.
Definition bitstream.cpp:170
bitstream()
Dummy default constructor.
Definition bitstream.cpp:46
virtual void setupstream()
Allocate a buffer and set up all pointers.
Definition bitstream.cpp:141
Byte_t * buffer_base
Pointer to the buffer.
Definition bitstream.h:61
size_t buf_used
Definition bitstream.h:60
int bitptr
Pointer to the current bits.
Definition bitstream.h:67
virtual void print() const =0
Print properties of stream (pure virt)
virtual int get_bytes_used()
Return the number of bytes used so far in this stream.
Definition bitstream.cpp:162
union bitstream::@0 buffptr
Pointer to the current word.
virtual ~bitstream()
Destructor deletes output buffer, closes file.
Definition bitstream.cpp:91
Word_t * Dptr
Pointer to the current word (as Word_t *).
Definition bitstream.h:65
Byte_t * beyondbuffer
Pointer just beyond buffer (convenience).
Definition bitstream.h:62
Input bit stream.
Definition bitstream.h:105
~ibitstream()
Destructor only uses base class destructor.
Definition bitstream.cpp:479
ibitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE)
Start ibitstream using a FILE ptr to an open file.
Definition bitstream.cpp:380
Word_t readbits(int nbits)
Read data from the buffer as unsigned ints.
Definition bitstream.cpp:556
void windup()
Put the current word to the buffer with upper 0 bits as needed.
Definition bitstream.cpp:543
virtual bool is_open() const
Is the IO file closed?
Definition bitstream.cpp:130
int readstring(char *s, int count=-1)
Read a null-terminated string of 8-bit characters from the bit stream.
Definition bitstream.cpp:590
Word_t read_unary()
Read a single unary-coded value.
Definition bitstream.cpp:622
virtual void close()
Close the IO file.
Definition bitstream.cpp:102
int fill()
Fill the read buffer and reset for more data.
Definition bitstream.cpp:661
virtual int get_bytes_used()
Return the number of bytes used so far in this stream.
Definition bitstream.cpp:509
void next_word()
Move to next word in buffer, refilling it if needed.
Definition bitstream.cpp:529
int32_t readbits_int(int nbits)
Read data from the buffer as (signed) ints.
Definition bitstream.cpp:607
void setupstream()
Allocate a buffer and set up all pointers, then fill buffer.
Definition bitstream.cpp:517
virtual void print() const
Print buffer contents.
Definition bitstream.cpp:636
Output bit stream.
Definition bitstream.h:78
void write_unary(unsigned int value)
Write a unary code for the value.
Definition bitstream.cpp:303
obitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE)
Start obitstream using a FILE ptr to an open file.
Definition bitstream.cpp:182
void writestring(const char *str, bool write_trailing_null=false)
Write 8-bit character strings to the buffer.
Definition bitstream.cpp:273
virtual void close()
Close the output stream by flushing and closing the FILE *.
Definition bitstream.cpp:211
void windup()
Put the current word to the buffer with upper 0 bits as needed.
Definition bitstream.cpp:224
virtual bool is_open() const
Is the IO file closed?
Definition bitstream.cpp:124
void flush(bool flush_trailing_bits)
Flush the write buffer and reset for more data.
Definition bitstream.cpp:351
void writebits(uint32_t data, int nbits)
Write data to the buffer.
Definition bitstream.cpp:237
void writeword(const T data)
Write all the bits of a word.
Definition bitstream.cpp:284
virtual void print() const
Print buffer contents.
Definition bitstream.cpp:320
FILE * fp
The I/O stream.
Definition bitstream.h:83