1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // Author: brianolson@google.com (Brian Olson)
32 //
33 // This file contains the implementation of classes GzipInputStream and
34 // GzipOutputStream.
35
36
37 #if HAVE_ZLIB
38 #include <google/protobuf/io/gzip_stream.h>
39
40 #include <google/protobuf/stubs/common.h>
41 #include <google/protobuf/stubs/logging.h>
42
43 namespace google {
44 namespace protobuf {
45 namespace io {
46
47 static const int kDefaultBufferSize = 65536;
48
GzipInputStream(ZeroCopyInputStream * sub_stream,Format format,int buffer_size)49 GzipInputStream::GzipInputStream(ZeroCopyInputStream* sub_stream, Format format,
50 int buffer_size)
51 : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) {
52 zcontext_.state = Z_NULL;
53 zcontext_.zalloc = Z_NULL;
54 zcontext_.zfree = Z_NULL;
55 zcontext_.opaque = Z_NULL;
56 zcontext_.total_out = 0;
57 zcontext_.next_in = NULL;
58 zcontext_.avail_in = 0;
59 zcontext_.total_in = 0;
60 zcontext_.msg = NULL;
61 if (buffer_size == -1) {
62 output_buffer_length_ = kDefaultBufferSize;
63 } else {
64 output_buffer_length_ = buffer_size;
65 }
66 output_buffer_ = operator new(output_buffer_length_);
67 GOOGLE_CHECK(output_buffer_ != NULL);
68 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
69 zcontext_.avail_out = output_buffer_length_;
70 output_position_ = output_buffer_;
71 }
~GzipInputStream()72 GzipInputStream::~GzipInputStream() {
73 operator delete(output_buffer_);
74 zerror_ = inflateEnd(&zcontext_);
75 }
76
internalInflateInit2(z_stream * zcontext,GzipInputStream::Format format)77 static inline int internalInflateInit2(z_stream* zcontext,
78 GzipInputStream::Format format) {
79 int windowBitsFormat = 0;
80 switch (format) {
81 case GzipInputStream::GZIP:
82 windowBitsFormat = 16;
83 break;
84 case GzipInputStream::AUTO:
85 windowBitsFormat = 32;
86 break;
87 case GzipInputStream::ZLIB:
88 windowBitsFormat = 0;
89 break;
90 }
91 return inflateInit2(zcontext, /* windowBits */ 15 | windowBitsFormat);
92 }
93
Inflate(int flush)94 int GzipInputStream::Inflate(int flush) {
95 if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) {
96 // previous inflate filled output buffer. don't change input params yet.
97 } else if (zcontext_.avail_in == 0) {
98 const void* in;
99 int in_size;
100 bool first = zcontext_.next_in == NULL;
101 bool ok = sub_stream_->Next(&in, &in_size);
102 if (!ok) {
103 zcontext_.next_out = NULL;
104 zcontext_.avail_out = 0;
105 return Z_STREAM_END;
106 }
107 zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in));
108 zcontext_.avail_in = in_size;
109 if (first) {
110 int error = internalInflateInit2(&zcontext_, format_);
111 if (error != Z_OK) {
112 return error;
113 }
114 }
115 }
116 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
117 zcontext_.avail_out = output_buffer_length_;
118 output_position_ = output_buffer_;
119 int error = inflate(&zcontext_, flush);
120 return error;
121 }
122
DoNextOutput(const void ** data,int * size)123 void GzipInputStream::DoNextOutput(const void** data, int* size) {
124 *data = output_position_;
125 *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_);
126 output_position_ = zcontext_.next_out;
127 }
128
129 // implements ZeroCopyInputStream ----------------------------------
Next(const void ** data,int * size)130 bool GzipInputStream::Next(const void** data, int* size) {
131 bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
132 (zerror_ == Z_BUF_ERROR);
133 if ((!ok) || (zcontext_.next_out == NULL)) {
134 return false;
135 }
136 if (zcontext_.next_out != output_position_) {
137 DoNextOutput(data, size);
138 return true;
139 }
140 if (zerror_ == Z_STREAM_END) {
141 if (zcontext_.next_out != NULL) {
142 // sub_stream_ may have concatenated streams to follow
143 zerror_ = inflateEnd(&zcontext_);
144 byte_count_ += zcontext_.total_out;
145 if (zerror_ != Z_OK) {
146 return false;
147 }
148 zerror_ = internalInflateInit2(&zcontext_, format_);
149 if (zerror_ != Z_OK) {
150 return false;
151 }
152 } else {
153 *data = NULL;
154 *size = 0;
155 return false;
156 }
157 }
158 zerror_ = Inflate(Z_NO_FLUSH);
159 if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) {
160 // The underlying stream's Next returned false inside Inflate.
161 return false;
162 }
163 ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
164 (zerror_ == Z_BUF_ERROR);
165 if (!ok) {
166 return false;
167 }
168 DoNextOutput(data, size);
169 return true;
170 }
BackUp(int count)171 void GzipInputStream::BackUp(int count) {
172 output_position_ = reinterpret_cast<void*>(
173 reinterpret_cast<uintptr_t>(output_position_) - count);
174 }
Skip(int count)175 bool GzipInputStream::Skip(int count) {
176 const void* data;
177 int size = 0;
178 bool ok = Next(&data, &size);
179 while (ok && (size < count)) {
180 count -= size;
181 ok = Next(&data, &size);
182 }
183 if (size > count) {
184 BackUp(size - count);
185 }
186 return ok;
187 }
ByteCount() const188 int64 GzipInputStream::ByteCount() const {
189 int64 ret = byte_count_ + zcontext_.total_out;
190 if (zcontext_.next_out != NULL && output_position_ != NULL) {
191 ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
192 reinterpret_cast<uintptr_t>(output_position_);
193 }
194 return ret;
195 }
196
197 // =========================================================================
198
Options()199 GzipOutputStream::Options::Options()
200 : format(GZIP),
201 buffer_size(kDefaultBufferSize),
202 compression_level(Z_DEFAULT_COMPRESSION),
203 compression_strategy(Z_DEFAULT_STRATEGY) {}
204
GzipOutputStream(ZeroCopyOutputStream * sub_stream)205 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) {
206 Init(sub_stream, Options());
207 }
208
GzipOutputStream(ZeroCopyOutputStream * sub_stream,const Options & options)209 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream,
210 const Options& options) {
211 Init(sub_stream, options);
212 }
213
Init(ZeroCopyOutputStream * sub_stream,const Options & options)214 void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
215 const Options& options) {
216 sub_stream_ = sub_stream;
217 sub_data_ = NULL;
218 sub_data_size_ = 0;
219
220 input_buffer_length_ = options.buffer_size;
221 input_buffer_ = operator new(input_buffer_length_);
222 GOOGLE_CHECK(input_buffer_ != NULL);
223
224 zcontext_.zalloc = Z_NULL;
225 zcontext_.zfree = Z_NULL;
226 zcontext_.opaque = Z_NULL;
227 zcontext_.next_out = NULL;
228 zcontext_.avail_out = 0;
229 zcontext_.total_out = 0;
230 zcontext_.next_in = NULL;
231 zcontext_.avail_in = 0;
232 zcontext_.total_in = 0;
233 zcontext_.msg = NULL;
234 // default to GZIP format
235 int windowBitsFormat = 16;
236 if (options.format == ZLIB) {
237 windowBitsFormat = 0;
238 }
239 zerror_ =
240 deflateInit2(&zcontext_, options.compression_level, Z_DEFLATED,
241 /* windowBits */ 15 | windowBitsFormat,
242 /* memLevel (default) */ 8, options.compression_strategy);
243 }
244
~GzipOutputStream()245 GzipOutputStream::~GzipOutputStream() {
246 Close();
247 operator delete(input_buffer_);
248 }
249
250 // private
Deflate(int flush)251 int GzipOutputStream::Deflate(int flush) {
252 int error = Z_OK;
253 do {
254 if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) {
255 bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_);
256 if (!ok) {
257 sub_data_ = NULL;
258 sub_data_size_ = 0;
259 return Z_BUF_ERROR;
260 }
261 GOOGLE_CHECK_GT(sub_data_size_, 0);
262 zcontext_.next_out = static_cast<Bytef*>(sub_data_);
263 zcontext_.avail_out = sub_data_size_;
264 }
265 error = deflate(&zcontext_, flush);
266 } while (error == Z_OK && zcontext_.avail_out == 0);
267 if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) {
268 // Notify lower layer of data.
269 sub_stream_->BackUp(zcontext_.avail_out);
270 // We don't own the buffer anymore.
271 sub_data_ = NULL;
272 sub_data_size_ = 0;
273 }
274 return error;
275 }
276
277 // implements ZeroCopyOutputStream ---------------------------------
Next(void ** data,int * size)278 bool GzipOutputStream::Next(void** data, int* size) {
279 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
280 return false;
281 }
282 if (zcontext_.avail_in != 0) {
283 zerror_ = Deflate(Z_NO_FLUSH);
284 if (zerror_ != Z_OK) {
285 return false;
286 }
287 }
288 if (zcontext_.avail_in == 0) {
289 // all input was consumed. reset the buffer.
290 zcontext_.next_in = static_cast<Bytef*>(input_buffer_);
291 zcontext_.avail_in = input_buffer_length_;
292 *data = input_buffer_;
293 *size = input_buffer_length_;
294 } else {
295 // The loop in Deflate should consume all avail_in
296 GOOGLE_LOG(DFATAL) << "Deflate left bytes unconsumed";
297 }
298 return true;
299 }
BackUp(int count)300 void GzipOutputStream::BackUp(int count) {
301 GOOGLE_CHECK_GE(zcontext_.avail_in, count);
302 zcontext_.avail_in -= count;
303 }
ByteCount() const304 int64 GzipOutputStream::ByteCount() const {
305 return zcontext_.total_in + zcontext_.avail_in;
306 }
307
Flush()308 bool GzipOutputStream::Flush() {
309 zerror_ = Deflate(Z_FULL_FLUSH);
310 // Return true if the flush succeeded or if it was a no-op.
311 return (zerror_ == Z_OK) ||
312 (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
313 zcontext_.avail_out != 0);
314 }
315
Close()316 bool GzipOutputStream::Close() {
317 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
318 return false;
319 }
320 do {
321 zerror_ = Deflate(Z_FINISH);
322 } while (zerror_ == Z_OK);
323 zerror_ = deflateEnd(&zcontext_);
324 bool ok = zerror_ == Z_OK;
325 zerror_ = Z_STREAM_END;
326 return ok;
327 }
328
329 } // namespace io
330 } // namespace protobuf
331 } // namespace google
332
333 #endif // HAVE_ZLIB
334