• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc.  All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 //     * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 //     * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 //     * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 
31 // Author: brianolson@google.com (Brian Olson)
32 //
33 // This file contains the implementation of classes GzipInputStream and
34 // GzipOutputStream.
35 
36 
37 #if HAVE_ZLIB
38 #include <google/protobuf/io/gzip_stream.h>
39 
40 #include <google/protobuf/stubs/common.h>
41 #include <google/protobuf/stubs/logging.h>
42 
43 namespace google {
44 namespace protobuf {
45 namespace io {
46 
47 static const int kDefaultBufferSize = 65536;
48 
GzipInputStream(ZeroCopyInputStream * sub_stream,Format format,int buffer_size)49 GzipInputStream::GzipInputStream(
50     ZeroCopyInputStream* sub_stream, Format format, int buffer_size)
51     : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) {
52   zcontext_.state = Z_NULL;
53   zcontext_.zalloc = Z_NULL;
54   zcontext_.zfree = Z_NULL;
55   zcontext_.opaque = Z_NULL;
56   zcontext_.total_out = 0;
57   zcontext_.next_in = NULL;
58   zcontext_.avail_in = 0;
59   zcontext_.total_in = 0;
60   zcontext_.msg = NULL;
61   if (buffer_size == -1) {
62     output_buffer_length_ = kDefaultBufferSize;
63   } else {
64     output_buffer_length_ = buffer_size;
65   }
66   output_buffer_ = operator new(output_buffer_length_);
67   GOOGLE_CHECK(output_buffer_ != NULL);
68   zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
69   zcontext_.avail_out = output_buffer_length_;
70   output_position_ = output_buffer_;
71 }
~GzipInputStream()72 GzipInputStream::~GzipInputStream() {
73   operator delete(output_buffer_);
74   zerror_ = inflateEnd(&zcontext_);
75 }
76 
internalInflateInit2(z_stream * zcontext,GzipInputStream::Format format)77 static inline int internalInflateInit2(
78     z_stream* zcontext, GzipInputStream::Format format) {
79   int windowBitsFormat = 0;
80   switch (format) {
81     case GzipInputStream::GZIP: windowBitsFormat = 16; break;
82     case GzipInputStream::AUTO: windowBitsFormat = 32; break;
83     case GzipInputStream::ZLIB: windowBitsFormat = 0; break;
84   }
85   return inflateInit2(zcontext, /* windowBits */15 | windowBitsFormat);
86 }
87 
Inflate(int flush)88 int GzipInputStream::Inflate(int flush) {
89   if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) {
90     // previous inflate filled output buffer. don't change input params yet.
91   } else if (zcontext_.avail_in == 0) {
92     const void* in;
93     int in_size;
94     bool first = zcontext_.next_in == NULL;
95     bool ok = sub_stream_->Next(&in, &in_size);
96     if (!ok) {
97       zcontext_.next_out = NULL;
98       zcontext_.avail_out = 0;
99       return Z_STREAM_END;
100     }
101     zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in));
102     zcontext_.avail_in = in_size;
103     if (first) {
104       int error = internalInflateInit2(&zcontext_, format_);
105       if (error != Z_OK) {
106         return error;
107       }
108     }
109   }
110   zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
111   zcontext_.avail_out = output_buffer_length_;
112   output_position_ = output_buffer_;
113   int error = inflate(&zcontext_, flush);
114   return error;
115 }
116 
DoNextOutput(const void ** data,int * size)117 void GzipInputStream::DoNextOutput(const void** data, int* size) {
118   *data = output_position_;
119   *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_);
120   output_position_ = zcontext_.next_out;
121 }
122 
123 // implements ZeroCopyInputStream ----------------------------------
Next(const void ** data,int * size)124 bool GzipInputStream::Next(const void** data, int* size) {
125   bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END)
126       || (zerror_ == Z_BUF_ERROR);
127   if ((!ok) || (zcontext_.next_out == NULL)) {
128     return false;
129   }
130   if (zcontext_.next_out != output_position_) {
131     DoNextOutput(data, size);
132     return true;
133   }
134   if (zerror_ == Z_STREAM_END) {
135     if (zcontext_.next_out != NULL) {
136       // sub_stream_ may have concatenated streams to follow
137       zerror_ = inflateEnd(&zcontext_);
138       byte_count_ += zcontext_.total_out;
139       if (zerror_ != Z_OK) {
140         return false;
141       }
142       zerror_ = internalInflateInit2(&zcontext_, format_);
143       if (zerror_ != Z_OK) {
144         return false;
145       }
146     } else {
147       *data = NULL;
148       *size = 0;
149       return false;
150     }
151   }
152   zerror_ = Inflate(Z_NO_FLUSH);
153   if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) {
154     // The underlying stream's Next returned false inside Inflate.
155     return false;
156   }
157   ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END)
158       || (zerror_ == Z_BUF_ERROR);
159   if (!ok) {
160     return false;
161   }
162   DoNextOutput(data, size);
163   return true;
164 }
BackUp(int count)165 void GzipInputStream::BackUp(int count) {
166   output_position_ = reinterpret_cast<void*>(
167       reinterpret_cast<uintptr_t>(output_position_) - count);
168 }
Skip(int count)169 bool GzipInputStream::Skip(int count) {
170   const void* data;
171   int size;
172   bool ok = Next(&data, &size);
173   while (ok && (size < count)) {
174     count -= size;
175     ok = Next(&data, &size);
176   }
177   if (size > count) {
178     BackUp(size - count);
179   }
180   return ok;
181 }
ByteCount() const182 int64 GzipInputStream::ByteCount() const {
183   int64 ret = byte_count_ + zcontext_.total_out;
184   if (zcontext_.next_out != NULL && output_position_ != NULL) {
185     ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
186            reinterpret_cast<uintptr_t>(output_position_);
187   }
188   return ret;
189 }
190 
191 // =========================================================================
192 
Options()193 GzipOutputStream::Options::Options()
194     : format(GZIP),
195       buffer_size(kDefaultBufferSize),
196       compression_level(Z_DEFAULT_COMPRESSION),
197       compression_strategy(Z_DEFAULT_STRATEGY) {}
198 
GzipOutputStream(ZeroCopyOutputStream * sub_stream)199 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) {
200   Init(sub_stream, Options());
201 }
202 
GzipOutputStream(ZeroCopyOutputStream * sub_stream,const Options & options)203 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream,
204                                    const Options& options) {
205   Init(sub_stream, options);
206 }
207 
Init(ZeroCopyOutputStream * sub_stream,const Options & options)208 void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
209                             const Options& options) {
210   sub_stream_ = sub_stream;
211   sub_data_ = NULL;
212   sub_data_size_ = 0;
213 
214   input_buffer_length_ = options.buffer_size;
215   input_buffer_ = operator new(input_buffer_length_);
216   GOOGLE_CHECK(input_buffer_ != NULL);
217 
218   zcontext_.zalloc = Z_NULL;
219   zcontext_.zfree = Z_NULL;
220   zcontext_.opaque = Z_NULL;
221   zcontext_.next_out = NULL;
222   zcontext_.avail_out = 0;
223   zcontext_.total_out = 0;
224   zcontext_.next_in = NULL;
225   zcontext_.avail_in = 0;
226   zcontext_.total_in = 0;
227   zcontext_.msg = NULL;
228   // default to GZIP format
229   int windowBitsFormat = 16;
230   if (options.format == ZLIB) {
231     windowBitsFormat = 0;
232   }
233   zerror_ = deflateInit2(
234       &zcontext_,
235       options.compression_level,
236       Z_DEFLATED,
237       /* windowBits */15 | windowBitsFormat,
238       /* memLevel (default) */8,
239       options.compression_strategy);
240 }
241 
~GzipOutputStream()242 GzipOutputStream::~GzipOutputStream() {
243   Close();
244   operator delete(input_buffer_);
245 }
246 
247 // private
Deflate(int flush)248 int GzipOutputStream::Deflate(int flush) {
249   int error = Z_OK;
250   do {
251     if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) {
252       bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_);
253       if (!ok) {
254         sub_data_ = NULL;
255         sub_data_size_ = 0;
256         return Z_BUF_ERROR;
257       }
258       GOOGLE_CHECK_GT(sub_data_size_, 0);
259       zcontext_.next_out = static_cast<Bytef*>(sub_data_);
260       zcontext_.avail_out = sub_data_size_;
261     }
262     error = deflate(&zcontext_, flush);
263   } while (error == Z_OK && zcontext_.avail_out == 0);
264   if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) {
265     // Notify lower layer of data.
266     sub_stream_->BackUp(zcontext_.avail_out);
267     // We don't own the buffer anymore.
268     sub_data_ = NULL;
269     sub_data_size_ = 0;
270   }
271   return error;
272 }
273 
274 // implements ZeroCopyOutputStream ---------------------------------
Next(void ** data,int * size)275 bool GzipOutputStream::Next(void** data, int* size) {
276   if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
277     return false;
278   }
279   if (zcontext_.avail_in != 0) {
280     zerror_ = Deflate(Z_NO_FLUSH);
281     if (zerror_ != Z_OK) {
282       return false;
283     }
284   }
285   if (zcontext_.avail_in == 0) {
286     // all input was consumed. reset the buffer.
287     zcontext_.next_in = static_cast<Bytef*>(input_buffer_);
288     zcontext_.avail_in = input_buffer_length_;
289     *data = input_buffer_;
290     *size = input_buffer_length_;
291   } else {
292     // The loop in Deflate should consume all avail_in
293     GOOGLE_LOG(DFATAL) << "Deflate left bytes unconsumed";
294   }
295   return true;
296 }
BackUp(int count)297 void GzipOutputStream::BackUp(int count) {
298   GOOGLE_CHECK_GE(zcontext_.avail_in, count);
299   zcontext_.avail_in -= count;
300 }
ByteCount() const301 int64 GzipOutputStream::ByteCount() const {
302   return zcontext_.total_in + zcontext_.avail_in;
303 }
304 
Flush()305 bool GzipOutputStream::Flush() {
306   zerror_ = Deflate(Z_FULL_FLUSH);
307   // Return true if the flush succeeded or if it was a no-op.
308   return  (zerror_ == Z_OK) ||
309       (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
310        zcontext_.avail_out != 0);
311 }
312 
Close()313 bool GzipOutputStream::Close() {
314   if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
315     return false;
316   }
317   do {
318     zerror_ = Deflate(Z_FINISH);
319   } while (zerror_ == Z_OK);
320   zerror_ = deflateEnd(&zcontext_);
321   bool ok = zerror_ == Z_OK;
322   zerror_ = Z_STREAM_END;
323   return ok;
324 }
325 
326 }  // namespace io
327 }  // namespace protobuf
328 }  // namespace google
329 
330 #endif  // HAVE_ZLIB
331