1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 // Author: brianolson@google.com (Brian Olson)
9 //
10 // This file contains the implementation of classes GzipInputStream and
11 // GzipOutputStream.
12
13
14 #if HAVE_ZLIB
15 #include "google/protobuf/io/gzip_stream.h"
16
17 #include "google/protobuf/stubs/common.h"
18 #include "absl/log/absl_check.h"
19 #include "absl/log/absl_log.h"
20 #include "google/protobuf/port.h"
21
22 namespace google {
23 namespace protobuf {
24 namespace io {
25
26 static const int kDefaultBufferSize = 65536;
27
GzipInputStream(ZeroCopyInputStream * sub_stream,Format format,int buffer_size)28 GzipInputStream::GzipInputStream(ZeroCopyInputStream* sub_stream, Format format,
29 int buffer_size)
30 : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) {
31 zcontext_.state = Z_NULL;
32 zcontext_.zalloc = Z_NULL;
33 zcontext_.zfree = Z_NULL;
34 zcontext_.opaque = Z_NULL;
35 zcontext_.total_out = 0;
36 zcontext_.next_in = nullptr;
37 zcontext_.avail_in = 0;
38 zcontext_.total_in = 0;
39 zcontext_.msg = nullptr;
40 if (buffer_size == -1) {
41 output_buffer_length_ = kDefaultBufferSize;
42 } else {
43 output_buffer_length_ = buffer_size;
44 }
45 output_buffer_ = operator new(output_buffer_length_);
46 ABSL_CHECK(output_buffer_ != nullptr);
47 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
48 zcontext_.avail_out = output_buffer_length_;
49 output_position_ = output_buffer_;
50 }
~GzipInputStream()51 GzipInputStream::~GzipInputStream() {
52 internal::SizedDelete(output_buffer_, output_buffer_length_);
53 zerror_ = inflateEnd(&zcontext_);
54 }
55
internalInflateInit2(z_stream * zcontext,GzipInputStream::Format format)56 static inline int internalInflateInit2(z_stream* zcontext,
57 GzipInputStream::Format format) {
58 int windowBitsFormat = 0;
59 switch (format) {
60 case GzipInputStream::GZIP:
61 windowBitsFormat = 16;
62 break;
63 case GzipInputStream::AUTO:
64 windowBitsFormat = 32;
65 break;
66 case GzipInputStream::ZLIB:
67 windowBitsFormat = 0;
68 break;
69 }
70 return inflateInit2(zcontext, /* windowBits */ 15 | windowBitsFormat);
71 }
72
Inflate(int flush)73 int GzipInputStream::Inflate(int flush) {
74 if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) {
75 // previous inflate filled output buffer. don't change input params yet.
76 } else if (zcontext_.avail_in == 0) {
77 const void* in;
78 int in_size;
79 bool first = zcontext_.next_in == NULL;
80 bool ok = sub_stream_->Next(&in, &in_size);
81 if (!ok) {
82 zcontext_.next_out = NULL;
83 zcontext_.avail_out = 0;
84 return Z_STREAM_END;
85 }
86 zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in));
87 zcontext_.avail_in = in_size;
88 if (first) {
89 int error = internalInflateInit2(&zcontext_, format_);
90 if (error != Z_OK) {
91 return error;
92 }
93 }
94 }
95 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
96 zcontext_.avail_out = output_buffer_length_;
97 output_position_ = output_buffer_;
98 int error = inflate(&zcontext_, flush);
99 return error;
100 }
101
DoNextOutput(const void ** data,int * size)102 void GzipInputStream::DoNextOutput(const void** data, int* size) {
103 *data = output_position_;
104 *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_);
105 output_position_ = zcontext_.next_out;
106 }
107
108 // implements ZeroCopyInputStream ----------------------------------
Next(const void ** data,int * size)109 bool GzipInputStream::Next(const void** data, int* size) {
110 bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
111 (zerror_ == Z_BUF_ERROR);
112 if ((!ok) || (zcontext_.next_out == NULL)) {
113 return false;
114 }
115 if (zcontext_.next_out != output_position_) {
116 DoNextOutput(data, size);
117 return true;
118 }
119 if (zerror_ == Z_STREAM_END) {
120 if (zcontext_.next_out != NULL) {
121 // sub_stream_ may have concatenated streams to follow
122 zerror_ = inflateEnd(&zcontext_);
123 byte_count_ += zcontext_.total_out;
124 if (zerror_ != Z_OK) {
125 return false;
126 }
127 zerror_ = internalInflateInit2(&zcontext_, format_);
128 if (zerror_ != Z_OK) {
129 return false;
130 }
131 } else {
132 *data = NULL;
133 *size = 0;
134 return false;
135 }
136 }
137 zerror_ = Inflate(Z_NO_FLUSH);
138 if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) {
139 // The underlying stream's Next returned false inside Inflate.
140 return false;
141 }
142 ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
143 (zerror_ == Z_BUF_ERROR);
144 if (!ok) {
145 return false;
146 }
147 DoNextOutput(data, size);
148 return true;
149 }
BackUp(int count)150 void GzipInputStream::BackUp(int count) {
151 output_position_ = reinterpret_cast<void*>(
152 reinterpret_cast<uintptr_t>(output_position_) - count);
153 }
Skip(int count)154 bool GzipInputStream::Skip(int count) {
155 const void* data;
156 int size = 0;
157 bool ok = Next(&data, &size);
158 while (ok && (size < count)) {
159 count -= size;
160 ok = Next(&data, &size);
161 }
162 if (size > count) {
163 BackUp(size - count);
164 }
165 return ok;
166 }
ByteCount() const167 int64_t GzipInputStream::ByteCount() const {
168 int64_t ret = byte_count_ + zcontext_.total_out;
169 if (zcontext_.next_out != NULL && output_position_ != NULL) {
170 ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
171 reinterpret_cast<uintptr_t>(output_position_);
172 }
173 return ret;
174 }
175
176 // =========================================================================
177
Options()178 GzipOutputStream::Options::Options()
179 : format(GZIP),
180 buffer_size(kDefaultBufferSize),
181 compression_level(Z_DEFAULT_COMPRESSION),
182 compression_strategy(Z_DEFAULT_STRATEGY) {}
183
GzipOutputStream(ZeroCopyOutputStream * sub_stream)184 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) {
185 Init(sub_stream, Options());
186 }
187
GzipOutputStream(ZeroCopyOutputStream * sub_stream,const Options & options)188 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream,
189 const Options& options) {
190 Init(sub_stream, options);
191 }
192
Init(ZeroCopyOutputStream * sub_stream,const Options & options)193 void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
194 const Options& options) {
195 sub_stream_ = sub_stream;
196 sub_data_ = NULL;
197 sub_data_size_ = 0;
198
199 input_buffer_length_ = options.buffer_size;
200 input_buffer_ = operator new(input_buffer_length_);
201 ABSL_CHECK(input_buffer_ != NULL);
202
203 zcontext_.zalloc = Z_NULL;
204 zcontext_.zfree = Z_NULL;
205 zcontext_.opaque = Z_NULL;
206 zcontext_.next_out = NULL;
207 zcontext_.avail_out = 0;
208 zcontext_.total_out = 0;
209 zcontext_.next_in = NULL;
210 zcontext_.avail_in = 0;
211 zcontext_.total_in = 0;
212 zcontext_.msg = NULL;
213 // default to GZIP format
214 int windowBitsFormat = 16;
215 if (options.format == ZLIB) {
216 windowBitsFormat = 0;
217 }
218 zerror_ =
219 deflateInit2(&zcontext_, options.compression_level, Z_DEFLATED,
220 /* windowBits */ 15 | windowBitsFormat,
221 /* memLevel (default) */ 8, options.compression_strategy);
222 }
223
~GzipOutputStream()224 GzipOutputStream::~GzipOutputStream() {
225 Close();
226 internal::SizedDelete(input_buffer_, input_buffer_length_);
227 }
228
229 // private
Deflate(int flush)230 int GzipOutputStream::Deflate(int flush) {
231 int error = Z_OK;
232 do {
233 if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) {
234 bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_);
235 if (!ok) {
236 sub_data_ = NULL;
237 sub_data_size_ = 0;
238 return Z_BUF_ERROR;
239 }
240 ABSL_CHECK_GT(sub_data_size_, 0);
241 zcontext_.next_out = static_cast<Bytef*>(sub_data_);
242 zcontext_.avail_out = sub_data_size_;
243 }
244 error = deflate(&zcontext_, flush);
245 } while (error == Z_OK && zcontext_.avail_out == 0);
246 if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) {
247 // Notify lower layer of data.
248 sub_stream_->BackUp(zcontext_.avail_out);
249 // We don't own the buffer anymore.
250 sub_data_ = NULL;
251 sub_data_size_ = 0;
252 }
253 return error;
254 }
255
256 // implements ZeroCopyOutputStream ---------------------------------
Next(void ** data,int * size)257 bool GzipOutputStream::Next(void** data, int* size) {
258 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
259 return false;
260 }
261 if (zcontext_.avail_in != 0) {
262 zerror_ = Deflate(Z_NO_FLUSH);
263 if (zerror_ != Z_OK) {
264 return false;
265 }
266 }
267 if (zcontext_.avail_in == 0) {
268 // all input was consumed. reset the buffer.
269 zcontext_.next_in = static_cast<Bytef*>(input_buffer_);
270 zcontext_.avail_in = input_buffer_length_;
271 *data = input_buffer_;
272 *size = input_buffer_length_;
273 } else {
274 // The loop in Deflate should consume all avail_in
275 ABSL_DLOG(FATAL) << "Deflate left bytes unconsumed";
276 }
277 return true;
278 }
BackUp(int count)279 void GzipOutputStream::BackUp(int count) {
280 ABSL_CHECK_GE(zcontext_.avail_in, static_cast<uInt>(count));
281 zcontext_.avail_in -= count;
282 }
ByteCount() const283 int64_t GzipOutputStream::ByteCount() const {
284 return zcontext_.total_in + zcontext_.avail_in;
285 }
286
Flush()287 bool GzipOutputStream::Flush() {
288 zerror_ = Deflate(Z_FULL_FLUSH);
289 // Return true if the flush succeeded or if it was a no-op.
290 return (zerror_ == Z_OK) ||
291 (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
292 zcontext_.avail_out != 0);
293 }
294
Close()295 bool GzipOutputStream::Close() {
296 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
297 return false;
298 }
299 do {
300 zerror_ = Deflate(Z_FINISH);
301 } while (zerror_ == Z_OK);
302 zerror_ = deflateEnd(&zcontext_);
303 bool ok = zerror_ == Z_OK;
304 zerror_ = Z_STREAM_END;
305 return ok;
306 }
307
308 } // namespace io
309 } // namespace protobuf
310 } // namespace google
311
312 #endif // HAVE_ZLIB
313