1 // Copyright (C) 2009 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // This file contains the implementation of classes GzipInputStream and
16 // GzipOutputStream. It is forked from protobuf because these classes are only
17 // provided in libprotobuf-full but we would like to link libicing against the
18 // smaller libprotobuf-lite instead.
19
20 #include "icing/portable/gzip_stream.h"
21 #include "icing/util/logging.h"
22
23 namespace icing {
24 namespace lib {
25 namespace protobuf_ports {
26
27 static const int kDefaultBufferSize = 65536;
28
GzipInputStream(ZeroCopyInputStream * sub_stream,Format format,int buffer_size)29 GzipInputStream::GzipInputStream(ZeroCopyInputStream* sub_stream, Format format,
30 int buffer_size)
31 : format_(format), sub_stream_(sub_stream), zerror_(Z_OK), byte_count_(0) {
32 zcontext_.state = Z_NULL;
33 zcontext_.zalloc = Z_NULL;
34 zcontext_.zfree = Z_NULL;
35 zcontext_.opaque = Z_NULL;
36 zcontext_.total_out = 0;
37 zcontext_.next_in = NULL;
38 zcontext_.avail_in = 0;
39 zcontext_.total_in = 0;
40 zcontext_.msg = NULL;
41 if (buffer_size == -1) {
42 output_buffer_length_ = kDefaultBufferSize;
43 } else {
44 output_buffer_length_ = buffer_size;
45 }
46 output_buffer_ = operator new(output_buffer_length_);
47 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
48 zcontext_.avail_out = output_buffer_length_;
49 output_position_ = output_buffer_;
50 }
~GzipInputStream()51 GzipInputStream::~GzipInputStream() {
52 operator delete(output_buffer_);
53 zerror_ = inflateEnd(&zcontext_);
54 }
55
internalInflateInit2(z_stream * zcontext,GzipInputStream::Format format)56 static inline int internalInflateInit2(z_stream* zcontext,
57 GzipInputStream::Format format) {
58 int windowBitsFormat = 0;
59 switch (format) {
60 case GzipInputStream::GZIP:
61 windowBitsFormat = 16;
62 break;
63 case GzipInputStream::AUTO:
64 windowBitsFormat = 32;
65 break;
66 case GzipInputStream::ZLIB:
67 windowBitsFormat = 0;
68 break;
69 }
70 return inflateInit2(zcontext, /* windowBits */ 15 | windowBitsFormat);
71 }
72
Inflate(int flush)73 int GzipInputStream::Inflate(int flush) {
74 if ((zerror_ == Z_OK) && (zcontext_.avail_out == 0)) {
75 // previous inflate filled output buffer. don't change input params yet.
76 } else if (zcontext_.avail_in == 0) {
77 const void* in;
78 int in_size;
79 bool first = zcontext_.next_in == NULL;
80 bool ok = sub_stream_->Next(&in, &in_size);
81 if (!ok) {
82 zcontext_.next_out = NULL;
83 zcontext_.avail_out = 0;
84 return Z_STREAM_END;
85 }
86 zcontext_.next_in = static_cast<Bytef*>(const_cast<void*>(in));
87 zcontext_.avail_in = in_size;
88 if (first) {
89 int error = internalInflateInit2(&zcontext_, format_);
90 if (error != Z_OK) {
91 return error;
92 }
93 }
94 }
95 zcontext_.next_out = static_cast<Bytef*>(output_buffer_);
96 zcontext_.avail_out = output_buffer_length_;
97 output_position_ = output_buffer_;
98 int error = inflate(&zcontext_, flush);
99 return error;
100 }
101
DoNextOutput(const void ** data,int * size)102 void GzipInputStream::DoNextOutput(const void** data, int* size) {
103 *data = output_position_;
104 *size = ((uintptr_t)zcontext_.next_out) - ((uintptr_t)output_position_);
105 output_position_ = zcontext_.next_out;
106 }
107
108 // implements ZeroCopyInputStream ----------------------------------
Next(const void ** data,int * size)109 bool GzipInputStream::Next(const void** data, int* size) {
110 bool ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
111 (zerror_ == Z_BUF_ERROR);
112 if ((!ok) || (zcontext_.next_out == NULL)) {
113 return false;
114 }
115 if (zcontext_.next_out != output_position_) {
116 DoNextOutput(data, size);
117 return true;
118 }
119 if (zerror_ == Z_STREAM_END) {
120 if (zcontext_.next_out != NULL) {
121 // sub_stream_ may have concatenated streams to follow
122 zerror_ = inflateEnd(&zcontext_);
123 byte_count_ += zcontext_.total_out;
124 if (zerror_ != Z_OK) {
125 return false;
126 }
127 zerror_ = internalInflateInit2(&zcontext_, format_);
128 if (zerror_ != Z_OK) {
129 return false;
130 }
131 } else {
132 *data = NULL;
133 *size = 0;
134 return false;
135 }
136 }
137 zerror_ = Inflate(Z_NO_FLUSH);
138 if ((zerror_ == Z_STREAM_END) && (zcontext_.next_out == NULL)) {
139 // The underlying stream's Next returned false inside Inflate.
140 return false;
141 }
142 ok = (zerror_ == Z_OK) || (zerror_ == Z_STREAM_END) ||
143 (zerror_ == Z_BUF_ERROR);
144 if (!ok) {
145 return false;
146 }
147 DoNextOutput(data, size);
148 return true;
149 }
BackUp(int count)150 void GzipInputStream::BackUp(int count) {
151 output_position_ = reinterpret_cast<void*>(
152 reinterpret_cast<uintptr_t>(output_position_) - count);
153 }
Skip(int count)154 bool GzipInputStream::Skip(int count) {
155 const void* data;
156 int size = 0;
157 bool ok = Next(&data, &size);
158 while (ok && (size < count)) {
159 count -= size;
160 ok = Next(&data, &size);
161 }
162 if (size > count) {
163 BackUp(size - count);
164 }
165 return ok;
166 }
ByteCount() const167 int64_t GzipInputStream::ByteCount() const {
168 int64_t ret = byte_count_ + zcontext_.total_out;
169 if (zcontext_.next_out != NULL && output_position_ != NULL) {
170 ret += reinterpret_cast<uintptr_t>(zcontext_.next_out) -
171 reinterpret_cast<uintptr_t>(output_position_);
172 }
173 return ret;
174 }
175
176 // =========================================================================
177
Options()178 GzipOutputStream::Options::Options()
179 : format(GZIP),
180 buffer_size(kDefaultBufferSize),
181 compression_level(Z_DEFAULT_COMPRESSION),
182 compression_strategy(Z_DEFAULT_STRATEGY) {}
183
GzipOutputStream(ZeroCopyOutputStream * sub_stream)184 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream) {
185 Init(sub_stream, Options());
186 }
187
GzipOutputStream(ZeroCopyOutputStream * sub_stream,const Options & options)188 GzipOutputStream::GzipOutputStream(ZeroCopyOutputStream* sub_stream,
189 const Options& options) {
190 Init(sub_stream, options);
191 }
192
Init(ZeroCopyOutputStream * sub_stream,const Options & options)193 void GzipOutputStream::Init(ZeroCopyOutputStream* sub_stream,
194 const Options& options) {
195 sub_stream_ = sub_stream;
196 sub_data_ = NULL;
197 sub_data_size_ = 0;
198
199 input_buffer_length_ = options.buffer_size;
200 input_buffer_ = operator new(input_buffer_length_);
201
202 zcontext_.zalloc = Z_NULL;
203 zcontext_.zfree = Z_NULL;
204 zcontext_.opaque = Z_NULL;
205 zcontext_.next_out = NULL;
206 zcontext_.avail_out = 0;
207 zcontext_.total_out = 0;
208 zcontext_.next_in = NULL;
209 zcontext_.avail_in = 0;
210 zcontext_.total_in = 0;
211 zcontext_.msg = NULL;
212 // default to GZIP format
213 int windowBitsFormat = 16;
214 if (options.format == ZLIB) {
215 windowBitsFormat = 0;
216 }
217 zerror_ =
218 deflateInit2(&zcontext_, options.compression_level, Z_DEFLATED,
219 /* windowBits */ 15 | windowBitsFormat,
220 /* memLevel (default) */ 8, options.compression_strategy);
221 }
222
~GzipOutputStream()223 GzipOutputStream::~GzipOutputStream() {
224 Close();
225 operator delete(input_buffer_);
226 }
227
228 // private
Deflate(int flush)229 int GzipOutputStream::Deflate(int flush) {
230 int error = Z_OK;
231 do {
232 if ((sub_data_ == NULL) || (zcontext_.avail_out == 0)) {
233 bool ok = sub_stream_->Next(&sub_data_, &sub_data_size_);
234 if (!ok) {
235 sub_data_ = NULL;
236 sub_data_size_ = 0;
237 return Z_BUF_ERROR;
238 }
239 if (sub_data_size_ <= 0) {
240 ICING_LOG(FATAL) << "Failed to advance underlying stream";
241 }
242 zcontext_.next_out = static_cast<Bytef*>(sub_data_);
243 zcontext_.avail_out = sub_data_size_;
244 }
245 error = deflate(&zcontext_, flush);
246 } while (error == Z_OK && zcontext_.avail_out == 0);
247 if ((flush == Z_FULL_FLUSH) || (flush == Z_FINISH)) {
248 // Notify lower layer of data.
249 sub_stream_->BackUp(zcontext_.avail_out);
250 // We don't own the buffer anymore.
251 sub_data_ = NULL;
252 sub_data_size_ = 0;
253 }
254 return error;
255 }
256
257 // implements ZeroCopyOutputStream ---------------------------------
Next(void ** data,int * size)258 bool GzipOutputStream::Next(void** data, int* size) {
259 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
260 return false;
261 }
262 if (zcontext_.avail_in != 0) {
263 zerror_ = Deflate(Z_NO_FLUSH);
264 if (zerror_ != Z_OK) {
265 return false;
266 }
267 }
268 if (zcontext_.avail_in == 0) {
269 // all input was consumed. reset the buffer.
270 zcontext_.next_in = static_cast<Bytef*>(input_buffer_);
271 zcontext_.avail_in = input_buffer_length_;
272 *data = input_buffer_;
273 *size = input_buffer_length_;
274 } else {
275 // The loop in Deflate should consume all avail_in
276 ICING_LOG(ERROR) << "Deflate left bytes unconsumed";
277 }
278 return true;
279 }
BackUp(int count)280 void GzipOutputStream::BackUp(int count) {
281 if (zcontext_.avail_in < static_cast<uInt>(count)) {
282 ICING_LOG(FATAL) << "Not enough data to back up " << count << " bytes";
283 }
284 zcontext_.avail_in -= count;
285 }
ByteCount() const286 int64_t GzipOutputStream::ByteCount() const {
287 return zcontext_.total_in + zcontext_.avail_in;
288 }
289
Flush()290 bool GzipOutputStream::Flush() {
291 zerror_ = Deflate(Z_FULL_FLUSH);
292 // Return true if the flush succeeded or if it was a no-op.
293 return (zerror_ == Z_OK) ||
294 (zerror_ == Z_BUF_ERROR && zcontext_.avail_in == 0 &&
295 zcontext_.avail_out != 0);
296 }
297
Close()298 bool GzipOutputStream::Close() {
299 if ((zerror_ != Z_OK) && (zerror_ != Z_BUF_ERROR)) {
300 return false;
301 }
302 do {
303 zerror_ = Deflate(Z_FINISH);
304 } while (zerror_ == Z_OK);
305 zerror_ = deflateEnd(&zcontext_);
306 bool ok = zerror_ == Z_OK;
307 zerror_ = Z_STREAM_END;
308 return ok;
309 }
310
311 } // namespace protobuf_ports
312 } // namespace lib
313 } // namespace icing
314