1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // Author: kenton@google.com (Kenton Varda)
32 // Based on original Protocol Buffers design by
33 // Sanjay Ghemawat, Jeff Dean, and others.
34 //
35 // Testing strategy: For each type of I/O (array, string, file, etc.) we
36 // create an output stream and write some data to it, then create a
37 // corresponding input stream to read the same data back and expect it to
38 // match. When the data is written, it is written in several small chunks
39 // of varying sizes, with a BackUp() after each chunk. It is read back
40 // similarly, but with chunks separated at different points. The whole
41 // process is run with a variety of block sizes for both the input and
42 // the output.
43 //
44 // TODO(kenton): Rewrite this test to bring it up to the standards of all
45 // the other proto2 tests. May want to wait for gTest to implement
46 // "parametized tests" so that one set of tests can be used on all the
47 // implementations.
48
49
50 #ifdef _MSC_VER
51 #include <io.h>
52 #else
53 #include <unistd.h>
54 #endif
55 #include <stdlib.h>
56 #include <sys/types.h>
57 #include <sys/stat.h>
58 #include <fcntl.h>
59 #include <errno.h>
60 #include <memory>
61 #ifndef _SHARED_PTR_H
62 #include <google/protobuf/stubs/shared_ptr.h>
63 #endif
64 #include <sstream>
65
66 #include <google/protobuf/testing/file.h>
67 #include <google/protobuf/io/coded_stream.h>
68 #include <google/protobuf/io/zero_copy_stream_impl.h>
69
70 #if HAVE_ZLIB
71 #include <google/protobuf/io/gzip_stream.h>
72 #endif
73
74 #include <google/protobuf/stubs/common.h>
75 #include <google/protobuf/stubs/logging.h>
76 #include <google/protobuf/testing/googletest.h>
77 #include <google/protobuf/testing/file.h>
78 #include <gtest/gtest.h>
79
80 namespace google {
81 namespace protobuf {
82 namespace io {
83 namespace {
84
85 #ifdef _WIN32
86 #define pipe(fds) _pipe(fds, 4096, O_BINARY)
87 #endif
88
89 #ifndef O_BINARY
90 #ifdef _O_BINARY
91 #define O_BINARY _O_BINARY
92 #else
93 #define O_BINARY 0 // If this isn't defined, the platform doesn't need it.
94 #endif
95 #endif
96
97 class IoTest : public testing::Test {
98 protected:
99 // Test helpers.
100
101 // Helper to write an array of data to an output stream.
102 bool WriteToOutput(ZeroCopyOutputStream* output, const void* data, int size);
103 // Helper to read a fixed-length array of data from an input stream.
104 int ReadFromInput(ZeroCopyInputStream* input, void* data, int size);
105 // Write a string to the output stream.
106 void WriteString(ZeroCopyOutputStream* output, const string& str);
107 // Read a number of bytes equal to the size of the given string and checks
108 // that it matches the string.
109 void ReadString(ZeroCopyInputStream* input, const string& str);
110 // Writes some text to the output stream in a particular order. Returns
111 // the number of bytes written, incase the caller needs that to set up an
112 // input stream.
113 int WriteStuff(ZeroCopyOutputStream* output);
114 // Reads text from an input stream and expects it to match what
115 // WriteStuff() writes.
116 void ReadStuff(ZeroCopyInputStream* input);
117
118 // Similar to WriteStuff, but performs more sophisticated testing.
119 int WriteStuffLarge(ZeroCopyOutputStream* output);
120 // Reads and tests a stream that should have been written to
121 // via WriteStuffLarge().
122 void ReadStuffLarge(ZeroCopyInputStream* input);
123
124 #if HAVE_ZLIB
125 string Compress(const string& data, const GzipOutputStream::Options& options);
126 string Uncompress(const string& data);
127 #endif
128
129 static const int kBlockSizes[];
130 static const int kBlockSizeCount;
131 };
132
133 const int IoTest::kBlockSizes[] = {-1, 1, 2, 5, 7, 10, 23, 64};
134 const int IoTest::kBlockSizeCount = GOOGLE_ARRAYSIZE(IoTest::kBlockSizes);
135
WriteToOutput(ZeroCopyOutputStream * output,const void * data,int size)136 bool IoTest::WriteToOutput(ZeroCopyOutputStream* output,
137 const void* data, int size) {
138 const uint8* in = reinterpret_cast<const uint8*>(data);
139 int in_size = size;
140
141 void* out;
142 int out_size;
143
144 while (true) {
145 if (!output->Next(&out, &out_size)) {
146 return false;
147 }
148 EXPECT_GT(out_size, 0);
149
150 if (in_size <= out_size) {
151 memcpy(out, in, in_size);
152 output->BackUp(out_size - in_size);
153 return true;
154 }
155
156 memcpy(out, in, out_size);
157 in += out_size;
158 in_size -= out_size;
159 }
160 }
161
162 #define MAX_REPEATED_ZEROS 100
163
ReadFromInput(ZeroCopyInputStream * input,void * data,int size)164 int IoTest::ReadFromInput(ZeroCopyInputStream* input, void* data, int size) {
165 uint8* out = reinterpret_cast<uint8*>(data);
166 int out_size = size;
167
168 const void* in;
169 int in_size = 0;
170
171 int repeated_zeros = 0;
172
173 while (true) {
174 if (!input->Next(&in, &in_size)) {
175 return size - out_size;
176 }
177 EXPECT_GT(in_size, -1);
178 if (in_size == 0) {
179 repeated_zeros++;
180 } else {
181 repeated_zeros = 0;
182 }
183 EXPECT_LT(repeated_zeros, MAX_REPEATED_ZEROS);
184
185 if (out_size <= in_size) {
186 memcpy(out, in, out_size);
187 if (in_size > out_size) {
188 input->BackUp(in_size - out_size);
189 }
190 return size; // Copied all of it.
191 }
192
193 memcpy(out, in, in_size);
194 out += in_size;
195 out_size -= in_size;
196 }
197 }
198
WriteString(ZeroCopyOutputStream * output,const string & str)199 void IoTest::WriteString(ZeroCopyOutputStream* output, const string& str) {
200 EXPECT_TRUE(WriteToOutput(output, str.c_str(), str.size()));
201 }
202
ReadString(ZeroCopyInputStream * input,const string & str)203 void IoTest::ReadString(ZeroCopyInputStream* input, const string& str) {
204 google::protobuf::scoped_array<char> buffer(new char[str.size() + 1]);
205 buffer[str.size()] = '\0';
206 EXPECT_EQ(ReadFromInput(input, buffer.get(), str.size()), str.size());
207 EXPECT_STREQ(str.c_str(), buffer.get());
208 }
209
WriteStuff(ZeroCopyOutputStream * output)210 int IoTest::WriteStuff(ZeroCopyOutputStream* output) {
211 WriteString(output, "Hello world!\n");
212 WriteString(output, "Some te");
213 WriteString(output, "xt. Blah blah.");
214 WriteString(output, "abcdefg");
215 WriteString(output, "01234567890123456789");
216 WriteString(output, "foobar");
217
218 EXPECT_EQ(output->ByteCount(), 68);
219
220 int result = output->ByteCount();
221 return result;
222 }
223
224 // Reads text from an input stream and expects it to match what WriteStuff()
225 // writes.
ReadStuff(ZeroCopyInputStream * input)226 void IoTest::ReadStuff(ZeroCopyInputStream* input) {
227 ReadString(input, "Hello world!\n");
228 ReadString(input, "Some text. ");
229 ReadString(input, "Blah ");
230 ReadString(input, "blah.");
231 ReadString(input, "abcdefg");
232 EXPECT_TRUE(input->Skip(20));
233 ReadString(input, "foo");
234 ReadString(input, "bar");
235
236 EXPECT_EQ(input->ByteCount(), 68);
237
238 uint8 byte;
239 EXPECT_EQ(ReadFromInput(input, &byte, 1), 0);
240 }
241
WriteStuffLarge(ZeroCopyOutputStream * output)242 int IoTest::WriteStuffLarge(ZeroCopyOutputStream* output) {
243 WriteString(output, "Hello world!\n");
244 WriteString(output, "Some te");
245 WriteString(output, "xt. Blah blah.");
246 WriteString(output, string(100000, 'x')); // A very long string
247 WriteString(output, string(100000, 'y')); // A very long string
248 WriteString(output, "01234567890123456789");
249
250 EXPECT_EQ(output->ByteCount(), 200055);
251
252 int result = output->ByteCount();
253 return result;
254 }
255
256 // Reads text from an input stream and expects it to match what WriteStuff()
257 // writes.
ReadStuffLarge(ZeroCopyInputStream * input)258 void IoTest::ReadStuffLarge(ZeroCopyInputStream* input) {
259 ReadString(input, "Hello world!\nSome text. ");
260 EXPECT_TRUE(input->Skip(5));
261 ReadString(input, "blah.");
262 EXPECT_TRUE(input->Skip(100000 - 10));
263 ReadString(input, string(10, 'x') + string(100000 - 20000, 'y'));
264 EXPECT_TRUE(input->Skip(20000 - 10));
265 ReadString(input, "yyyyyyyyyy01234567890123456789");
266
267 EXPECT_EQ(input->ByteCount(), 200055);
268
269 uint8 byte;
270 EXPECT_EQ(ReadFromInput(input, &byte, 1), 0);
271 }
272
273 // ===================================================================
274
TEST_F(IoTest,ArrayIo)275 TEST_F(IoTest, ArrayIo) {
276 const int kBufferSize = 256;
277 uint8 buffer[kBufferSize];
278
279 for (int i = 0; i < kBlockSizeCount; i++) {
280 for (int j = 0; j < kBlockSizeCount; j++) {
281 int size;
282 {
283 ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
284 size = WriteStuff(&output);
285 }
286 {
287 ArrayInputStream input(buffer, size, kBlockSizes[j]);
288 ReadStuff(&input);
289 }
290 }
291 }
292 }
293
TEST_F(IoTest,TwoSessionWrite)294 TEST_F(IoTest, TwoSessionWrite) {
295 // Test that two concatenated write sessions read correctly
296
297 static const char* strA = "0123456789";
298 static const char* strB = "WhirledPeas";
299 const int kBufferSize = 2*1024;
300 uint8* buffer = new uint8[kBufferSize];
301 char* temp_buffer = new char[40];
302
303 for (int i = 0; i < kBlockSizeCount; i++) {
304 for (int j = 0; j < kBlockSizeCount; j++) {
305 ArrayOutputStream* output =
306 new ArrayOutputStream(buffer, kBufferSize, kBlockSizes[i]);
307 CodedOutputStream* coded_output = new CodedOutputStream(output);
308 coded_output->WriteVarint32(strlen(strA));
309 coded_output->WriteRaw(strA, strlen(strA));
310 delete coded_output; // flush
311 int64 pos = output->ByteCount();
312 delete output;
313 output = new ArrayOutputStream(
314 buffer + pos, kBufferSize - pos, kBlockSizes[i]);
315 coded_output = new CodedOutputStream(output);
316 coded_output->WriteVarint32(strlen(strB));
317 coded_output->WriteRaw(strB, strlen(strB));
318 delete coded_output; // flush
319 int64 size = pos + output->ByteCount();
320 delete output;
321
322 ArrayInputStream* input =
323 new ArrayInputStream(buffer, size, kBlockSizes[j]);
324 CodedInputStream* coded_input = new CodedInputStream(input);
325 uint32 insize;
326 EXPECT_TRUE(coded_input->ReadVarint32(&insize));
327 EXPECT_EQ(strlen(strA), insize);
328 EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
329 EXPECT_EQ(0, memcmp(temp_buffer, strA, insize));
330
331 EXPECT_TRUE(coded_input->ReadVarint32(&insize));
332 EXPECT_EQ(strlen(strB), insize);
333 EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
334 EXPECT_EQ(0, memcmp(temp_buffer, strB, insize));
335
336 delete coded_input;
337 delete input;
338 }
339 }
340
341 delete [] temp_buffer;
342 delete [] buffer;
343 }
344
345 #if HAVE_ZLIB
TEST_F(IoTest,GzipIo)346 TEST_F(IoTest, GzipIo) {
347 const int kBufferSize = 2*1024;
348 uint8* buffer = new uint8[kBufferSize];
349 for (int i = 0; i < kBlockSizeCount; i++) {
350 for (int j = 0; j < kBlockSizeCount; j++) {
351 for (int z = 0; z < kBlockSizeCount; z++) {
352 int gzip_buffer_size = kBlockSizes[z];
353 int size;
354 {
355 ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
356 GzipOutputStream::Options options;
357 options.format = GzipOutputStream::GZIP;
358 if (gzip_buffer_size != -1) {
359 options.buffer_size = gzip_buffer_size;
360 }
361 GzipOutputStream gzout(&output, options);
362 WriteStuff(&gzout);
363 gzout.Close();
364 size = output.ByteCount();
365 }
366 {
367 ArrayInputStream input(buffer, size, kBlockSizes[j]);
368 GzipInputStream gzin(
369 &input, GzipInputStream::GZIP, gzip_buffer_size);
370 ReadStuff(&gzin);
371 }
372 }
373 }
374 }
375 delete [] buffer;
376 }
377
TEST_F(IoTest,GzipIoWithFlush)378 TEST_F(IoTest, GzipIoWithFlush) {
379 const int kBufferSize = 2*1024;
380 uint8* buffer = new uint8[kBufferSize];
381 // We start with i = 4 as we want a block size > 6. With block size <= 6
382 // Flush() fills up the entire 2K buffer with flush markers and the test
383 // fails. See documentation for Flush() for more detail.
384 for (int i = 4; i < kBlockSizeCount; i++) {
385 for (int j = 0; j < kBlockSizeCount; j++) {
386 for (int z = 0; z < kBlockSizeCount; z++) {
387 int gzip_buffer_size = kBlockSizes[z];
388 int size;
389 {
390 ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
391 GzipOutputStream::Options options;
392 options.format = GzipOutputStream::GZIP;
393 if (gzip_buffer_size != -1) {
394 options.buffer_size = gzip_buffer_size;
395 }
396 GzipOutputStream gzout(&output, options);
397 WriteStuff(&gzout);
398 EXPECT_TRUE(gzout.Flush());
399 gzout.Close();
400 size = output.ByteCount();
401 }
402 {
403 ArrayInputStream input(buffer, size, kBlockSizes[j]);
404 GzipInputStream gzin(
405 &input, GzipInputStream::GZIP, gzip_buffer_size);
406 ReadStuff(&gzin);
407 }
408 }
409 }
410 }
411 delete [] buffer;
412 }
413
TEST_F(IoTest,GzipIoContiguousFlushes)414 TEST_F(IoTest, GzipIoContiguousFlushes) {
415 const int kBufferSize = 2*1024;
416 uint8* buffer = new uint8[kBufferSize];
417
418 int block_size = kBlockSizes[4];
419 int gzip_buffer_size = block_size;
420 int size;
421
422 ArrayOutputStream output(buffer, kBufferSize, block_size);
423 GzipOutputStream::Options options;
424 options.format = GzipOutputStream::GZIP;
425 if (gzip_buffer_size != -1) {
426 options.buffer_size = gzip_buffer_size;
427 }
428 GzipOutputStream gzout(&output, options);
429 WriteStuff(&gzout);
430 EXPECT_TRUE(gzout.Flush());
431 EXPECT_TRUE(gzout.Flush());
432 gzout.Close();
433 size = output.ByteCount();
434
435 ArrayInputStream input(buffer, size, block_size);
436 GzipInputStream gzin(
437 &input, GzipInputStream::GZIP, gzip_buffer_size);
438 ReadStuff(&gzin);
439
440 delete [] buffer;
441 }
442
TEST_F(IoTest,GzipIoReadAfterFlush)443 TEST_F(IoTest, GzipIoReadAfterFlush) {
444 const int kBufferSize = 2*1024;
445 uint8* buffer = new uint8[kBufferSize];
446
447 int block_size = kBlockSizes[4];
448 int gzip_buffer_size = block_size;
449 int size;
450 ArrayOutputStream output(buffer, kBufferSize, block_size);
451 GzipOutputStream::Options options;
452 options.format = GzipOutputStream::GZIP;
453 if (gzip_buffer_size != -1) {
454 options.buffer_size = gzip_buffer_size;
455 }
456
457 GzipOutputStream gzout(&output, options);
458 WriteStuff(&gzout);
459 EXPECT_TRUE(gzout.Flush());
460 size = output.ByteCount();
461
462 ArrayInputStream input(buffer, size, block_size);
463 GzipInputStream gzin(
464 &input, GzipInputStream::GZIP, gzip_buffer_size);
465 ReadStuff(&gzin);
466
467 gzout.Close();
468
469 delete [] buffer;
470 }
471
TEST_F(IoTest,ZlibIo)472 TEST_F(IoTest, ZlibIo) {
473 const int kBufferSize = 2*1024;
474 uint8* buffer = new uint8[kBufferSize];
475 for (int i = 0; i < kBlockSizeCount; i++) {
476 for (int j = 0; j < kBlockSizeCount; j++) {
477 for (int z = 0; z < kBlockSizeCount; z++) {
478 int gzip_buffer_size = kBlockSizes[z];
479 int size;
480 {
481 ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
482 GzipOutputStream::Options options;
483 options.format = GzipOutputStream::ZLIB;
484 if (gzip_buffer_size != -1) {
485 options.buffer_size = gzip_buffer_size;
486 }
487 GzipOutputStream gzout(&output, options);
488 WriteStuff(&gzout);
489 gzout.Close();
490 size = output.ByteCount();
491 }
492 {
493 ArrayInputStream input(buffer, size, kBlockSizes[j]);
494 GzipInputStream gzin(
495 &input, GzipInputStream::ZLIB, gzip_buffer_size);
496 ReadStuff(&gzin);
497 }
498 }
499 }
500 }
501 delete [] buffer;
502 }
503
TEST_F(IoTest,ZlibIoInputAutodetect)504 TEST_F(IoTest, ZlibIoInputAutodetect) {
505 const int kBufferSize = 2*1024;
506 uint8* buffer = new uint8[kBufferSize];
507 int size;
508 {
509 ArrayOutputStream output(buffer, kBufferSize);
510 GzipOutputStream::Options options;
511 options.format = GzipOutputStream::ZLIB;
512 GzipOutputStream gzout(&output, options);
513 WriteStuff(&gzout);
514 gzout.Close();
515 size = output.ByteCount();
516 }
517 {
518 ArrayInputStream input(buffer, size);
519 GzipInputStream gzin(&input, GzipInputStream::AUTO);
520 ReadStuff(&gzin);
521 }
522 {
523 ArrayOutputStream output(buffer, kBufferSize);
524 GzipOutputStream::Options options;
525 options.format = GzipOutputStream::GZIP;
526 GzipOutputStream gzout(&output, options);
527 WriteStuff(&gzout);
528 gzout.Close();
529 size = output.ByteCount();
530 }
531 {
532 ArrayInputStream input(buffer, size);
533 GzipInputStream gzin(&input, GzipInputStream::AUTO);
534 ReadStuff(&gzin);
535 }
536 delete [] buffer;
537 }
538
Compress(const string & data,const GzipOutputStream::Options & options)539 string IoTest::Compress(const string& data,
540 const GzipOutputStream::Options& options) {
541 string result;
542 {
543 StringOutputStream output(&result);
544 GzipOutputStream gzout(&output, options);
545 WriteToOutput(&gzout, data.data(), data.size());
546 }
547 return result;
548 }
549
Uncompress(const string & data)550 string IoTest::Uncompress(const string& data) {
551 string result;
552 {
553 ArrayInputStream input(data.data(), data.size());
554 GzipInputStream gzin(&input);
555 const void* buffer;
556 int size;
557 while (gzin.Next(&buffer, &size)) {
558 result.append(reinterpret_cast<const char*>(buffer), size);
559 }
560 }
561 return result;
562 }
563
TEST_F(IoTest,CompressionOptions)564 TEST_F(IoTest, CompressionOptions) {
565 // Some ad-hoc testing of compression options.
566
567 string golden;
568 GOOGLE_CHECK_OK(File::GetContents(
569 TestSourceDir() +
570 "/google/protobuf/testdata/golden_message",
571 &golden, true));
572
573 GzipOutputStream::Options options;
574 string gzip_compressed = Compress(golden, options);
575
576 options.compression_level = 0;
577 string not_compressed = Compress(golden, options);
578
579 // Try zlib compression for fun.
580 options = GzipOutputStream::Options();
581 options.format = GzipOutputStream::ZLIB;
582 string zlib_compressed = Compress(golden, options);
583
584 // Uncompressed should be bigger than the original since it should have some
585 // sort of header.
586 EXPECT_GT(not_compressed.size(), golden.size());
587
588 // Higher compression levels should result in smaller sizes.
589 EXPECT_LT(zlib_compressed.size(), not_compressed.size());
590
591 // ZLIB format should differ from GZIP format.
592 EXPECT_TRUE(zlib_compressed != gzip_compressed);
593
594 // Everything should decompress correctly.
595 EXPECT_TRUE(Uncompress(not_compressed) == golden);
596 EXPECT_TRUE(Uncompress(gzip_compressed) == golden);
597 EXPECT_TRUE(Uncompress(zlib_compressed) == golden);
598 }
599
TEST_F(IoTest,TwoSessionWriteGzip)600 TEST_F(IoTest, TwoSessionWriteGzip) {
601 // Test that two concatenated gzip streams can be read correctly
602
603 static const char* strA = "0123456789";
604 static const char* strB = "QuickBrownFox";
605 const int kBufferSize = 2*1024;
606 uint8* buffer = new uint8[kBufferSize];
607 char* temp_buffer = new char[40];
608
609 for (int i = 0; i < kBlockSizeCount; i++) {
610 for (int j = 0; j < kBlockSizeCount; j++) {
611 ArrayOutputStream* output =
612 new ArrayOutputStream(buffer, kBufferSize, kBlockSizes[i]);
613 GzipOutputStream* gzout = new GzipOutputStream(output);
614 CodedOutputStream* coded_output = new CodedOutputStream(gzout);
615 int32 outlen = strlen(strA) + 1;
616 coded_output->WriteVarint32(outlen);
617 coded_output->WriteRaw(strA, outlen);
618 delete coded_output; // flush
619 delete gzout; // flush
620 int64 pos = output->ByteCount();
621 delete output;
622 output = new ArrayOutputStream(
623 buffer + pos, kBufferSize - pos, kBlockSizes[i]);
624 gzout = new GzipOutputStream(output);
625 coded_output = new CodedOutputStream(gzout);
626 outlen = strlen(strB) + 1;
627 coded_output->WriteVarint32(outlen);
628 coded_output->WriteRaw(strB, outlen);
629 delete coded_output; // flush
630 delete gzout; // flush
631 int64 size = pos + output->ByteCount();
632 delete output;
633
634 ArrayInputStream* input =
635 new ArrayInputStream(buffer, size, kBlockSizes[j]);
636 GzipInputStream* gzin = new GzipInputStream(input);
637 CodedInputStream* coded_input = new CodedInputStream(gzin);
638 uint32 insize;
639 EXPECT_TRUE(coded_input->ReadVarint32(&insize));
640 EXPECT_EQ(strlen(strA) + 1, insize);
641 EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
642 EXPECT_EQ(0, memcmp(temp_buffer, strA, insize))
643 << "strA=" << strA << " in=" << temp_buffer;
644
645 EXPECT_TRUE(coded_input->ReadVarint32(&insize));
646 EXPECT_EQ(strlen(strB) + 1, insize);
647 EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
648 EXPECT_EQ(0, memcmp(temp_buffer, strB, insize))
649 << " out_block_size=" << kBlockSizes[i]
650 << " in_block_size=" << kBlockSizes[j]
651 << " pos=" << pos
652 << " size=" << size
653 << " strB=" << strB << " in=" << temp_buffer;
654
655 delete coded_input;
656 delete gzin;
657 delete input;
658 }
659 }
660
661 delete [] temp_buffer;
662 delete [] buffer;
663 }
664
TEST_F(IoTest,GzipInputByteCountAfterClosed)665 TEST_F(IoTest, GzipInputByteCountAfterClosed) {
666 string golden = "abcdefghijklmnopqrstuvwxyz";
667 string compressed = Compress(golden, GzipOutputStream::Options());
668
669 for (int i = 0; i < kBlockSizeCount; i++) {
670 ArrayInputStream arr_input(compressed.data(), compressed.size(),
671 kBlockSizes[i]);
672 GzipInputStream gz_input(&arr_input);
673 const void* buffer;
674 int size;
675 while (gz_input.Next(&buffer, &size)) {
676 EXPECT_LE(gz_input.ByteCount(), golden.size());
677 }
678 EXPECT_EQ(golden.size(), gz_input.ByteCount());
679 }
680 }
681
TEST_F(IoTest,GzipInputByteCountAfterClosedConcatenatedStreams)682 TEST_F(IoTest, GzipInputByteCountAfterClosedConcatenatedStreams) {
683 string golden1 = "abcdefghijklmnopqrstuvwxyz";
684 string golden2 = "the quick brown fox jumps over the lazy dog";
685 const size_t total_size = golden1.size() + golden2.size();
686 string compressed = Compress(golden1, GzipOutputStream::Options()) +
687 Compress(golden2, GzipOutputStream::Options());
688
689 for (int i = 0; i < kBlockSizeCount; i++) {
690 ArrayInputStream arr_input(compressed.data(), compressed.size(),
691 kBlockSizes[i]);
692 GzipInputStream gz_input(&arr_input);
693 const void* buffer;
694 int size;
695 while (gz_input.Next(&buffer, &size)) {
696 EXPECT_LE(gz_input.ByteCount(), total_size);
697 }
698 EXPECT_EQ(total_size, gz_input.ByteCount());
699 }
700 }
701 #endif
702
703 // There is no string input, only string output. Also, it doesn't support
704 // explicit block sizes. So, we'll only run one test and we'll use
705 // ArrayInput to read back the results.
TEST_F(IoTest,StringIo)706 TEST_F(IoTest, StringIo) {
707 string str;
708 {
709 StringOutputStream output(&str);
710 WriteStuff(&output);
711 }
712 {
713 ArrayInputStream input(str.data(), str.size());
714 ReadStuff(&input);
715 }
716 }
717
718
719 // To test files, we create a temporary file, write, read, truncate, repeat.
TEST_F(IoTest,FileIo)720 TEST_F(IoTest, FileIo) {
721 string filename = TestTempDir() + "/zero_copy_stream_test_file";
722
723 for (int i = 0; i < kBlockSizeCount; i++) {
724 for (int j = 0; j < kBlockSizeCount; j++) {
725 // Make a temporary file.
726 int file =
727 open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0777);
728 ASSERT_GE(file, 0);
729
730 {
731 FileOutputStream output(file, kBlockSizes[i]);
732 WriteStuff(&output);
733 EXPECT_EQ(0, output.GetErrno());
734 }
735
736 // Rewind.
737 ASSERT_NE(lseek(file, 0, SEEK_SET), (off_t)-1);
738
739 {
740 FileInputStream input(file, kBlockSizes[j]);
741 ReadStuff(&input);
742 EXPECT_EQ(0, input.GetErrno());
743 }
744
745 close(file);
746 }
747 }
748 }
749
750 #if HAVE_ZLIB
TEST_F(IoTest,GzipFileIo)751 TEST_F(IoTest, GzipFileIo) {
752 string filename = TestTempDir() + "/zero_copy_stream_test_file";
753
754 for (int i = 0; i < kBlockSizeCount; i++) {
755 for (int j = 0; j < kBlockSizeCount; j++) {
756 // Make a temporary file.
757 int file =
758 open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0777);
759 ASSERT_GE(file, 0);
760 {
761 FileOutputStream output(file, kBlockSizes[i]);
762 GzipOutputStream gzout(&output);
763 WriteStuffLarge(&gzout);
764 gzout.Close();
765 output.Flush();
766 EXPECT_EQ(0, output.GetErrno());
767 }
768
769 // Rewind.
770 ASSERT_NE(lseek(file, 0, SEEK_SET), (off_t)-1);
771
772 {
773 FileInputStream input(file, kBlockSizes[j]);
774 GzipInputStream gzin(&input);
775 ReadStuffLarge(&gzin);
776 EXPECT_EQ(0, input.GetErrno());
777 }
778
779 close(file);
780 }
781 }
782 }
783 #endif
784
785 // MSVC raises various debugging exceptions if we try to use a file
786 // descriptor of -1, defeating our tests below. This class will disable
787 // these debug assertions while in scope.
788 class MsvcDebugDisabler {
789 public:
790 #if defined(_MSC_VER) && _MSC_VER >= 1400
MsvcDebugDisabler()791 MsvcDebugDisabler() {
792 old_handler_ = _set_invalid_parameter_handler(MyHandler);
793 old_mode_ = _CrtSetReportMode(_CRT_ASSERT, 0);
794 }
~MsvcDebugDisabler()795 ~MsvcDebugDisabler() {
796 old_handler_ = _set_invalid_parameter_handler(old_handler_);
797 old_mode_ = _CrtSetReportMode(_CRT_ASSERT, old_mode_);
798 }
799
MyHandler(const wchar_t * expr,const wchar_t * func,const wchar_t * file,unsigned int line,uintptr_t pReserved)800 static void MyHandler(const wchar_t *expr,
801 const wchar_t *func,
802 const wchar_t *file,
803 unsigned int line,
804 uintptr_t pReserved) {
805 // do nothing
806 }
807
808 _invalid_parameter_handler old_handler_;
809 int old_mode_;
810 #else
811 // Dummy constructor and destructor to ensure that GCC doesn't complain
812 // that debug_disabler is an unused variable.
813 MsvcDebugDisabler() {}
814 ~MsvcDebugDisabler() {}
815 #endif
816 };
817
818 // Test that FileInputStreams report errors correctly.
TEST_F(IoTest,FileReadError)819 TEST_F(IoTest, FileReadError) {
820 MsvcDebugDisabler debug_disabler;
821
822 // -1 = invalid file descriptor.
823 FileInputStream input(-1);
824
825 const void* buffer;
826 int size;
827 EXPECT_FALSE(input.Next(&buffer, &size));
828 EXPECT_EQ(EBADF, input.GetErrno());
829 }
830
831 // Test that FileOutputStreams report errors correctly.
TEST_F(IoTest,FileWriteError)832 TEST_F(IoTest, FileWriteError) {
833 MsvcDebugDisabler debug_disabler;
834
835 // -1 = invalid file descriptor.
836 FileOutputStream input(-1);
837
838 void* buffer;
839 int size;
840
841 // The first call to Next() succeeds because it doesn't have anything to
842 // write yet.
843 EXPECT_TRUE(input.Next(&buffer, &size));
844
845 // Second call fails.
846 EXPECT_FALSE(input.Next(&buffer, &size));
847
848 EXPECT_EQ(EBADF, input.GetErrno());
849 }
850
851 // Pipes are not seekable, so File{Input,Output}Stream ends up doing some
852 // different things to handle them. We'll test by writing to a pipe and
853 // reading back from it.
TEST_F(IoTest,PipeIo)854 TEST_F(IoTest, PipeIo) {
855 int files[2];
856
857 for (int i = 0; i < kBlockSizeCount; i++) {
858 for (int j = 0; j < kBlockSizeCount; j++) {
859 // Need to create a new pipe each time because ReadStuff() expects
860 // to see EOF at the end.
861 ASSERT_EQ(pipe(files), 0);
862
863 {
864 FileOutputStream output(files[1], kBlockSizes[i]);
865 WriteStuff(&output);
866 EXPECT_EQ(0, output.GetErrno());
867 }
868 close(files[1]); // Send EOF.
869
870 {
871 FileInputStream input(files[0], kBlockSizes[j]);
872 ReadStuff(&input);
873 EXPECT_EQ(0, input.GetErrno());
874 }
875 close(files[0]);
876 }
877 }
878 }
879
880 // Test using C++ iostreams.
TEST_F(IoTest,IostreamIo)881 TEST_F(IoTest, IostreamIo) {
882 for (int i = 0; i < kBlockSizeCount; i++) {
883 for (int j = 0; j < kBlockSizeCount; j++) {
884 {
885 stringstream stream;
886
887 {
888 OstreamOutputStream output(&stream, kBlockSizes[i]);
889 WriteStuff(&output);
890 EXPECT_FALSE(stream.fail());
891 }
892
893 {
894 IstreamInputStream input(&stream, kBlockSizes[j]);
895 ReadStuff(&input);
896 EXPECT_TRUE(stream.eof());
897 }
898 }
899
900 {
901 stringstream stream;
902
903 {
904 OstreamOutputStream output(&stream, kBlockSizes[i]);
905 WriteStuffLarge(&output);
906 EXPECT_FALSE(stream.fail());
907 }
908
909 {
910 IstreamInputStream input(&stream, kBlockSizes[j]);
911 ReadStuffLarge(&input);
912 EXPECT_TRUE(stream.eof());
913 }
914 }
915 }
916 }
917 }
918
919 // To test ConcatenatingInputStream, we create several ArrayInputStreams
920 // covering a buffer and then concatenate them.
TEST_F(IoTest,ConcatenatingInputStream)921 TEST_F(IoTest, ConcatenatingInputStream) {
922 const int kBufferSize = 256;
923 uint8 buffer[kBufferSize];
924
925 // Fill the buffer.
926 ArrayOutputStream output(buffer, kBufferSize);
927 WriteStuff(&output);
928
929 // Now split it up into multiple streams of varying sizes.
930 ASSERT_EQ(68, output.ByteCount()); // Test depends on this.
931 ArrayInputStream input1(buffer , 12);
932 ArrayInputStream input2(buffer + 12, 7);
933 ArrayInputStream input3(buffer + 19, 6);
934 ArrayInputStream input4(buffer + 25, 15);
935 ArrayInputStream input5(buffer + 40, 0);
936 // Note: We want to make sure we have a stream boundary somewhere between
937 // bytes 42 and 62, which is the range that it Skip()ed by ReadStuff(). This
938 // tests that a bug that existed in the original code for Skip() is fixed.
939 ArrayInputStream input6(buffer + 40, 10);
940 ArrayInputStream input7(buffer + 50, 18); // Total = 68 bytes.
941
942 ZeroCopyInputStream* streams[] =
943 {&input1, &input2, &input3, &input4, &input5, &input6, &input7};
944
945 // Create the concatenating stream and read.
946 ConcatenatingInputStream input(streams, GOOGLE_ARRAYSIZE(streams));
947 ReadStuff(&input);
948 }
949
950 // To test LimitingInputStream, we write our golden text to a buffer, then
951 // create an ArrayInputStream that contains the whole buffer (not just the
952 // bytes written), then use a LimitingInputStream to limit it just to the
953 // bytes written.
TEST_F(IoTest,LimitingInputStream)954 TEST_F(IoTest, LimitingInputStream) {
955 const int kBufferSize = 256;
956 uint8 buffer[kBufferSize];
957
958 // Fill the buffer.
959 ArrayOutputStream output(buffer, kBufferSize);
960 WriteStuff(&output);
961
962 // Set up input.
963 ArrayInputStream array_input(buffer, kBufferSize);
964 LimitingInputStream input(&array_input, output.ByteCount());
965
966 ReadStuff(&input);
967 }
968
969 // Checks that ByteCount works correctly for LimitingInputStreams where the
970 // underlying stream has already been read.
TEST_F(IoTest,LimitingInputStreamByteCount)971 TEST_F(IoTest, LimitingInputStreamByteCount) {
972 const int kHalfBufferSize = 128;
973 const int kBufferSize = kHalfBufferSize * 2;
974 uint8 buffer[kBufferSize];
975
976 // Set up input. Only allow half to be read at once.
977 ArrayInputStream array_input(buffer, kBufferSize, kHalfBufferSize);
978 const void* data;
979 int size;
980 EXPECT_TRUE(array_input.Next(&data, &size));
981 EXPECT_EQ(kHalfBufferSize, array_input.ByteCount());
982 // kHalfBufferSize - 1 to test limiting logic as well.
983 LimitingInputStream input(&array_input, kHalfBufferSize - 1);
984 EXPECT_EQ(0, input.ByteCount());
985 EXPECT_TRUE(input.Next(&data, &size));
986 EXPECT_EQ(kHalfBufferSize - 1 , input.ByteCount());
987 }
988
989 // Check that a zero-size array doesn't confuse the code.
TEST(ZeroSizeArray,Input)990 TEST(ZeroSizeArray, Input) {
991 ArrayInputStream input(NULL, 0);
992 const void* data;
993 int size;
994 EXPECT_FALSE(input.Next(&data, &size));
995 }
996
TEST(ZeroSizeArray,Output)997 TEST(ZeroSizeArray, Output) {
998 ArrayOutputStream output(NULL, 0);
999 void* data;
1000 int size;
1001 EXPECT_FALSE(output.Next(&data, &size));
1002 }
1003
1004 } // namespace
1005 } // namespace io
1006 } // namespace protobuf
1007 } // namespace google
1008