• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc.  All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 //     * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 //     * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 //     * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 
31 // Author: kenton@google.com (Kenton Varda)
32 //  Based on original Protocol Buffers design by
33 //  Sanjay Ghemawat, Jeff Dean, and others.
34 //
35 // Testing strategy:  For each type of I/O (array, string, file, etc.) we
36 // create an output stream and write some data to it, then create a
37 // corresponding input stream to read the same data back and expect it to
38 // match.  When the data is written, it is written in several small chunks
39 // of varying sizes, with a BackUp() after each chunk.  It is read back
40 // similarly, but with chunks separated at different points.  The whole
41 // process is run with a variety of block sizes for both the input and
42 // the output.
43 //
44 // TODO(kenton):  Rewrite this test to bring it up to the standards of all
45 //   the other proto2 tests.  May want to wait for gTest to implement
46 //   "parametized tests" so that one set of tests can be used on all the
47 //   implementations.
48 
49 
50 #ifdef _MSC_VER
51 #include <io.h>
52 #else
53 #include <unistd.h>
54 #endif
55 #include <stdlib.h>
56 #include <sys/types.h>
57 #include <sys/stat.h>
58 #include <fcntl.h>
59 #include <errno.h>
60 #include <memory>
61 #ifndef _SHARED_PTR_H
62 #include <google/protobuf/stubs/shared_ptr.h>
63 #endif
64 #include <sstream>
65 
66 #include <google/protobuf/testing/file.h>
67 #include <google/protobuf/io/coded_stream.h>
68 #include <google/protobuf/io/zero_copy_stream_impl.h>
69 
70 #if HAVE_ZLIB
71 #include <google/protobuf/io/gzip_stream.h>
72 #endif
73 
74 #include <google/protobuf/stubs/common.h>
75 #include <google/protobuf/stubs/logging.h>
76 #include <google/protobuf/testing/googletest.h>
77 #include <google/protobuf/testing/file.h>
78 #include <gtest/gtest.h>
79 
80 namespace google {
81 namespace protobuf {
82 namespace io {
83 namespace {
84 
85 #ifdef _WIN32
86 #define pipe(fds) _pipe(fds, 4096, O_BINARY)
87 #endif
88 
89 #ifndef O_BINARY
90 #ifdef _O_BINARY
91 #define O_BINARY _O_BINARY
92 #else
93 #define O_BINARY 0     // If this isn't defined, the platform doesn't need it.
94 #endif
95 #endif
96 
97 class IoTest : public testing::Test {
98  protected:
99   // Test helpers.
100 
101   // Helper to write an array of data to an output stream.
102   bool WriteToOutput(ZeroCopyOutputStream* output, const void* data, int size);
103   // Helper to read a fixed-length array of data from an input stream.
104   int ReadFromInput(ZeroCopyInputStream* input, void* data, int size);
105   // Write a string to the output stream.
106   void WriteString(ZeroCopyOutputStream* output, const string& str);
107   // Read a number of bytes equal to the size of the given string and checks
108   // that it matches the string.
109   void ReadString(ZeroCopyInputStream* input, const string& str);
110   // Writes some text to the output stream in a particular order.  Returns
111   // the number of bytes written, incase the caller needs that to set up an
112   // input stream.
113   int WriteStuff(ZeroCopyOutputStream* output);
114   // Reads text from an input stream and expects it to match what
115   // WriteStuff() writes.
116   void ReadStuff(ZeroCopyInputStream* input);
117 
118   // Similar to WriteStuff, but performs more sophisticated testing.
119   int WriteStuffLarge(ZeroCopyOutputStream* output);
120   // Reads and tests a stream that should have been written to
121   // via WriteStuffLarge().
122   void ReadStuffLarge(ZeroCopyInputStream* input);
123 
124 #if HAVE_ZLIB
125   string Compress(const string& data, const GzipOutputStream::Options& options);
126   string Uncompress(const string& data);
127 #endif
128 
129   static const int kBlockSizes[];
130   static const int kBlockSizeCount;
131 };
132 
133 const int IoTest::kBlockSizes[] = {-1, 1, 2, 5, 7, 10, 23, 64};
134 const int IoTest::kBlockSizeCount = GOOGLE_ARRAYSIZE(IoTest::kBlockSizes);
135 
WriteToOutput(ZeroCopyOutputStream * output,const void * data,int size)136 bool IoTest::WriteToOutput(ZeroCopyOutputStream* output,
137                            const void* data, int size) {
138   const uint8* in = reinterpret_cast<const uint8*>(data);
139   int in_size = size;
140 
141   void* out;
142   int out_size;
143 
144   while (true) {
145     if (!output->Next(&out, &out_size)) {
146       return false;
147     }
148     EXPECT_GT(out_size, 0);
149 
150     if (in_size <= out_size) {
151       memcpy(out, in, in_size);
152       output->BackUp(out_size - in_size);
153       return true;
154     }
155 
156     memcpy(out, in, out_size);
157     in += out_size;
158     in_size -= out_size;
159   }
160 }
161 
162 #define MAX_REPEATED_ZEROS 100
163 
ReadFromInput(ZeroCopyInputStream * input,void * data,int size)164 int IoTest::ReadFromInput(ZeroCopyInputStream* input, void* data, int size) {
165   uint8* out = reinterpret_cast<uint8*>(data);
166   int out_size = size;
167 
168   const void* in;
169   int in_size = 0;
170 
171   int repeated_zeros = 0;
172 
173   while (true) {
174     if (!input->Next(&in, &in_size)) {
175       return size - out_size;
176     }
177     EXPECT_GT(in_size, -1);
178     if (in_size == 0) {
179       repeated_zeros++;
180     } else {
181       repeated_zeros = 0;
182     }
183     EXPECT_LT(repeated_zeros, MAX_REPEATED_ZEROS);
184 
185     if (out_size <= in_size) {
186       memcpy(out, in, out_size);
187       if (in_size > out_size) {
188         input->BackUp(in_size - out_size);
189       }
190       return size;  // Copied all of it.
191     }
192 
193     memcpy(out, in, in_size);
194     out += in_size;
195     out_size -= in_size;
196   }
197 }
198 
WriteString(ZeroCopyOutputStream * output,const string & str)199 void IoTest::WriteString(ZeroCopyOutputStream* output, const string& str) {
200   EXPECT_TRUE(WriteToOutput(output, str.c_str(), str.size()));
201 }
202 
ReadString(ZeroCopyInputStream * input,const string & str)203 void IoTest::ReadString(ZeroCopyInputStream* input, const string& str) {
204   google::protobuf::scoped_array<char> buffer(new char[str.size() + 1]);
205   buffer[str.size()] = '\0';
206   EXPECT_EQ(ReadFromInput(input, buffer.get(), str.size()), str.size());
207   EXPECT_STREQ(str.c_str(), buffer.get());
208 }
209 
WriteStuff(ZeroCopyOutputStream * output)210 int IoTest::WriteStuff(ZeroCopyOutputStream* output) {
211   WriteString(output, "Hello world!\n");
212   WriteString(output, "Some te");
213   WriteString(output, "xt.  Blah blah.");
214   WriteString(output, "abcdefg");
215   WriteString(output, "01234567890123456789");
216   WriteString(output, "foobar");
217 
218   EXPECT_EQ(output->ByteCount(), 68);
219 
220   int result = output->ByteCount();
221   return result;
222 }
223 
224 // Reads text from an input stream and expects it to match what WriteStuff()
225 // writes.
ReadStuff(ZeroCopyInputStream * input)226 void IoTest::ReadStuff(ZeroCopyInputStream* input) {
227   ReadString(input, "Hello world!\n");
228   ReadString(input, "Some text.  ");
229   ReadString(input, "Blah ");
230   ReadString(input, "blah.");
231   ReadString(input, "abcdefg");
232   EXPECT_TRUE(input->Skip(20));
233   ReadString(input, "foo");
234   ReadString(input, "bar");
235 
236   EXPECT_EQ(input->ByteCount(), 68);
237 
238   uint8 byte;
239   EXPECT_EQ(ReadFromInput(input, &byte, 1), 0);
240 }
241 
WriteStuffLarge(ZeroCopyOutputStream * output)242 int IoTest::WriteStuffLarge(ZeroCopyOutputStream* output) {
243   WriteString(output, "Hello world!\n");
244   WriteString(output, "Some te");
245   WriteString(output, "xt.  Blah blah.");
246   WriteString(output, string(100000, 'x'));  // A very long string
247   WriteString(output, string(100000, 'y'));  // A very long string
248   WriteString(output, "01234567890123456789");
249 
250   EXPECT_EQ(output->ByteCount(), 200055);
251 
252   int result = output->ByteCount();
253   return result;
254 }
255 
256 // Reads text from an input stream and expects it to match what WriteStuff()
257 // writes.
ReadStuffLarge(ZeroCopyInputStream * input)258 void IoTest::ReadStuffLarge(ZeroCopyInputStream* input) {
259   ReadString(input, "Hello world!\nSome text.  ");
260   EXPECT_TRUE(input->Skip(5));
261   ReadString(input, "blah.");
262   EXPECT_TRUE(input->Skip(100000 - 10));
263   ReadString(input, string(10, 'x') + string(100000 - 20000, 'y'));
264   EXPECT_TRUE(input->Skip(20000 - 10));
265   ReadString(input, "yyyyyyyyyy01234567890123456789");
266 
267   EXPECT_EQ(input->ByteCount(), 200055);
268 
269   uint8 byte;
270   EXPECT_EQ(ReadFromInput(input, &byte, 1), 0);
271 }
272 
273 // ===================================================================
274 
TEST_F(IoTest,ArrayIo)275 TEST_F(IoTest, ArrayIo) {
276   const int kBufferSize = 256;
277   uint8 buffer[kBufferSize];
278 
279   for (int i = 0; i < kBlockSizeCount; i++) {
280     for (int j = 0; j < kBlockSizeCount; j++) {
281       int size;
282       {
283         ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
284         size = WriteStuff(&output);
285       }
286       {
287         ArrayInputStream input(buffer, size, kBlockSizes[j]);
288         ReadStuff(&input);
289       }
290     }
291   }
292 }
293 
TEST_F(IoTest,TwoSessionWrite)294 TEST_F(IoTest, TwoSessionWrite) {
295   // Test that two concatenated write sessions read correctly
296 
297   static const char* strA = "0123456789";
298   static const char* strB = "WhirledPeas";
299   const int kBufferSize = 2*1024;
300   uint8* buffer = new uint8[kBufferSize];
301   char* temp_buffer = new char[40];
302 
303   for (int i = 0; i < kBlockSizeCount; i++) {
304     for (int j = 0; j < kBlockSizeCount; j++) {
305       ArrayOutputStream* output =
306           new ArrayOutputStream(buffer, kBufferSize, kBlockSizes[i]);
307       CodedOutputStream* coded_output = new CodedOutputStream(output);
308       coded_output->WriteVarint32(strlen(strA));
309       coded_output->WriteRaw(strA, strlen(strA));
310       delete coded_output;  // flush
311       int64 pos = output->ByteCount();
312       delete output;
313       output = new ArrayOutputStream(
314           buffer + pos, kBufferSize - pos, kBlockSizes[i]);
315       coded_output = new CodedOutputStream(output);
316       coded_output->WriteVarint32(strlen(strB));
317       coded_output->WriteRaw(strB, strlen(strB));
318       delete coded_output;  // flush
319       int64 size = pos + output->ByteCount();
320       delete output;
321 
322       ArrayInputStream* input =
323           new ArrayInputStream(buffer, size, kBlockSizes[j]);
324       CodedInputStream* coded_input = new CodedInputStream(input);
325       uint32 insize;
326       EXPECT_TRUE(coded_input->ReadVarint32(&insize));
327       EXPECT_EQ(strlen(strA), insize);
328       EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
329       EXPECT_EQ(0, memcmp(temp_buffer, strA, insize));
330 
331       EXPECT_TRUE(coded_input->ReadVarint32(&insize));
332       EXPECT_EQ(strlen(strB), insize);
333       EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
334       EXPECT_EQ(0, memcmp(temp_buffer, strB, insize));
335 
336       delete coded_input;
337       delete input;
338     }
339   }
340 
341   delete [] temp_buffer;
342   delete [] buffer;
343 }
344 
345 #if HAVE_ZLIB
TEST_F(IoTest,GzipIo)346 TEST_F(IoTest, GzipIo) {
347   const int kBufferSize = 2*1024;
348   uint8* buffer = new uint8[kBufferSize];
349   for (int i = 0; i < kBlockSizeCount; i++) {
350     for (int j = 0; j < kBlockSizeCount; j++) {
351       for (int z = 0; z < kBlockSizeCount; z++) {
352         int gzip_buffer_size = kBlockSizes[z];
353         int size;
354         {
355           ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
356           GzipOutputStream::Options options;
357           options.format = GzipOutputStream::GZIP;
358           if (gzip_buffer_size != -1) {
359             options.buffer_size = gzip_buffer_size;
360           }
361           GzipOutputStream gzout(&output, options);
362           WriteStuff(&gzout);
363           gzout.Close();
364           size = output.ByteCount();
365         }
366         {
367           ArrayInputStream input(buffer, size, kBlockSizes[j]);
368           GzipInputStream gzin(
369               &input, GzipInputStream::GZIP, gzip_buffer_size);
370           ReadStuff(&gzin);
371         }
372       }
373     }
374   }
375   delete [] buffer;
376 }
377 
TEST_F(IoTest,GzipIoWithFlush)378 TEST_F(IoTest, GzipIoWithFlush) {
379   const int kBufferSize = 2*1024;
380   uint8* buffer = new uint8[kBufferSize];
381   // We start with i = 4 as we want a block size > 6. With block size <= 6
382   // Flush() fills up the entire 2K buffer with flush markers and the test
383   // fails. See documentation for Flush() for more detail.
384   for (int i = 4; i < kBlockSizeCount; i++) {
385     for (int j = 0; j < kBlockSizeCount; j++) {
386       for (int z = 0; z < kBlockSizeCount; z++) {
387         int gzip_buffer_size = kBlockSizes[z];
388         int size;
389         {
390           ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
391           GzipOutputStream::Options options;
392           options.format = GzipOutputStream::GZIP;
393           if (gzip_buffer_size != -1) {
394             options.buffer_size = gzip_buffer_size;
395           }
396           GzipOutputStream gzout(&output, options);
397           WriteStuff(&gzout);
398           EXPECT_TRUE(gzout.Flush());
399           gzout.Close();
400           size = output.ByteCount();
401         }
402         {
403           ArrayInputStream input(buffer, size, kBlockSizes[j]);
404           GzipInputStream gzin(
405               &input, GzipInputStream::GZIP, gzip_buffer_size);
406           ReadStuff(&gzin);
407         }
408       }
409     }
410   }
411   delete [] buffer;
412 }
413 
TEST_F(IoTest,GzipIoContiguousFlushes)414 TEST_F(IoTest, GzipIoContiguousFlushes) {
415   const int kBufferSize = 2*1024;
416   uint8* buffer = new uint8[kBufferSize];
417 
418   int block_size = kBlockSizes[4];
419   int gzip_buffer_size = block_size;
420   int size;
421 
422   ArrayOutputStream output(buffer, kBufferSize, block_size);
423   GzipOutputStream::Options options;
424   options.format = GzipOutputStream::GZIP;
425   if (gzip_buffer_size != -1) {
426     options.buffer_size = gzip_buffer_size;
427   }
428   GzipOutputStream gzout(&output, options);
429   WriteStuff(&gzout);
430   EXPECT_TRUE(gzout.Flush());
431   EXPECT_TRUE(gzout.Flush());
432   gzout.Close();
433   size = output.ByteCount();
434 
435   ArrayInputStream input(buffer, size, block_size);
436   GzipInputStream gzin(
437       &input, GzipInputStream::GZIP, gzip_buffer_size);
438   ReadStuff(&gzin);
439 
440   delete [] buffer;
441 }
442 
TEST_F(IoTest,GzipIoReadAfterFlush)443 TEST_F(IoTest, GzipIoReadAfterFlush) {
444   const int kBufferSize = 2*1024;
445   uint8* buffer = new uint8[kBufferSize];
446 
447   int block_size = kBlockSizes[4];
448   int gzip_buffer_size = block_size;
449   int size;
450   ArrayOutputStream output(buffer, kBufferSize, block_size);
451   GzipOutputStream::Options options;
452   options.format = GzipOutputStream::GZIP;
453   if (gzip_buffer_size != -1) {
454     options.buffer_size = gzip_buffer_size;
455   }
456 
457   GzipOutputStream gzout(&output, options);
458   WriteStuff(&gzout);
459   EXPECT_TRUE(gzout.Flush());
460   size = output.ByteCount();
461 
462   ArrayInputStream input(buffer, size, block_size);
463   GzipInputStream gzin(
464       &input, GzipInputStream::GZIP, gzip_buffer_size);
465   ReadStuff(&gzin);
466 
467   gzout.Close();
468 
469   delete [] buffer;
470 }
471 
TEST_F(IoTest,ZlibIo)472 TEST_F(IoTest, ZlibIo) {
473   const int kBufferSize = 2*1024;
474   uint8* buffer = new uint8[kBufferSize];
475   for (int i = 0; i < kBlockSizeCount; i++) {
476     for (int j = 0; j < kBlockSizeCount; j++) {
477       for (int z = 0; z < kBlockSizeCount; z++) {
478         int gzip_buffer_size = kBlockSizes[z];
479         int size;
480         {
481           ArrayOutputStream output(buffer, kBufferSize, kBlockSizes[i]);
482           GzipOutputStream::Options options;
483           options.format = GzipOutputStream::ZLIB;
484           if (gzip_buffer_size != -1) {
485             options.buffer_size = gzip_buffer_size;
486           }
487           GzipOutputStream gzout(&output, options);
488           WriteStuff(&gzout);
489           gzout.Close();
490           size = output.ByteCount();
491         }
492         {
493           ArrayInputStream input(buffer, size, kBlockSizes[j]);
494           GzipInputStream gzin(
495               &input, GzipInputStream::ZLIB, gzip_buffer_size);
496           ReadStuff(&gzin);
497         }
498       }
499     }
500   }
501   delete [] buffer;
502 }
503 
TEST_F(IoTest,ZlibIoInputAutodetect)504 TEST_F(IoTest, ZlibIoInputAutodetect) {
505   const int kBufferSize = 2*1024;
506   uint8* buffer = new uint8[kBufferSize];
507   int size;
508   {
509     ArrayOutputStream output(buffer, kBufferSize);
510     GzipOutputStream::Options options;
511     options.format = GzipOutputStream::ZLIB;
512     GzipOutputStream gzout(&output, options);
513     WriteStuff(&gzout);
514     gzout.Close();
515     size = output.ByteCount();
516   }
517   {
518     ArrayInputStream input(buffer, size);
519     GzipInputStream gzin(&input, GzipInputStream::AUTO);
520     ReadStuff(&gzin);
521   }
522   {
523     ArrayOutputStream output(buffer, kBufferSize);
524     GzipOutputStream::Options options;
525     options.format = GzipOutputStream::GZIP;
526     GzipOutputStream gzout(&output, options);
527     WriteStuff(&gzout);
528     gzout.Close();
529     size = output.ByteCount();
530   }
531   {
532     ArrayInputStream input(buffer, size);
533     GzipInputStream gzin(&input, GzipInputStream::AUTO);
534     ReadStuff(&gzin);
535   }
536   delete [] buffer;
537 }
538 
Compress(const string & data,const GzipOutputStream::Options & options)539 string IoTest::Compress(const string& data,
540                         const GzipOutputStream::Options& options) {
541   string result;
542   {
543     StringOutputStream output(&result);
544     GzipOutputStream gzout(&output, options);
545     WriteToOutput(&gzout, data.data(), data.size());
546   }
547   return result;
548 }
549 
Uncompress(const string & data)550 string IoTest::Uncompress(const string& data) {
551   string result;
552   {
553     ArrayInputStream input(data.data(), data.size());
554     GzipInputStream gzin(&input);
555     const void* buffer;
556     int size;
557     while (gzin.Next(&buffer, &size)) {
558       result.append(reinterpret_cast<const char*>(buffer), size);
559     }
560   }
561   return result;
562 }
563 
TEST_F(IoTest,CompressionOptions)564 TEST_F(IoTest, CompressionOptions) {
565   // Some ad-hoc testing of compression options.
566 
567   string golden;
568   GOOGLE_CHECK_OK(File::GetContents(
569       TestSourceDir() +
570           "/google/protobuf/testdata/golden_message",
571       &golden, true));
572 
573   GzipOutputStream::Options options;
574   string gzip_compressed = Compress(golden, options);
575 
576   options.compression_level = 0;
577   string not_compressed = Compress(golden, options);
578 
579   // Try zlib compression for fun.
580   options = GzipOutputStream::Options();
581   options.format = GzipOutputStream::ZLIB;
582   string zlib_compressed = Compress(golden, options);
583 
584   // Uncompressed should be bigger than the original since it should have some
585   // sort of header.
586   EXPECT_GT(not_compressed.size(), golden.size());
587 
588   // Higher compression levels should result in smaller sizes.
589   EXPECT_LT(zlib_compressed.size(), not_compressed.size());
590 
591   // ZLIB format should differ from GZIP format.
592   EXPECT_TRUE(zlib_compressed != gzip_compressed);
593 
594   // Everything should decompress correctly.
595   EXPECT_TRUE(Uncompress(not_compressed) == golden);
596   EXPECT_TRUE(Uncompress(gzip_compressed) == golden);
597   EXPECT_TRUE(Uncompress(zlib_compressed) == golden);
598 }
599 
TEST_F(IoTest,TwoSessionWriteGzip)600 TEST_F(IoTest, TwoSessionWriteGzip) {
601   // Test that two concatenated gzip streams can be read correctly
602 
603   static const char* strA = "0123456789";
604   static const char* strB = "QuickBrownFox";
605   const int kBufferSize = 2*1024;
606   uint8* buffer = new uint8[kBufferSize];
607   char* temp_buffer = new char[40];
608 
609   for (int i = 0; i < kBlockSizeCount; i++) {
610     for (int j = 0; j < kBlockSizeCount; j++) {
611       ArrayOutputStream* output =
612           new ArrayOutputStream(buffer, kBufferSize, kBlockSizes[i]);
613       GzipOutputStream* gzout = new GzipOutputStream(output);
614       CodedOutputStream* coded_output = new CodedOutputStream(gzout);
615       int32 outlen = strlen(strA) + 1;
616       coded_output->WriteVarint32(outlen);
617       coded_output->WriteRaw(strA, outlen);
618       delete coded_output;  // flush
619       delete gzout;  // flush
620       int64 pos = output->ByteCount();
621       delete output;
622       output = new ArrayOutputStream(
623           buffer + pos, kBufferSize - pos, kBlockSizes[i]);
624       gzout = new GzipOutputStream(output);
625       coded_output = new CodedOutputStream(gzout);
626       outlen = strlen(strB) + 1;
627       coded_output->WriteVarint32(outlen);
628       coded_output->WriteRaw(strB, outlen);
629       delete coded_output;  // flush
630       delete gzout;  // flush
631       int64 size = pos + output->ByteCount();
632       delete output;
633 
634       ArrayInputStream* input =
635           new ArrayInputStream(buffer, size, kBlockSizes[j]);
636       GzipInputStream* gzin = new GzipInputStream(input);
637       CodedInputStream* coded_input = new CodedInputStream(gzin);
638       uint32 insize;
639       EXPECT_TRUE(coded_input->ReadVarint32(&insize));
640       EXPECT_EQ(strlen(strA) + 1, insize);
641       EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
642       EXPECT_EQ(0, memcmp(temp_buffer, strA, insize))
643           << "strA=" << strA << " in=" << temp_buffer;
644 
645       EXPECT_TRUE(coded_input->ReadVarint32(&insize));
646       EXPECT_EQ(strlen(strB) + 1, insize);
647       EXPECT_TRUE(coded_input->ReadRaw(temp_buffer, insize));
648       EXPECT_EQ(0, memcmp(temp_buffer, strB, insize))
649           << " out_block_size=" << kBlockSizes[i]
650           << " in_block_size=" << kBlockSizes[j]
651           << " pos=" << pos
652           << " size=" << size
653           << " strB=" << strB << " in=" << temp_buffer;
654 
655       delete coded_input;
656       delete gzin;
657       delete input;
658     }
659   }
660 
661   delete [] temp_buffer;
662   delete [] buffer;
663 }
664 
TEST_F(IoTest,GzipInputByteCountAfterClosed)665 TEST_F(IoTest, GzipInputByteCountAfterClosed) {
666   string golden = "abcdefghijklmnopqrstuvwxyz";
667   string compressed = Compress(golden, GzipOutputStream::Options());
668 
669   for (int i = 0; i < kBlockSizeCount; i++) {
670     ArrayInputStream arr_input(compressed.data(), compressed.size(),
671                                kBlockSizes[i]);
672     GzipInputStream gz_input(&arr_input);
673     const void* buffer;
674     int size;
675     while (gz_input.Next(&buffer, &size)) {
676       EXPECT_LE(gz_input.ByteCount(), golden.size());
677     }
678     EXPECT_EQ(golden.size(), gz_input.ByteCount());
679   }
680 }
681 
TEST_F(IoTest,GzipInputByteCountAfterClosedConcatenatedStreams)682 TEST_F(IoTest, GzipInputByteCountAfterClosedConcatenatedStreams) {
683   string golden1 = "abcdefghijklmnopqrstuvwxyz";
684   string golden2 = "the quick brown fox jumps over the lazy dog";
685   const size_t total_size = golden1.size() + golden2.size();
686   string compressed = Compress(golden1, GzipOutputStream::Options()) +
687                       Compress(golden2, GzipOutputStream::Options());
688 
689   for (int i = 0; i < kBlockSizeCount; i++) {
690     ArrayInputStream arr_input(compressed.data(), compressed.size(),
691                                kBlockSizes[i]);
692     GzipInputStream gz_input(&arr_input);
693     const void* buffer;
694     int size;
695     while (gz_input.Next(&buffer, &size)) {
696       EXPECT_LE(gz_input.ByteCount(), total_size);
697     }
698     EXPECT_EQ(total_size, gz_input.ByteCount());
699   }
700 }
701 #endif
702 
703 // There is no string input, only string output.  Also, it doesn't support
704 // explicit block sizes.  So, we'll only run one test and we'll use
705 // ArrayInput to read back the results.
TEST_F(IoTest,StringIo)706 TEST_F(IoTest, StringIo) {
707   string str;
708   {
709     StringOutputStream output(&str);
710     WriteStuff(&output);
711   }
712   {
713     ArrayInputStream input(str.data(), str.size());
714     ReadStuff(&input);
715   }
716 }
717 
718 
719 // To test files, we create a temporary file, write, read, truncate, repeat.
TEST_F(IoTest,FileIo)720 TEST_F(IoTest, FileIo) {
721   string filename = TestTempDir() + "/zero_copy_stream_test_file";
722 
723   for (int i = 0; i < kBlockSizeCount; i++) {
724     for (int j = 0; j < kBlockSizeCount; j++) {
725       // Make a temporary file.
726       int file =
727         open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0777);
728       ASSERT_GE(file, 0);
729 
730       {
731         FileOutputStream output(file, kBlockSizes[i]);
732         WriteStuff(&output);
733         EXPECT_EQ(0, output.GetErrno());
734       }
735 
736       // Rewind.
737       ASSERT_NE(lseek(file, 0, SEEK_SET), (off_t)-1);
738 
739       {
740         FileInputStream input(file, kBlockSizes[j]);
741         ReadStuff(&input);
742         EXPECT_EQ(0, input.GetErrno());
743       }
744 
745       close(file);
746     }
747   }
748 }
749 
750 #if HAVE_ZLIB
TEST_F(IoTest,GzipFileIo)751 TEST_F(IoTest, GzipFileIo) {
752   string filename = TestTempDir() + "/zero_copy_stream_test_file";
753 
754   for (int i = 0; i < kBlockSizeCount; i++) {
755     for (int j = 0; j < kBlockSizeCount; j++) {
756       // Make a temporary file.
757       int file =
758         open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0777);
759       ASSERT_GE(file, 0);
760       {
761         FileOutputStream output(file, kBlockSizes[i]);
762         GzipOutputStream gzout(&output);
763         WriteStuffLarge(&gzout);
764         gzout.Close();
765         output.Flush();
766         EXPECT_EQ(0, output.GetErrno());
767       }
768 
769       // Rewind.
770       ASSERT_NE(lseek(file, 0, SEEK_SET), (off_t)-1);
771 
772       {
773         FileInputStream input(file, kBlockSizes[j]);
774         GzipInputStream gzin(&input);
775         ReadStuffLarge(&gzin);
776         EXPECT_EQ(0, input.GetErrno());
777       }
778 
779       close(file);
780     }
781   }
782 }
783 #endif
784 
785 // MSVC raises various debugging exceptions if we try to use a file
786 // descriptor of -1, defeating our tests below.  This class will disable
787 // these debug assertions while in scope.
788 class MsvcDebugDisabler {
789  public:
790 #if defined(_MSC_VER) && _MSC_VER >= 1400
MsvcDebugDisabler()791   MsvcDebugDisabler() {
792     old_handler_ = _set_invalid_parameter_handler(MyHandler);
793     old_mode_ = _CrtSetReportMode(_CRT_ASSERT, 0);
794   }
~MsvcDebugDisabler()795   ~MsvcDebugDisabler() {
796     old_handler_ = _set_invalid_parameter_handler(old_handler_);
797     old_mode_ = _CrtSetReportMode(_CRT_ASSERT, old_mode_);
798   }
799 
MyHandler(const wchar_t * expr,const wchar_t * func,const wchar_t * file,unsigned int line,uintptr_t pReserved)800   static void MyHandler(const wchar_t *expr,
801                         const wchar_t *func,
802                         const wchar_t *file,
803                         unsigned int line,
804                         uintptr_t pReserved) {
805     // do nothing
806   }
807 
808   _invalid_parameter_handler old_handler_;
809   int old_mode_;
810 #else
811   // Dummy constructor and destructor to ensure that GCC doesn't complain
812   // that debug_disabler is an unused variable.
813   MsvcDebugDisabler() {}
814   ~MsvcDebugDisabler() {}
815 #endif
816 };
817 
818 // Test that FileInputStreams report errors correctly.
TEST_F(IoTest,FileReadError)819 TEST_F(IoTest, FileReadError) {
820   MsvcDebugDisabler debug_disabler;
821 
822   // -1 = invalid file descriptor.
823   FileInputStream input(-1);
824 
825   const void* buffer;
826   int size;
827   EXPECT_FALSE(input.Next(&buffer, &size));
828   EXPECT_EQ(EBADF, input.GetErrno());
829 }
830 
831 // Test that FileOutputStreams report errors correctly.
TEST_F(IoTest,FileWriteError)832 TEST_F(IoTest, FileWriteError) {
833   MsvcDebugDisabler debug_disabler;
834 
835   // -1 = invalid file descriptor.
836   FileOutputStream input(-1);
837 
838   void* buffer;
839   int size;
840 
841   // The first call to Next() succeeds because it doesn't have anything to
842   // write yet.
843   EXPECT_TRUE(input.Next(&buffer, &size));
844 
845   // Second call fails.
846   EXPECT_FALSE(input.Next(&buffer, &size));
847 
848   EXPECT_EQ(EBADF, input.GetErrno());
849 }
850 
851 // Pipes are not seekable, so File{Input,Output}Stream ends up doing some
852 // different things to handle them.  We'll test by writing to a pipe and
853 // reading back from it.
TEST_F(IoTest,PipeIo)854 TEST_F(IoTest, PipeIo) {
855   int files[2];
856 
857   for (int i = 0; i < kBlockSizeCount; i++) {
858     for (int j = 0; j < kBlockSizeCount; j++) {
859       // Need to create a new pipe each time because ReadStuff() expects
860       // to see EOF at the end.
861       ASSERT_EQ(pipe(files), 0);
862 
863       {
864         FileOutputStream output(files[1], kBlockSizes[i]);
865         WriteStuff(&output);
866         EXPECT_EQ(0, output.GetErrno());
867       }
868       close(files[1]);  // Send EOF.
869 
870       {
871         FileInputStream input(files[0], kBlockSizes[j]);
872         ReadStuff(&input);
873         EXPECT_EQ(0, input.GetErrno());
874       }
875       close(files[0]);
876     }
877   }
878 }
879 
880 // Test using C++ iostreams.
TEST_F(IoTest,IostreamIo)881 TEST_F(IoTest, IostreamIo) {
882   for (int i = 0; i < kBlockSizeCount; i++) {
883     for (int j = 0; j < kBlockSizeCount; j++) {
884       {
885         stringstream stream;
886 
887         {
888           OstreamOutputStream output(&stream, kBlockSizes[i]);
889           WriteStuff(&output);
890           EXPECT_FALSE(stream.fail());
891         }
892 
893         {
894           IstreamInputStream input(&stream, kBlockSizes[j]);
895           ReadStuff(&input);
896           EXPECT_TRUE(stream.eof());
897         }
898       }
899 
900       {
901         stringstream stream;
902 
903         {
904           OstreamOutputStream output(&stream, kBlockSizes[i]);
905           WriteStuffLarge(&output);
906           EXPECT_FALSE(stream.fail());
907         }
908 
909         {
910           IstreamInputStream input(&stream, kBlockSizes[j]);
911           ReadStuffLarge(&input);
912           EXPECT_TRUE(stream.eof());
913         }
914       }
915     }
916   }
917 }
918 
919 // To test ConcatenatingInputStream, we create several ArrayInputStreams
920 // covering a buffer and then concatenate them.
TEST_F(IoTest,ConcatenatingInputStream)921 TEST_F(IoTest, ConcatenatingInputStream) {
922   const int kBufferSize = 256;
923   uint8 buffer[kBufferSize];
924 
925   // Fill the buffer.
926   ArrayOutputStream output(buffer, kBufferSize);
927   WriteStuff(&output);
928 
929   // Now split it up into multiple streams of varying sizes.
930   ASSERT_EQ(68, output.ByteCount());  // Test depends on this.
931   ArrayInputStream input1(buffer     , 12);
932   ArrayInputStream input2(buffer + 12,  7);
933   ArrayInputStream input3(buffer + 19,  6);
934   ArrayInputStream input4(buffer + 25, 15);
935   ArrayInputStream input5(buffer + 40,  0);
936   // Note:  We want to make sure we have a stream boundary somewhere between
937   // bytes 42 and 62, which is the range that it Skip()ed by ReadStuff().  This
938   // tests that a bug that existed in the original code for Skip() is fixed.
939   ArrayInputStream input6(buffer + 40, 10);
940   ArrayInputStream input7(buffer + 50, 18);  // Total = 68 bytes.
941 
942   ZeroCopyInputStream* streams[] =
943     {&input1, &input2, &input3, &input4, &input5, &input6, &input7};
944 
945   // Create the concatenating stream and read.
946   ConcatenatingInputStream input(streams, GOOGLE_ARRAYSIZE(streams));
947   ReadStuff(&input);
948 }
949 
950 // To test LimitingInputStream, we write our golden text to a buffer, then
951 // create an ArrayInputStream that contains the whole buffer (not just the
952 // bytes written), then use a LimitingInputStream to limit it just to the
953 // bytes written.
TEST_F(IoTest,LimitingInputStream)954 TEST_F(IoTest, LimitingInputStream) {
955   const int kBufferSize = 256;
956   uint8 buffer[kBufferSize];
957 
958   // Fill the buffer.
959   ArrayOutputStream output(buffer, kBufferSize);
960   WriteStuff(&output);
961 
962   // Set up input.
963   ArrayInputStream array_input(buffer, kBufferSize);
964   LimitingInputStream input(&array_input, output.ByteCount());
965 
966   ReadStuff(&input);
967 }
968 
969 // Checks that ByteCount works correctly for LimitingInputStreams where the
970 // underlying stream has already been read.
TEST_F(IoTest,LimitingInputStreamByteCount)971 TEST_F(IoTest, LimitingInputStreamByteCount) {
972   const int kHalfBufferSize = 128;
973   const int kBufferSize = kHalfBufferSize * 2;
974   uint8 buffer[kBufferSize];
975 
976   // Set up input. Only allow half to be read at once.
977   ArrayInputStream array_input(buffer, kBufferSize, kHalfBufferSize);
978   const void* data;
979   int size;
980   EXPECT_TRUE(array_input.Next(&data, &size));
981   EXPECT_EQ(kHalfBufferSize, array_input.ByteCount());
982   // kHalfBufferSize - 1 to test limiting logic as well.
983   LimitingInputStream input(&array_input, kHalfBufferSize - 1);
984   EXPECT_EQ(0, input.ByteCount());
985   EXPECT_TRUE(input.Next(&data, &size));
986   EXPECT_EQ(kHalfBufferSize - 1 , input.ByteCount());
987 }
988 
989 // Check that a zero-size array doesn't confuse the code.
TEST(ZeroSizeArray,Input)990 TEST(ZeroSizeArray, Input) {
991   ArrayInputStream input(NULL, 0);
992   const void* data;
993   int size;
994   EXPECT_FALSE(input.Next(&data, &size));
995 }
996 
TEST(ZeroSizeArray,Output)997 TEST(ZeroSizeArray, Output) {
998   ArrayOutputStream output(NULL, 0);
999   void* data;
1000   int size;
1001   EXPECT_FALSE(output.Next(&data, &size));
1002 }
1003 
1004 }  // namespace
1005 }  // namespace io
1006 }  // namespace protobuf
1007 }  // namespace google
1008