1 #region Copyright notice and license 2 // Protocol Buffers - Google's data interchange format 3 // Copyright 2008 Google Inc. All rights reserved. 4 // https://developers.google.com/protocol-buffers/ 5 // 6 // Redistribution and use in source and binary forms, with or without 7 // modification, are permitted provided that the following conditions are 8 // met: 9 // 10 // * Redistributions of source code must retain the above copyright 11 // notice, this list of conditions and the following disclaimer. 12 // * Redistributions in binary form must reproduce the above 13 // copyright notice, this list of conditions and the following disclaimer 14 // in the documentation and/or other materials provided with the 15 // distribution. 16 // * Neither the name of Google Inc. nor the names of its 17 // contributors may be used to endorse or promote products derived from 18 // this software without specific prior written permission. 19 // 20 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 #endregion 32 33 using System; 34 using System.Buffers; 35 using System.IO; 36 using Google.Protobuf.TestProtos; 37 using Proto2 = Google.Protobuf.TestProtos.Proto2; 38 using NUnit.Framework; 39 40 namespace Google.Protobuf 41 { 42 public class CodedInputStreamTest 43 { 44 /// <summary> 45 /// Helper to construct a byte array from a bunch of bytes. The inputs are 46 /// actually ints so that I can use hex notation and not get stupid errors 47 /// about precision. 48 /// </summary> Bytes(params int[] bytesAsInts)49 private static byte[] Bytes(params int[] bytesAsInts) 50 { 51 byte[] bytes = new byte[bytesAsInts.Length]; 52 for (int i = 0; i < bytesAsInts.Length; i++) 53 { 54 bytes[i] = (byte) bytesAsInts[i]; 55 } 56 return bytes; 57 } 58 59 /// <summary> 60 /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() 61 /// </summary> AssertReadVarint(byte[] data, ulong value)62 private static void AssertReadVarint(byte[] data, ulong value) 63 { 64 CodedInputStream input = new CodedInputStream(data); 65 Assert.AreEqual((uint) value, input.ReadRawVarint32()); 66 Assert.IsTrue(input.IsAtEnd); 67 68 input = new CodedInputStream(data); 69 Assert.AreEqual(value, input.ReadRawVarint64()); 70 Assert.IsTrue(input.IsAtEnd); 71 72 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 73 { 74 Assert.AreEqual((uint) value, ctx.ReadUInt32()); 75 }, true); 76 77 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 78 { 79 Assert.AreEqual(value, ctx.ReadUInt64()); 80 }, true); 81 82 // Try different block sizes. 83 for (int bufferSize = 1; bufferSize <= 16; bufferSize *= 2) 84 { 85 input = new CodedInputStream(new SmallBlockInputStream(data, bufferSize)); 86 Assert.AreEqual((uint) value, input.ReadRawVarint32()); 87 88 input = new CodedInputStream(new SmallBlockInputStream(data, bufferSize)); 89 Assert.AreEqual(value, input.ReadRawVarint64()); 90 Assert.IsTrue(input.IsAtEnd); 91 92 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, bufferSize), (ref ParseContext ctx) => 93 { 94 Assert.AreEqual((uint) value, ctx.ReadUInt32()); 95 }, true); 96 97 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, bufferSize), (ref ParseContext ctx) => 98 { 99 Assert.AreEqual(value, ctx.ReadUInt64()); 100 }, true); 101 } 102 103 // Try reading directly from a MemoryStream. We want to verify that it 104 // doesn't read past the end of the input, so write an extra byte - this 105 // lets us test the position at the end. 106 MemoryStream memoryStream = new MemoryStream(); 107 memoryStream.Write(data, 0, data.Length); 108 memoryStream.WriteByte(0); 109 memoryStream.Position = 0; 110 Assert.AreEqual((uint) value, CodedInputStream.ReadRawVarint32(memoryStream)); 111 Assert.AreEqual(data.Length, memoryStream.Position); 112 } 113 114 /// <summary> 115 /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and 116 /// expects them to fail with an InvalidProtocolBufferException whose 117 /// description matches the given one. 118 /// </summary> AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data)119 private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) 120 { 121 CodedInputStream input = new CodedInputStream(data); 122 var exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint32()); 123 Assert.AreEqual(expected.Message, exception.Message); 124 125 input = new CodedInputStream(data); 126 exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint64()); 127 Assert.AreEqual(expected.Message, exception.Message); 128 129 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 130 { 131 try 132 { 133 ctx.ReadUInt32(); 134 Assert.Fail(); 135 } 136 catch (InvalidProtocolBufferException ex) 137 { 138 Assert.AreEqual(expected.Message, ex.Message); 139 } 140 }, false); 141 142 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 143 { 144 try 145 { 146 ctx.ReadUInt64(); 147 Assert.Fail(); 148 } 149 catch (InvalidProtocolBufferException ex) 150 { 151 Assert.AreEqual(expected.Message, ex.Message); 152 } 153 }, false); 154 155 // Make sure we get the same error when reading directly from a Stream. 156 exception = Assert.Throws<InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data))); 157 Assert.AreEqual(expected.Message, exception.Message); 158 } 159 ParseContextAssertAction(ref ParseContext ctx)160 private delegate void ParseContextAssertAction(ref ParseContext ctx); 161 AssertReadFromParseContext(ReadOnlySequence<byte> input, ParseContextAssertAction assertAction, bool assertIsAtEnd)162 private static void AssertReadFromParseContext(ReadOnlySequence<byte> input, ParseContextAssertAction assertAction, bool assertIsAtEnd) 163 { 164 ParseContext.Initialize(input, out ParseContext parseCtx); 165 assertAction(ref parseCtx); 166 if (assertIsAtEnd) 167 { 168 Assert.IsTrue(SegmentedBufferHelper.IsAtEnd(ref parseCtx.buffer, ref parseCtx.state)); 169 } 170 } 171 172 [Test] ReadVarint()173 public void ReadVarint() 174 { 175 AssertReadVarint(Bytes(0x00), 0); 176 AssertReadVarint(Bytes(0x01), 1); 177 AssertReadVarint(Bytes(0x7f), 127); 178 // 14882 179 AssertReadVarint(Bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7)); 180 // 2961488830 181 AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b), 182 (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | 183 (0x0bL << 28)); 184 185 // 64-bit 186 // 7256456126 187 AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b), 188 (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | 189 (0x1bL << 28)); 190 // 41256202580718336 191 AssertReadVarint(Bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49), 192 (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) | 193 (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49)); 194 // 11964378330978735131 195 AssertReadVarint(Bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01), 196 (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) | 197 (0x3bUL << 28) | (0x56UL << 35) | (0x00UL << 42) | 198 (0x05UL << 49) | (0x26UL << 56) | (0x01UL << 63)); 199 200 // Failures 201 AssertReadVarintFailure( 202 InvalidProtocolBufferException.MalformedVarint(), 203 Bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 204 0x00)); 205 AssertReadVarintFailure( 206 InvalidProtocolBufferException.TruncatedMessage(), 207 Bytes(0x80)); 208 } 209 210 /// <summary> 211 /// Parses the given bytes using ReadRawLittleEndian32() and checks 212 /// that the result matches the given value. 213 /// </summary> AssertReadLittleEndian32(byte[] data, uint value)214 private static void AssertReadLittleEndian32(byte[] data, uint value) 215 { 216 CodedInputStream input = new CodedInputStream(data); 217 Assert.AreEqual(value, input.ReadRawLittleEndian32()); 218 Assert.IsTrue(input.IsAtEnd); 219 220 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 221 { 222 Assert.AreEqual(value, ctx.ReadFixed32()); 223 }, true); 224 225 // Try different block sizes. 226 for (int blockSize = 1; blockSize <= 16; blockSize *= 2) 227 { 228 input = new CodedInputStream( 229 new SmallBlockInputStream(data, blockSize)); 230 Assert.AreEqual(value, input.ReadRawLittleEndian32()); 231 Assert.IsTrue(input.IsAtEnd); 232 233 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, blockSize), (ref ParseContext ctx) => 234 { 235 Assert.AreEqual(value, ctx.ReadFixed32()); 236 }, true); 237 } 238 } 239 240 /// <summary> 241 /// Parses the given bytes using ReadRawLittleEndian64() and checks 242 /// that the result matches the given value. 243 /// </summary> AssertReadLittleEndian64(byte[] data, ulong value)244 private static void AssertReadLittleEndian64(byte[] data, ulong value) 245 { 246 CodedInputStream input = new CodedInputStream(data); 247 Assert.AreEqual(value, input.ReadRawLittleEndian64()); 248 Assert.IsTrue(input.IsAtEnd); 249 250 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 251 { 252 Assert.AreEqual(value, ctx.ReadFixed64()); 253 }, true); 254 255 // Try different block sizes. 256 for (int blockSize = 1; blockSize <= 16; blockSize *= 2) 257 { 258 input = new CodedInputStream( 259 new SmallBlockInputStream(data, blockSize)); 260 Assert.AreEqual(value, input.ReadRawLittleEndian64()); 261 Assert.IsTrue(input.IsAtEnd); 262 263 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, blockSize), (ref ParseContext ctx) => 264 { 265 Assert.AreEqual(value, ctx.ReadFixed64()); 266 }, true); 267 } 268 } 269 270 [Test] ReadLittleEndian()271 public void ReadLittleEndian() 272 { 273 AssertReadLittleEndian32(Bytes(0x78, 0x56, 0x34, 0x12), 0x12345678); 274 AssertReadLittleEndian32(Bytes(0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef0); 275 276 AssertReadLittleEndian64(Bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12), 277 0x123456789abcdef0L); 278 AssertReadLittleEndian64( 279 Bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef012345678UL); 280 } 281 282 [Test] DecodeZigZag32()283 public void DecodeZigZag32() 284 { 285 Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag32(0)); 286 Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag32(1)); 287 Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag32(2)); 288 Assert.AreEqual(-2, ParsingPrimitives.DecodeZigZag32(3)); 289 Assert.AreEqual(0x3FFFFFFF, ParsingPrimitives.DecodeZigZag32(0x7FFFFFFE)); 290 Assert.AreEqual(unchecked((int) 0xC0000000), ParsingPrimitives.DecodeZigZag32(0x7FFFFFFF)); 291 Assert.AreEqual(0x7FFFFFFF, ParsingPrimitives.DecodeZigZag32(0xFFFFFFFE)); 292 Assert.AreEqual(unchecked((int) 0x80000000), ParsingPrimitives.DecodeZigZag32(0xFFFFFFFF)); 293 } 294 295 [Test] DecodeZigZag64()296 public void DecodeZigZag64() 297 { 298 Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag64(0)); 299 Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag64(1)); 300 Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag64(2)); 301 Assert.AreEqual(-2, ParsingPrimitives.DecodeZigZag64(3)); 302 Assert.AreEqual(0x000000003FFFFFFFL, ParsingPrimitives.DecodeZigZag64(0x000000007FFFFFFEL)); 303 Assert.AreEqual(unchecked((long) 0xFFFFFFFFC0000000L), ParsingPrimitives.DecodeZigZag64(0x000000007FFFFFFFL)); 304 Assert.AreEqual(0x000000007FFFFFFFL, ParsingPrimitives.DecodeZigZag64(0x00000000FFFFFFFEL)); 305 Assert.AreEqual(unchecked((long) 0xFFFFFFFF80000000L), ParsingPrimitives.DecodeZigZag64(0x00000000FFFFFFFFL)); 306 Assert.AreEqual(0x7FFFFFFFFFFFFFFFL, ParsingPrimitives.DecodeZigZag64(0xFFFFFFFFFFFFFFFEL)); 307 Assert.AreEqual(unchecked((long) 0x8000000000000000L), ParsingPrimitives.DecodeZigZag64(0xFFFFFFFFFFFFFFFFL)); 308 } 309 310 [Test] ReadWholeMessage_VaryingBlockSizes()311 public void ReadWholeMessage_VaryingBlockSizes() 312 { 313 TestAllTypes message = SampleMessages.CreateFullTestAllTypes(); 314 315 byte[] rawBytes = message.ToByteArray(); 316 Assert.AreEqual(rawBytes.Length, message.CalculateSize()); 317 TestAllTypes message2 = TestAllTypes.Parser.ParseFrom(rawBytes); 318 Assert.AreEqual(message, message2); 319 320 // Try different block sizes. 321 for (int blockSize = 1; blockSize < 256; blockSize *= 2) 322 { 323 message2 = TestAllTypes.Parser.ParseFrom(new SmallBlockInputStream(rawBytes, blockSize)); 324 Assert.AreEqual(message, message2); 325 } 326 } 327 328 [Test] ReadWholeMessage_VaryingBlockSizes_FromSequence()329 public void ReadWholeMessage_VaryingBlockSizes_FromSequence() 330 { 331 TestAllTypes message = SampleMessages.CreateFullTestAllTypes(); 332 333 byte[] rawBytes = message.ToByteArray(); 334 Assert.AreEqual(rawBytes.Length, message.CalculateSize()); 335 TestAllTypes message2 = TestAllTypes.Parser.ParseFrom(rawBytes); 336 Assert.AreEqual(message, message2); 337 338 // Try different block sizes. 339 for (int blockSize = 1; blockSize < 256; blockSize *= 2) 340 { 341 message2 = TestAllTypes.Parser.ParseFrom(ReadOnlySequenceFactory.CreateWithContent(rawBytes, blockSize)); 342 Assert.AreEqual(message, message2); 343 } 344 } 345 346 [Test] ReadInt32Wrapper_VariableBlockSizes()347 public void ReadInt32Wrapper_VariableBlockSizes() 348 { 349 byte[] rawBytes = new byte[] { 202, 1, 11, 8, 254, 255, 255, 255, 255, 255, 255, 255, 255, 1 }; 350 351 for (int blockSize = 1; blockSize <= rawBytes.Length; blockSize++) 352 { 353 ReadOnlySequence<byte> data = ReadOnlySequenceFactory.CreateWithContent(rawBytes, blockSize); 354 AssertReadFromParseContext(data, (ref ParseContext ctx) => 355 { 356 ctx.ReadTag(); 357 358 var value = ParsingPrimitivesWrappers.ReadInt32Wrapper(ref ctx); 359 360 Assert.AreEqual(-2, value); 361 }, true); 362 } 363 } 364 365 [Test] ReadHugeBlob()366 public void ReadHugeBlob() 367 { 368 // Allocate and initialize a 1MB blob. 369 byte[] blob = new byte[1 << 20]; 370 for (int i = 0; i < blob.Length; i++) 371 { 372 blob[i] = (byte) i; 373 } 374 375 // Make a message containing it. 376 var message = new TestAllTypes { SingleBytes = ByteString.CopyFrom(blob) }; 377 378 // Serialize and parse it. Make sure to parse from an InputStream, not 379 // directly from a ByteString, so that CodedInputStream uses buffered 380 // reading. 381 TestAllTypes message2 = TestAllTypes.Parser.ParseFrom(message.ToByteString()); 382 383 Assert.AreEqual(message, message2); 384 } 385 386 [Test] ReadMaliciouslyLargeBlob()387 public void ReadMaliciouslyLargeBlob() 388 { 389 MemoryStream ms = new MemoryStream(); 390 CodedOutputStream output = new CodedOutputStream(ms); 391 392 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 393 output.WriteRawVarint32(tag); 394 output.WriteRawVarint32(0x7FFFFFFF); 395 output.WriteRawBytes(new byte[32]); // Pad with a few random bytes. 396 output.Flush(); 397 ms.Position = 0; 398 399 CodedInputStream input = new CodedInputStream(ms); 400 Assert.AreEqual(tag, input.ReadTag()); 401 402 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes()); 403 } 404 405 [Test] ReadBlobGreaterThanCurrentLimit()406 public void ReadBlobGreaterThanCurrentLimit() 407 { 408 MemoryStream ms = new MemoryStream(); 409 CodedOutputStream output = new CodedOutputStream(ms); 410 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 411 output.WriteRawVarint32(tag); 412 output.WriteRawVarint32(4); 413 output.WriteRawBytes(new byte[4]); // Pad with a few random bytes. 414 output.Flush(); 415 ms.Position = 0; 416 417 CodedInputStream input = new CodedInputStream(ms); 418 Assert.AreEqual(tag, input.ReadTag()); 419 420 // Specify limit smaller than data length 421 input.PushLimit(3); 422 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes()); 423 424 AssertReadFromParseContext(new ReadOnlySequence<byte>(ms.ToArray()), (ref ParseContext ctx) => 425 { 426 Assert.AreEqual(tag, ctx.ReadTag()); 427 SegmentedBufferHelper.PushLimit(ref ctx.state, 3); 428 try 429 { 430 ctx.ReadBytes(); 431 Assert.Fail(); 432 } 433 catch (InvalidProtocolBufferException) {} 434 }, true); 435 } 436 437 [Test] ReadStringGreaterThanCurrentLimit()438 public void ReadStringGreaterThanCurrentLimit() 439 { 440 MemoryStream ms = new MemoryStream(); 441 CodedOutputStream output = new CodedOutputStream(ms); 442 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 443 output.WriteRawVarint32(tag); 444 output.WriteRawVarint32(4); 445 output.WriteRawBytes(new byte[4]); // Pad with a few random bytes. 446 output.Flush(); 447 ms.Position = 0; 448 449 CodedInputStream input = new CodedInputStream(ms.ToArray()); 450 Assert.AreEqual(tag, input.ReadTag()); 451 452 // Specify limit smaller than data length 453 input.PushLimit(3); 454 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadString()); 455 456 AssertReadFromParseContext(new ReadOnlySequence<byte>(ms.ToArray()), (ref ParseContext ctx) => 457 { 458 Assert.AreEqual(tag, ctx.ReadTag()); 459 SegmentedBufferHelper.PushLimit(ref ctx.state, 3); 460 try 461 { 462 ctx.ReadString(); 463 Assert.Fail(); 464 } 465 catch (InvalidProtocolBufferException) { } 466 }, true); 467 } 468 469 // Representations of a tag for field 0 with various wire types 470 [Test] 471 [TestCase(0)] 472 [TestCase(1)] 473 [TestCase(2)] 474 [TestCase(3)] 475 [TestCase(4)] 476 [TestCase(5)] ReadTag_ZeroFieldRejected(byte tag)477 public void ReadTag_ZeroFieldRejected(byte tag) 478 { 479 CodedInputStream cis = new CodedInputStream(new byte[] { tag }); 480 Assert.Throws<InvalidProtocolBufferException>(() => cis.ReadTag()); 481 } 482 MakeRecursiveMessage(int depth)483 internal static TestRecursiveMessage MakeRecursiveMessage(int depth) 484 { 485 if (depth == 0) 486 { 487 return new TestRecursiveMessage { I = 5 }; 488 } 489 else 490 { 491 return new TestRecursiveMessage { A = MakeRecursiveMessage(depth - 1) }; 492 } 493 } 494 AssertMessageDepth(TestRecursiveMessage message, int depth)495 internal static void AssertMessageDepth(TestRecursiveMessage message, int depth) 496 { 497 if (depth == 0) 498 { 499 Assert.IsNull(message.A); 500 Assert.AreEqual(5, message.I); 501 } 502 else 503 { 504 Assert.IsNotNull(message.A); 505 AssertMessageDepth(message.A, depth - 1); 506 } 507 } 508 509 [Test] MaliciousRecursion()510 public void MaliciousRecursion() 511 { 512 ByteString atRecursiveLimit = MakeRecursiveMessage(CodedInputStream.DefaultRecursionLimit).ToByteString(); 513 ByteString beyondRecursiveLimit = MakeRecursiveMessage(CodedInputStream.DefaultRecursionLimit + 1).ToByteString(); 514 515 AssertMessageDepth(TestRecursiveMessage.Parser.ParseFrom(atRecursiveLimit), CodedInputStream.DefaultRecursionLimit); 516 517 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(beyondRecursiveLimit)); 518 519 CodedInputStream input = CodedInputStream.CreateWithLimits(new MemoryStream(atRecursiveLimit.ToByteArray()), 1000000, CodedInputStream.DefaultRecursionLimit - 1); 520 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(input)); 521 } 522 MakeMaliciousRecursionUnknownFieldsPayload(int recursionDepth)523 private static byte[] MakeMaliciousRecursionUnknownFieldsPayload(int recursionDepth) 524 { 525 // generate recursively nested groups that will be parsed as unknown fields 526 int unknownFieldNumber = 14; // an unused field number 527 MemoryStream ms = new MemoryStream(); 528 CodedOutputStream output = new CodedOutputStream(ms); 529 for (int i = 0; i < recursionDepth; i++) 530 { 531 output.WriteTag(WireFormat.MakeTag(unknownFieldNumber, WireFormat.WireType.StartGroup)); 532 } 533 for (int i = 0; i < recursionDepth; i++) 534 { 535 output.WriteTag(WireFormat.MakeTag(unknownFieldNumber, WireFormat.WireType.EndGroup)); 536 } 537 output.Flush(); 538 return ms.ToArray(); 539 } 540 541 [Test] MaliciousRecursion_UnknownFields()542 public void MaliciousRecursion_UnknownFields() 543 { 544 byte[] payloadAtRecursiveLimit = MakeMaliciousRecursionUnknownFieldsPayload(CodedInputStream.DefaultRecursionLimit); 545 byte[] payloadBeyondRecursiveLimit = MakeMaliciousRecursionUnknownFieldsPayload(CodedInputStream.DefaultRecursionLimit + 1); 546 547 Assert.DoesNotThrow(() => TestRecursiveMessage.Parser.ParseFrom(payloadAtRecursiveLimit)); 548 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(payloadBeyondRecursiveLimit)); 549 } 550 551 [Test] ReadGroup_WrongEndGroupTag()552 public void ReadGroup_WrongEndGroupTag() 553 { 554 int groupFieldNumber = Proto2.TestAllTypes.OptionalGroupFieldNumber; 555 556 // write Proto2.TestAllTypes with "optional_group" set, but use wrong EndGroup closing tag 557 MemoryStream ms = new MemoryStream(); 558 CodedOutputStream output = new CodedOutputStream(ms); 559 output.WriteTag(WireFormat.MakeTag(groupFieldNumber, WireFormat.WireType.StartGroup)); 560 output.WriteGroup(new Proto2.TestAllTypes.Types.OptionalGroup { A = 12345 }); 561 // end group with different field number 562 output.WriteTag(WireFormat.MakeTag(groupFieldNumber + 1, WireFormat.WireType.EndGroup)); 563 output.Flush(); 564 var payload = ms.ToArray(); 565 566 Assert.Throws<InvalidProtocolBufferException>(() => Proto2.TestAllTypes.Parser.ParseFrom(payload)); 567 } 568 569 [Test] ReadGroup_UnknownFields_WrongEndGroupTag()570 public void ReadGroup_UnknownFields_WrongEndGroupTag() 571 { 572 MemoryStream ms = new MemoryStream(); 573 CodedOutputStream output = new CodedOutputStream(ms); 574 output.WriteTag(WireFormat.MakeTag(14, WireFormat.WireType.StartGroup)); 575 // end group with different field number 576 output.WriteTag(WireFormat.MakeTag(15, WireFormat.WireType.EndGroup)); 577 output.Flush(); 578 var payload = ms.ToArray(); 579 580 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(payload)); 581 } 582 583 [Test] SizeLimit()584 public void SizeLimit() 585 { 586 // Have to use a Stream rather than ByteString.CreateCodedInput as SizeLimit doesn't 587 // apply to the latter case. 588 MemoryStream ms = new MemoryStream(SampleMessages.CreateFullTestAllTypes().ToByteArray()); 589 CodedInputStream input = CodedInputStream.CreateWithLimits(ms, 16, 100); 590 Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseFrom(input)); 591 } 592 593 /// <summary> 594 /// Tests that if we read an string that contains invalid UTF-8, no exception 595 /// is thrown. Instead, the invalid bytes are replaced with the Unicode 596 /// "replacement character" U+FFFD. 597 /// </summary> 598 [Test] ReadInvalidUtf8()599 public void ReadInvalidUtf8() 600 { 601 MemoryStream ms = new MemoryStream(); 602 CodedOutputStream output = new CodedOutputStream(ms); 603 604 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 605 output.WriteRawVarint32(tag); 606 output.WriteRawVarint32(1); 607 output.WriteRawBytes(new byte[] {0x80}); 608 output.Flush(); 609 ms.Position = 0; 610 611 CodedInputStream input = new CodedInputStream(ms); 612 613 Assert.AreEqual(tag, input.ReadTag()); 614 string text = input.ReadString(); 615 Assert.AreEqual('\ufffd', text[0]); 616 } 617 618 [Test] ReadNegativeSizedStringThrowsInvalidProtocolBufferException()619 public void ReadNegativeSizedStringThrowsInvalidProtocolBufferException() 620 { 621 MemoryStream ms = new MemoryStream(); 622 CodedOutputStream output = new CodedOutputStream(ms); 623 624 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 625 output.WriteRawVarint32(tag); 626 output.WriteLength(-1); 627 output.Flush(); 628 ms.Position = 0; 629 630 CodedInputStream input = new CodedInputStream(ms); 631 632 Assert.AreEqual(tag, input.ReadTag()); 633 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadString()); 634 } 635 636 [Test] ReadNegativeSizedBytesThrowsInvalidProtocolBufferException()637 public void ReadNegativeSizedBytesThrowsInvalidProtocolBufferException() 638 { 639 MemoryStream ms = new MemoryStream(); 640 CodedOutputStream output = new CodedOutputStream(ms); 641 642 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 643 output.WriteRawVarint32(tag); 644 output.WriteLength(-1); 645 output.Flush(); 646 ms.Position = 0; 647 648 CodedInputStream input = new CodedInputStream(ms); 649 650 Assert.AreEqual(tag, input.ReadTag()); 651 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes()); 652 } 653 654 /// <summary> 655 /// A stream which limits the number of bytes it reads at a time. 656 /// We use this to make sure that CodedInputStream doesn't screw up when 657 /// reading in small blocks. 658 /// </summary> 659 private sealed class SmallBlockInputStream : MemoryStream 660 { 661 private readonly int blockSize; 662 SmallBlockInputStream(byte[] data, int blockSize)663 public SmallBlockInputStream(byte[] data, int blockSize) 664 : base(data) 665 { 666 this.blockSize = blockSize; 667 } 668 Read(byte[] buffer, int offset, int count)669 public override int Read(byte[] buffer, int offset, int count) 670 { 671 return base.Read(buffer, offset, Math.Min(count, blockSize)); 672 } 673 } 674 675 [Test] TestNegativeEnum()676 public void TestNegativeEnum() 677 { 678 byte[] bytes = { 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01 }; 679 CodedInputStream input = new CodedInputStream(bytes); 680 Assert.AreEqual((int)SampleEnum.NegativeValue, input.ReadEnum()); 681 Assert.IsTrue(input.IsAtEnd); 682 } 683 684 //Issue 71: CodedInputStream.ReadBytes go to slow path unnecessarily 685 [Test] TestSlowPathAvoidance()686 public void TestSlowPathAvoidance() 687 { 688 using (var ms = new MemoryStream()) 689 { 690 CodedOutputStream output = new CodedOutputStream(ms); 691 output.WriteTag(1, WireFormat.WireType.LengthDelimited); 692 output.WriteBytes(ByteString.CopyFrom(new byte[100])); 693 output.WriteTag(2, WireFormat.WireType.LengthDelimited); 694 output.WriteBytes(ByteString.CopyFrom(new byte[100])); 695 output.Flush(); 696 697 ms.Position = 0; 698 CodedInputStream input = new CodedInputStream(ms, new byte[ms.Length / 2], 0, 0, false); 699 700 uint tag = input.ReadTag(); 701 Assert.AreEqual(1, WireFormat.GetTagFieldNumber(tag)); 702 Assert.AreEqual(100, input.ReadBytes().Length); 703 704 tag = input.ReadTag(); 705 Assert.AreEqual(2, WireFormat.GetTagFieldNumber(tag)); 706 Assert.AreEqual(100, input.ReadBytes().Length); 707 } 708 } 709 710 [Test] Tag0Throws()711 public void Tag0Throws() 712 { 713 var input = new CodedInputStream(new byte[] { 0 }); 714 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadTag()); 715 } 716 717 [Test] SkipGroup()718 public void SkipGroup() 719 { 720 // Create an output stream with a group in: 721 // Field 1: string "field 1" 722 // Field 2: group containing: 723 // Field 1: fixed int32 value 100 724 // Field 2: string "ignore me" 725 // Field 3: nested group containing 726 // Field 1: fixed int64 value 1000 727 // Field 3: string "field 3" 728 var stream = new MemoryStream(); 729 var output = new CodedOutputStream(stream); 730 output.WriteTag(1, WireFormat.WireType.LengthDelimited); 731 output.WriteString("field 1"); 732 733 // The outer group... 734 output.WriteTag(2, WireFormat.WireType.StartGroup); 735 output.WriteTag(1, WireFormat.WireType.Fixed32); 736 output.WriteFixed32(100); 737 output.WriteTag(2, WireFormat.WireType.LengthDelimited); 738 output.WriteString("ignore me"); 739 // The nested group... 740 output.WriteTag(3, WireFormat.WireType.StartGroup); 741 output.WriteTag(1, WireFormat.WireType.Fixed64); 742 output.WriteFixed64(1000); 743 // Note: Not sure the field number is relevant for end group... 744 output.WriteTag(3, WireFormat.WireType.EndGroup); 745 746 // End the outer group 747 output.WriteTag(2, WireFormat.WireType.EndGroup); 748 749 output.WriteTag(3, WireFormat.WireType.LengthDelimited); 750 output.WriteString("field 3"); 751 output.Flush(); 752 stream.Position = 0; 753 754 // Now act like a generated client 755 var input = new CodedInputStream(stream); 756 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited), input.ReadTag()); 757 Assert.AreEqual("field 1", input.ReadString()); 758 Assert.AreEqual(WireFormat.MakeTag(2, WireFormat.WireType.StartGroup), input.ReadTag()); 759 input.SkipLastField(); // Should consume the whole group, including the nested one. 760 Assert.AreEqual(WireFormat.MakeTag(3, WireFormat.WireType.LengthDelimited), input.ReadTag()); 761 Assert.AreEqual("field 3", input.ReadString()); 762 } 763 764 [Test] SkipGroup_WrongEndGroupTag()765 public void SkipGroup_WrongEndGroupTag() 766 { 767 // Create an output stream with: 768 // Field 1: string "field 1" 769 // Start group 2 770 // Field 3: fixed int32 771 // End group 4 (should give an error) 772 var stream = new MemoryStream(); 773 var output = new CodedOutputStream(stream); 774 output.WriteTag(1, WireFormat.WireType.LengthDelimited); 775 output.WriteString("field 1"); 776 777 // The outer group... 778 output.WriteTag(2, WireFormat.WireType.StartGroup); 779 output.WriteTag(3, WireFormat.WireType.Fixed32); 780 output.WriteFixed32(100); 781 output.WriteTag(4, WireFormat.WireType.EndGroup); 782 output.Flush(); 783 stream.Position = 0; 784 785 // Now act like a generated client 786 var input = new CodedInputStream(stream); 787 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited), input.ReadTag()); 788 Assert.AreEqual("field 1", input.ReadString()); 789 Assert.AreEqual(WireFormat.MakeTag(2, WireFormat.WireType.StartGroup), input.ReadTag()); 790 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 791 } 792 793 [Test] RogueEndGroupTag()794 public void RogueEndGroupTag() 795 { 796 // If we have an end-group tag without a leading start-group tag, generated 797 // code will just call SkipLastField... so that should fail. 798 799 var stream = new MemoryStream(); 800 var output = new CodedOutputStream(stream); 801 output.WriteTag(1, WireFormat.WireType.EndGroup); 802 output.Flush(); 803 stream.Position = 0; 804 805 var input = new CodedInputStream(stream); 806 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.EndGroup), input.ReadTag()); 807 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 808 } 809 810 [Test] EndOfStreamReachedWhileSkippingGroup()811 public void EndOfStreamReachedWhileSkippingGroup() 812 { 813 var stream = new MemoryStream(); 814 var output = new CodedOutputStream(stream); 815 output.WriteTag(1, WireFormat.WireType.StartGroup); 816 output.WriteTag(2, WireFormat.WireType.StartGroup); 817 output.WriteTag(2, WireFormat.WireType.EndGroup); 818 819 output.Flush(); 820 stream.Position = 0; 821 822 // Now act like a generated client 823 var input = new CodedInputStream(stream); 824 input.ReadTag(); 825 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 826 } 827 828 [Test] RecursionLimitAppliedWhileSkippingGroup()829 public void RecursionLimitAppliedWhileSkippingGroup() 830 { 831 var stream = new MemoryStream(); 832 var output = new CodedOutputStream(stream); 833 for (int i = 0; i < CodedInputStream.DefaultRecursionLimit + 1; i++) 834 { 835 output.WriteTag(1, WireFormat.WireType.StartGroup); 836 } 837 for (int i = 0; i < CodedInputStream.DefaultRecursionLimit + 1; i++) 838 { 839 output.WriteTag(1, WireFormat.WireType.EndGroup); 840 } 841 output.Flush(); 842 stream.Position = 0; 843 844 // Now act like a generated client 845 var input = new CodedInputStream(stream); 846 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.StartGroup), input.ReadTag()); 847 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 848 } 849 850 [Test] Construction_Invalid()851 public void Construction_Invalid() 852 { 853 Assert.Throws<ArgumentNullException>(() => new CodedInputStream((byte[]) null)); 854 Assert.Throws<ArgumentNullException>(() => new CodedInputStream(null, 0, 0)); 855 Assert.Throws<ArgumentNullException>(() => new CodedInputStream((Stream) null)); 856 Assert.Throws<ArgumentOutOfRangeException>(() => new CodedInputStream(new byte[10], 100, 0)); 857 Assert.Throws<ArgumentOutOfRangeException>(() => new CodedInputStream(new byte[10], 5, 10)); 858 } 859 860 [Test] CreateWithLimits_InvalidLimits()861 public void CreateWithLimits_InvalidLimits() 862 { 863 var stream = new MemoryStream(); 864 Assert.Throws<ArgumentOutOfRangeException>(() => CodedInputStream.CreateWithLimits(stream, 0, 1)); 865 Assert.Throws<ArgumentOutOfRangeException>(() => CodedInputStream.CreateWithLimits(stream, 1, 0)); 866 } 867 868 [Test] Dispose_DisposesUnderlyingStream()869 public void Dispose_DisposesUnderlyingStream() 870 { 871 var memoryStream = new MemoryStream(); 872 Assert.IsTrue(memoryStream.CanRead); 873 using (var cis = new CodedInputStream(memoryStream)) 874 { 875 } 876 Assert.IsFalse(memoryStream.CanRead); // Disposed 877 } 878 879 [Test] Dispose_WithLeaveOpen()880 public void Dispose_WithLeaveOpen() 881 { 882 var memoryStream = new MemoryStream(); 883 Assert.IsTrue(memoryStream.CanRead); 884 using (var cis = new CodedInputStream(memoryStream, true)) 885 { 886 } 887 Assert.IsTrue(memoryStream.CanRead); // We left the stream open 888 } 889 890 [Test] Dispose_FromByteArray()891 public void Dispose_FromByteArray() 892 { 893 var stream = new CodedInputStream(new byte[10]); 894 stream.Dispose(); 895 } 896 897 [Test] TestParseMessagesCloseTo2G()898 public void TestParseMessagesCloseTo2G() 899 { 900 byte[] serializedMessage = GenerateBigSerializedMessage(); 901 // How many of these big messages do we need to take us near our 2GB limit? 902 int count = Int32.MaxValue / serializedMessage.Length; 903 // Now make a MemoryStream that will fake a near-2GB stream of messages by returning 904 // our big serialized message 'count' times. 905 using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count)) 906 { 907 Assert.DoesNotThrow(()=>TestAllTypes.Parser.ParseFrom(stream)); 908 } 909 } 910 911 [Test] TestParseMessagesOver2G()912 public void TestParseMessagesOver2G() 913 { 914 byte[] serializedMessage = GenerateBigSerializedMessage(); 915 // How many of these big messages do we need to take us near our 2GB limit? 916 int count = Int32.MaxValue / serializedMessage.Length; 917 // Now add one to take us over the 2GB limit 918 count++; 919 // Now make a MemoryStream that will fake a near-2GB stream of messages by returning 920 // our big serialized message 'count' times. 921 using (RepeatingMemoryStream stream = new RepeatingMemoryStream(serializedMessage, count)) 922 { 923 Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseFrom(stream), 924 "Protocol message was too large. May be malicious. " + 925 "Use CodedInputStream.SetSizeLimit() to increase the size limit."); 926 } 927 } 928 929 /// <returns>A serialized big message</returns> GenerateBigSerializedMessage()930 private static byte[] GenerateBigSerializedMessage() 931 { 932 byte[] value = new byte[16 * 1024 * 1024]; 933 TestAllTypes message = SampleMessages.CreateFullTestAllTypes(); 934 message.SingleBytes = ByteString.CopyFrom(value); 935 return message.ToByteArray(); 936 } 937 938 /// <summary> 939 /// A MemoryStream that repeats a byte arrays' content a number of times. 940 /// Simulates really large input without consuming loads of memory. Used above 941 /// to test the parsing behavior when the input size exceeds 2GB or close to it. 942 /// </summary> 943 private class RepeatingMemoryStream: MemoryStream 944 { 945 private readonly byte[] bytes; 946 private readonly int maxIterations; 947 private int index = 0; 948 RepeatingMemoryStream(byte[] bytes, int maxIterations)949 public RepeatingMemoryStream(byte[] bytes, int maxIterations) 950 { 951 this.bytes = bytes; 952 this.maxIterations = maxIterations; 953 } 954 Read(byte[] buffer, int offset, int count)955 public override int Read(byte[] buffer, int offset, int count) 956 { 957 if (bytes.Length == 0) 958 { 959 return 0; 960 } 961 int numBytesCopiedTotal = 0; 962 while (numBytesCopiedTotal < count && index < maxIterations) 963 { 964 int numBytesToCopy = Math.Min(bytes.Length - (int)Position, count); 965 Array.Copy(bytes, (int)Position, buffer, offset, numBytesToCopy); 966 numBytesCopiedTotal += numBytesToCopy; 967 offset += numBytesToCopy; 968 count -= numBytesCopiedTotal; 969 Position += numBytesToCopy; 970 if (Position >= bytes.Length) 971 { 972 Position = 0; 973 index++; 974 } 975 } 976 return numBytesCopiedTotal; 977 } 978 } 979 } 980 } 981