1 #region Copyright notice and license 2 // Protocol Buffers - Google's data interchange format 3 // Copyright 2008 Google Inc. All rights reserved. 4 // 5 // Use of this source code is governed by a BSD-style 6 // license that can be found in the LICENSE file or at 7 // https://developers.google.com/open-source/licenses/bsd 8 #endregion 9 10 using System; 11 using System.Buffers; 12 using System.IO; 13 using Google.Protobuf.TestProtos; 14 using Proto2 = Google.Protobuf.TestProtos.Proto2; 15 using NUnit.Framework; 16 17 namespace Google.Protobuf 18 { 19 public class CodedInputStreamTest 20 { 21 /// <summary> 22 /// Helper to construct a byte array from a bunch of bytes. The inputs are 23 /// actually ints so that I can use hex notation and not get stupid errors 24 /// about precision. 25 /// </summary> Bytes(params int[] bytesAsInts)26 private static byte[] Bytes(params int[] bytesAsInts) 27 { 28 byte[] bytes = new byte[bytesAsInts.Length]; 29 for (int i = 0; i < bytesAsInts.Length; i++) 30 { 31 bytes[i] = (byte) bytesAsInts[i]; 32 } 33 return bytes; 34 } 35 36 /// <summary> 37 /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() 38 /// </summary> AssertReadVarint(byte[] data, ulong value)39 private static void AssertReadVarint(byte[] data, ulong value) 40 { 41 CodedInputStream input = new CodedInputStream(data); 42 Assert.AreEqual((uint) value, input.ReadRawVarint32()); 43 Assert.IsTrue(input.IsAtEnd); 44 45 input = new CodedInputStream(data); 46 Assert.AreEqual(value, input.ReadRawVarint64()); 47 Assert.IsTrue(input.IsAtEnd); 48 49 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 50 { 51 Assert.AreEqual((uint) value, ctx.ReadUInt32()); 52 }, true); 53 54 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 55 { 56 Assert.AreEqual(value, ctx.ReadUInt64()); 57 }, true); 58 59 // Try different block sizes. 60 for (int bufferSize = 1; bufferSize <= 16; bufferSize *= 2) 61 { 62 input = new CodedInputStream(new SmallBlockInputStream(data, bufferSize)); 63 Assert.AreEqual((uint) value, input.ReadRawVarint32()); 64 65 input = new CodedInputStream(new SmallBlockInputStream(data, bufferSize)); 66 Assert.AreEqual(value, input.ReadRawVarint64()); 67 Assert.IsTrue(input.IsAtEnd); 68 69 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, bufferSize), (ref ParseContext ctx) => 70 { 71 Assert.AreEqual((uint) value, ctx.ReadUInt32()); 72 }, true); 73 74 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, bufferSize), (ref ParseContext ctx) => 75 { 76 Assert.AreEqual(value, ctx.ReadUInt64()); 77 }, true); 78 } 79 80 // Try reading directly from a MemoryStream. We want to verify that it 81 // doesn't read past the end of the input, so write an extra byte - this 82 // lets us test the position at the end. 83 MemoryStream memoryStream = new MemoryStream(); 84 memoryStream.Write(data, 0, data.Length); 85 memoryStream.WriteByte(0); 86 memoryStream.Position = 0; 87 Assert.AreEqual((uint) value, CodedInputStream.ReadRawVarint32(memoryStream)); 88 Assert.AreEqual(data.Length, memoryStream.Position); 89 } 90 91 /// <summary> 92 /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and 93 /// expects them to fail with an InvalidProtocolBufferException whose 94 /// description matches the given one. 95 /// </summary> AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data)96 private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) 97 { 98 CodedInputStream input = new CodedInputStream(data); 99 var exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint32()); 100 Assert.AreEqual(expected.Message, exception.Message); 101 102 input = new CodedInputStream(data); 103 exception = Assert.Throws<InvalidProtocolBufferException>(() => input.ReadRawVarint64()); 104 Assert.AreEqual(expected.Message, exception.Message); 105 106 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 107 { 108 try 109 { 110 ctx.ReadUInt32(); 111 Assert.Fail(); 112 } 113 catch (InvalidProtocolBufferException ex) 114 { 115 Assert.AreEqual(expected.Message, ex.Message); 116 } 117 }, false); 118 119 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 120 { 121 try 122 { 123 ctx.ReadUInt64(); 124 Assert.Fail(); 125 } 126 catch (InvalidProtocolBufferException ex) 127 { 128 Assert.AreEqual(expected.Message, ex.Message); 129 } 130 }, false); 131 132 // Make sure we get the same error when reading directly from a Stream. 133 exception = Assert.Throws<InvalidProtocolBufferException>(() => CodedInputStream.ReadRawVarint32(new MemoryStream(data))); 134 Assert.AreEqual(expected.Message, exception.Message); 135 } 136 ParseContextAssertAction(ref ParseContext ctx)137 private delegate void ParseContextAssertAction(ref ParseContext ctx); 138 AssertReadFromParseContext(ReadOnlySequence<byte> input, ParseContextAssertAction assertAction, bool assertIsAtEnd)139 private static void AssertReadFromParseContext(ReadOnlySequence<byte> input, ParseContextAssertAction assertAction, bool assertIsAtEnd) 140 { 141 // Check as ReadOnlySequence<byte> 142 ParseContext.Initialize(input, out ParseContext parseCtx); 143 assertAction(ref parseCtx); 144 if (assertIsAtEnd) 145 { 146 Assert.IsTrue(SegmentedBufferHelper.IsAtEnd(ref parseCtx.buffer, ref parseCtx.state)); 147 } 148 149 // Check as ReadOnlySpan<byte> 150 ParseContext.Initialize(input.ToArray().AsSpan(), out ParseContext spanParseContext); 151 assertAction(ref spanParseContext); 152 if (assertIsAtEnd) 153 { 154 Assert.IsTrue(SegmentedBufferHelper.IsAtEnd(ref spanParseContext.buffer, ref spanParseContext.state)); 155 } 156 } 157 158 [Test] ReadVarint()159 public void ReadVarint() 160 { 161 AssertReadVarint(Bytes(0x00), 0); 162 AssertReadVarint(Bytes(0x01), 1); 163 AssertReadVarint(Bytes(0x7f), 127); 164 // 14882 165 AssertReadVarint(Bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7)); 166 // 2961488830 167 AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b), 168 (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | 169 (0x0bL << 28)); 170 171 // 64-bit 172 // 7256456126 173 AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b), 174 (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | 175 (0x1bL << 28)); 176 // 41256202580718336 177 AssertReadVarint(Bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49), 178 (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) | 179 (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49)); 180 // 11964378330978735131 181 AssertReadVarint(Bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01), 182 (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) | 183 (0x3bUL << 28) | (0x56UL << 35) | (0x00UL << 42) | 184 (0x05UL << 49) | (0x26UL << 56) | (0x01UL << 63)); 185 186 // Failures 187 AssertReadVarintFailure( 188 InvalidProtocolBufferException.MalformedVarint(), 189 Bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 190 0x00)); 191 AssertReadVarintFailure( 192 InvalidProtocolBufferException.TruncatedMessage(), 193 Bytes(0x80)); 194 } 195 196 /// <summary> 197 /// Parses the given bytes using ReadRawLittleEndian32() and checks 198 /// that the result matches the given value. 199 /// </summary> AssertReadLittleEndian32(byte[] data, uint value)200 private static void AssertReadLittleEndian32(byte[] data, uint value) 201 { 202 CodedInputStream input = new CodedInputStream(data); 203 Assert.AreEqual(value, input.ReadRawLittleEndian32()); 204 Assert.IsTrue(input.IsAtEnd); 205 206 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 207 { 208 Assert.AreEqual(value, ctx.ReadFixed32()); 209 }, true); 210 211 // Try different block sizes. 212 for (int blockSize = 1; blockSize <= 16; blockSize *= 2) 213 { 214 input = new CodedInputStream( 215 new SmallBlockInputStream(data, blockSize)); 216 Assert.AreEqual(value, input.ReadRawLittleEndian32()); 217 Assert.IsTrue(input.IsAtEnd); 218 219 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, blockSize), (ref ParseContext ctx) => 220 { 221 Assert.AreEqual(value, ctx.ReadFixed32()); 222 }, true); 223 } 224 } 225 226 /// <summary> 227 /// Parses the given bytes using ReadRawLittleEndian64() and checks 228 /// that the result matches the given value. 229 /// </summary> AssertReadLittleEndian64(byte[] data, ulong value)230 private static void AssertReadLittleEndian64(byte[] data, ulong value) 231 { 232 CodedInputStream input = new CodedInputStream(data); 233 Assert.AreEqual(value, input.ReadRawLittleEndian64()); 234 Assert.IsTrue(input.IsAtEnd); 235 236 AssertReadFromParseContext(new ReadOnlySequence<byte>(data), (ref ParseContext ctx) => 237 { 238 Assert.AreEqual(value, ctx.ReadFixed64()); 239 }, true); 240 241 // Try different block sizes. 242 for (int blockSize = 1; blockSize <= 16; blockSize *= 2) 243 { 244 input = new CodedInputStream( 245 new SmallBlockInputStream(data, blockSize)); 246 Assert.AreEqual(value, input.ReadRawLittleEndian64()); 247 Assert.IsTrue(input.IsAtEnd); 248 249 AssertReadFromParseContext(ReadOnlySequenceFactory.CreateWithContent(data, blockSize), (ref ParseContext ctx) => 250 { 251 Assert.AreEqual(value, ctx.ReadFixed64()); 252 }, true); 253 } 254 } 255 256 [Test] ReadLittleEndian()257 public void ReadLittleEndian() 258 { 259 AssertReadLittleEndian32(Bytes(0x78, 0x56, 0x34, 0x12), 0x12345678); 260 AssertReadLittleEndian32(Bytes(0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef0); 261 262 AssertReadLittleEndian64(Bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12), 263 0x123456789abcdef0L); 264 AssertReadLittleEndian64( 265 Bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef012345678UL); 266 } 267 268 [Test] DecodeZigZag32()269 public void DecodeZigZag32() 270 { 271 Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag32(0)); 272 Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag32(1)); 273 Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag32(2)); 274 Assert.AreEqual(-2, ParsingPrimitives.DecodeZigZag32(3)); 275 Assert.AreEqual(0x3FFFFFFF, ParsingPrimitives.DecodeZigZag32(0x7FFFFFFE)); 276 Assert.AreEqual(unchecked((int) 0xC0000000), ParsingPrimitives.DecodeZigZag32(0x7FFFFFFF)); 277 Assert.AreEqual(0x7FFFFFFF, ParsingPrimitives.DecodeZigZag32(0xFFFFFFFE)); 278 Assert.AreEqual(unchecked((int) 0x80000000), ParsingPrimitives.DecodeZigZag32(0xFFFFFFFF)); 279 } 280 281 [Test] DecodeZigZag64()282 public void DecodeZigZag64() 283 { 284 Assert.AreEqual(0, ParsingPrimitives.DecodeZigZag64(0)); 285 Assert.AreEqual(-1, ParsingPrimitives.DecodeZigZag64(1)); 286 Assert.AreEqual(1, ParsingPrimitives.DecodeZigZag64(2)); 287 Assert.AreEqual(-2, ParsingPrimitives.DecodeZigZag64(3)); 288 Assert.AreEqual(0x000000003FFFFFFFL, ParsingPrimitives.DecodeZigZag64(0x000000007FFFFFFEL)); 289 Assert.AreEqual(unchecked((long) 0xFFFFFFFFC0000000L), ParsingPrimitives.DecodeZigZag64(0x000000007FFFFFFFL)); 290 Assert.AreEqual(0x000000007FFFFFFFL, ParsingPrimitives.DecodeZigZag64(0x00000000FFFFFFFEL)); 291 Assert.AreEqual(unchecked((long) 0xFFFFFFFF80000000L), ParsingPrimitives.DecodeZigZag64(0x00000000FFFFFFFFL)); 292 Assert.AreEqual(0x7FFFFFFFFFFFFFFFL, ParsingPrimitives.DecodeZigZag64(0xFFFFFFFFFFFFFFFEL)); 293 Assert.AreEqual(unchecked((long) 0x8000000000000000L), ParsingPrimitives.DecodeZigZag64(0xFFFFFFFFFFFFFFFFL)); 294 } 295 296 [Test] ReadWholeMessage_VaryingBlockSizes()297 public void ReadWholeMessage_VaryingBlockSizes() 298 { 299 TestAllTypes message = SampleMessages.CreateFullTestAllTypes(); 300 301 byte[] rawBytes = message.ToByteArray(); 302 Assert.AreEqual(rawBytes.Length, message.CalculateSize()); 303 TestAllTypes message2 = TestAllTypes.Parser.ParseFrom(rawBytes); 304 Assert.AreEqual(message, message2); 305 306 // Try different block sizes. 307 for (int blockSize = 1; blockSize < 256; blockSize *= 2) 308 { 309 message2 = TestAllTypes.Parser.ParseFrom(new SmallBlockInputStream(rawBytes, blockSize)); 310 Assert.AreEqual(message, message2); 311 } 312 } 313 314 [Test] ReadWholeMessage_VaryingBlockSizes_FromSequence()315 public void ReadWholeMessage_VaryingBlockSizes_FromSequence() 316 { 317 TestAllTypes message = SampleMessages.CreateFullTestAllTypes(); 318 319 byte[] rawBytes = message.ToByteArray(); 320 Assert.AreEqual(rawBytes.Length, message.CalculateSize()); 321 TestAllTypes message2 = TestAllTypes.Parser.ParseFrom(rawBytes); 322 Assert.AreEqual(message, message2); 323 324 // Try different block sizes. 325 for (int blockSize = 1; blockSize < 256; blockSize *= 2) 326 { 327 message2 = TestAllTypes.Parser.ParseFrom(ReadOnlySequenceFactory.CreateWithContent(rawBytes, blockSize)); 328 Assert.AreEqual(message, message2); 329 } 330 } 331 332 [Test] ReadInt32Wrapper_VariableBlockSizes()333 public void ReadInt32Wrapper_VariableBlockSizes() 334 { 335 byte[] rawBytes = new byte[] { 202, 1, 11, 8, 254, 255, 255, 255, 255, 255, 255, 255, 255, 1 }; 336 337 for (int blockSize = 1; blockSize <= rawBytes.Length; blockSize++) 338 { 339 ReadOnlySequence<byte> data = ReadOnlySequenceFactory.CreateWithContent(rawBytes, blockSize); 340 AssertReadFromParseContext(data, (ref ParseContext ctx) => 341 { 342 ctx.ReadTag(); 343 344 var value = ParsingPrimitivesWrappers.ReadInt32Wrapper(ref ctx); 345 346 Assert.AreEqual(-2, value); 347 }, true); 348 } 349 } 350 351 [Test] ReadHugeBlob()352 public void ReadHugeBlob() 353 { 354 // Allocate and initialize a 1MB blob. 355 byte[] blob = new byte[1 << 20]; 356 for (int i = 0; i < blob.Length; i++) 357 { 358 blob[i] = (byte) i; 359 } 360 361 // Make a message containing it. 362 var message = new TestAllTypes { SingleBytes = ByteString.CopyFrom(blob) }; 363 364 // Serialize and parse it. Make sure to parse from an InputStream, not 365 // directly from a ByteString, so that CodedInputStream uses buffered 366 // reading. 367 TestAllTypes message2 = TestAllTypes.Parser.ParseFrom(message.ToByteString()); 368 369 Assert.AreEqual(message, message2); 370 } 371 372 [Test] ReadMaliciouslyLargeBlob()373 public void ReadMaliciouslyLargeBlob() 374 { 375 MemoryStream ms = new MemoryStream(); 376 CodedOutputStream output = new CodedOutputStream(ms); 377 378 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 379 output.WriteRawVarint32(tag); 380 output.WriteRawVarint32(0x7FFFFFFF); 381 output.WriteRawBytes(new byte[32]); // Pad with a few random bytes. 382 output.Flush(); 383 ms.Position = 0; 384 385 CodedInputStream input = new CodedInputStream(ms); 386 Assert.AreEqual(tag, input.ReadTag()); 387 388 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes()); 389 } 390 391 [Test] ReadBlobGreaterThanCurrentLimit()392 public void ReadBlobGreaterThanCurrentLimit() 393 { 394 MemoryStream ms = new MemoryStream(); 395 CodedOutputStream output = new CodedOutputStream(ms); 396 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 397 output.WriteRawVarint32(tag); 398 output.WriteRawVarint32(4); 399 output.WriteRawBytes(new byte[4]); // Pad with a few random bytes. 400 output.Flush(); 401 ms.Position = 0; 402 403 CodedInputStream input = new CodedInputStream(ms); 404 Assert.AreEqual(tag, input.ReadTag()); 405 406 // Specify limit smaller than data length 407 input.PushLimit(3); 408 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes()); 409 410 AssertReadFromParseContext(new ReadOnlySequence<byte>(ms.ToArray()), (ref ParseContext ctx) => 411 { 412 Assert.AreEqual(tag, ctx.ReadTag()); 413 SegmentedBufferHelper.PushLimit(ref ctx.state, 3); 414 try 415 { 416 ctx.ReadBytes(); 417 Assert.Fail(); 418 } 419 catch (InvalidProtocolBufferException) {} 420 }, true); 421 } 422 423 [Test] ReadStringGreaterThanCurrentLimit()424 public void ReadStringGreaterThanCurrentLimit() 425 { 426 MemoryStream ms = new MemoryStream(); 427 CodedOutputStream output = new CodedOutputStream(ms); 428 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 429 output.WriteRawVarint32(tag); 430 output.WriteRawVarint32(4); 431 output.WriteRawBytes(new byte[4]); // Pad with a few random bytes. 432 output.Flush(); 433 ms.Position = 0; 434 435 CodedInputStream input = new CodedInputStream(ms.ToArray()); 436 Assert.AreEqual(tag, input.ReadTag()); 437 438 // Specify limit smaller than data length 439 input.PushLimit(3); 440 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadString()); 441 442 AssertReadFromParseContext(new ReadOnlySequence<byte>(ms.ToArray()), (ref ParseContext ctx) => 443 { 444 Assert.AreEqual(tag, ctx.ReadTag()); 445 SegmentedBufferHelper.PushLimit(ref ctx.state, 3); 446 try 447 { 448 ctx.ReadString(); 449 Assert.Fail(); 450 } 451 catch (InvalidProtocolBufferException) { } 452 }, true); 453 } 454 455 // Representations of a tag for field 0 with various wire types 456 [Test] 457 [TestCase(0)] 458 [TestCase(1)] 459 [TestCase(2)] 460 [TestCase(3)] 461 [TestCase(4)] 462 [TestCase(5)] ReadTag_ZeroFieldRejected(byte tag)463 public void ReadTag_ZeroFieldRejected(byte tag) 464 { 465 CodedInputStream cis = new CodedInputStream(new byte[] { tag }); 466 Assert.Throws<InvalidProtocolBufferException>(() => cis.ReadTag()); 467 } 468 MakeRecursiveMessage(int depth)469 internal static TestRecursiveMessage MakeRecursiveMessage(int depth) 470 { 471 if (depth == 0) 472 { 473 return new TestRecursiveMessage { I = 5 }; 474 } 475 else 476 { 477 return new TestRecursiveMessage { A = MakeRecursiveMessage(depth - 1) }; 478 } 479 } 480 AssertMessageDepth(TestRecursiveMessage message, int depth)481 internal static void AssertMessageDepth(TestRecursiveMessage message, int depth) 482 { 483 if (depth == 0) 484 { 485 Assert.IsNull(message.A); 486 Assert.AreEqual(5, message.I); 487 } 488 else 489 { 490 Assert.IsNotNull(message.A); 491 AssertMessageDepth(message.A, depth - 1); 492 } 493 } 494 495 [Test] MaliciousRecursion()496 public void MaliciousRecursion() 497 { 498 ByteString atRecursiveLimit = MakeRecursiveMessage(CodedInputStream.DefaultRecursionLimit).ToByteString(); 499 ByteString beyondRecursiveLimit = MakeRecursiveMessage(CodedInputStream.DefaultRecursionLimit + 1).ToByteString(); 500 501 AssertMessageDepth(TestRecursiveMessage.Parser.ParseFrom(atRecursiveLimit), CodedInputStream.DefaultRecursionLimit); 502 503 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(beyondRecursiveLimit)); 504 505 CodedInputStream input = CodedInputStream.CreateWithLimits(new MemoryStream(atRecursiveLimit.ToByteArray()), 1000000, CodedInputStream.DefaultRecursionLimit - 1); 506 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(input)); 507 } 508 MakeMaliciousRecursionUnknownFieldsPayload(int recursionDepth)509 private static byte[] MakeMaliciousRecursionUnknownFieldsPayload(int recursionDepth) 510 { 511 // generate recursively nested groups that will be parsed as unknown fields 512 int unknownFieldNumber = 14; // an unused field number 513 MemoryStream ms = new MemoryStream(); 514 CodedOutputStream output = new CodedOutputStream(ms); 515 for (int i = 0; i < recursionDepth; i++) 516 { 517 output.WriteTag(WireFormat.MakeTag(unknownFieldNumber, WireFormat.WireType.StartGroup)); 518 } 519 for (int i = 0; i < recursionDepth; i++) 520 { 521 output.WriteTag(WireFormat.MakeTag(unknownFieldNumber, WireFormat.WireType.EndGroup)); 522 } 523 output.Flush(); 524 return ms.ToArray(); 525 } 526 527 [Test] MaliciousRecursion_UnknownFields()528 public void MaliciousRecursion_UnknownFields() 529 { 530 byte[] payloadAtRecursiveLimit = MakeMaliciousRecursionUnknownFieldsPayload(CodedInputStream.DefaultRecursionLimit); 531 byte[] payloadBeyondRecursiveLimit = MakeMaliciousRecursionUnknownFieldsPayload(CodedInputStream.DefaultRecursionLimit + 1); 532 533 Assert.DoesNotThrow(() => TestRecursiveMessage.Parser.ParseFrom(payloadAtRecursiveLimit)); 534 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(payloadBeyondRecursiveLimit)); 535 } 536 537 [Test] ReadGroup_WrongEndGroupTag()538 public void ReadGroup_WrongEndGroupTag() 539 { 540 int groupFieldNumber = Proto2.TestAllTypes.OptionalGroupFieldNumber; 541 542 // write Proto2.TestAllTypes with "optional_group" set, but use wrong EndGroup closing tag 543 var ms = new MemoryStream(); 544 var output = new CodedOutputStream(ms); 545 output.WriteTag(WireFormat.MakeTag(groupFieldNumber, WireFormat.WireType.StartGroup)); 546 output.WriteGroup(new Proto2.TestAllTypes.Types.OptionalGroup { A = 12345 }); 547 // end group with different field number 548 output.WriteTag(WireFormat.MakeTag(groupFieldNumber + 1, WireFormat.WireType.EndGroup)); 549 output.Flush(); 550 var payload = ms.ToArray(); 551 552 Assert.Throws<InvalidProtocolBufferException>(() => Proto2.TestAllTypes.Parser.ParseFrom(payload)); 553 } 554 555 [Test] ReadGroup_UnknownFields_WrongEndGroupTag()556 public void ReadGroup_UnknownFields_WrongEndGroupTag() 557 { 558 var ms = new MemoryStream(); 559 var output = new CodedOutputStream(ms); 560 output.WriteTag(WireFormat.MakeTag(14, WireFormat.WireType.StartGroup)); 561 // end group with different field number 562 output.WriteTag(WireFormat.MakeTag(15, WireFormat.WireType.EndGroup)); 563 output.Flush(); 564 var payload = ms.ToArray(); 565 566 Assert.Throws<InvalidProtocolBufferException>(() => TestRecursiveMessage.Parser.ParseFrom(payload)); 567 } 568 569 [Test] SizeLimit()570 public void SizeLimit() 571 { 572 // Have to use a Stream rather than ByteString.CreateCodedInput as SizeLimit doesn't 573 // apply to the latter case. 574 MemoryStream ms = new MemoryStream(SampleMessages.CreateFullTestAllTypes().ToByteArray()); 575 CodedInputStream input = CodedInputStream.CreateWithLimits(ms, 16, 100); 576 Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseFrom(input)); 577 } 578 579 /// <summary> 580 /// Tests that if we read an string that contains invalid UTF-8, no exception 581 /// is thrown. Instead, the invalid bytes are replaced with the Unicode 582 /// "replacement character" U+FFFD. 583 /// </summary> 584 [Test] ReadInvalidUtf8()585 public void ReadInvalidUtf8() 586 { 587 MemoryStream ms = new MemoryStream(); 588 CodedOutputStream output = new CodedOutputStream(ms); 589 590 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 591 output.WriteRawVarint32(tag); 592 output.WriteRawVarint32(1); 593 output.WriteRawBytes(new byte[] {0x80}); 594 output.Flush(); 595 ms.Position = 0; 596 597 CodedInputStream input = new CodedInputStream(ms); 598 599 Assert.AreEqual(tag, input.ReadTag()); 600 string text = input.ReadString(); 601 Assert.AreEqual('\ufffd', text[0]); 602 } 603 604 [Test] ReadNegativeSizedStringThrowsInvalidProtocolBufferException()605 public void ReadNegativeSizedStringThrowsInvalidProtocolBufferException() 606 { 607 MemoryStream ms = new MemoryStream(); 608 CodedOutputStream output = new CodedOutputStream(ms); 609 610 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 611 output.WriteRawVarint32(tag); 612 output.WriteLength(-1); 613 output.Flush(); 614 ms.Position = 0; 615 616 CodedInputStream input = new CodedInputStream(ms); 617 618 Assert.AreEqual(tag, input.ReadTag()); 619 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadString()); 620 } 621 622 [Test] ReadNegativeSizedBytesThrowsInvalidProtocolBufferException()623 public void ReadNegativeSizedBytesThrowsInvalidProtocolBufferException() 624 { 625 MemoryStream ms = new MemoryStream(); 626 CodedOutputStream output = new CodedOutputStream(ms); 627 628 uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); 629 output.WriteRawVarint32(tag); 630 output.WriteLength(-1); 631 output.Flush(); 632 ms.Position = 0; 633 634 var input = new CodedInputStream(ms); 635 636 Assert.AreEqual(tag, input.ReadTag()); 637 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadBytes()); 638 } 639 640 /// <summary> 641 /// A stream which limits the number of bytes it reads at a time. 642 /// We use this to make sure that CodedInputStream doesn't screw up when 643 /// reading in small blocks. 644 /// </summary> 645 private sealed class SmallBlockInputStream : MemoryStream 646 { 647 private readonly int blockSize; 648 SmallBlockInputStream(byte[] data, int blockSize)649 public SmallBlockInputStream(byte[] data, int blockSize) 650 : base(data) 651 { 652 this.blockSize = blockSize; 653 } 654 Read(byte[] buffer, int offset, int count)655 public override int Read(byte[] buffer, int offset, int count) 656 { 657 return base.Read(buffer, offset, Math.Min(count, blockSize)); 658 } 659 } 660 661 [Test] TestNegativeEnum()662 public void TestNegativeEnum() 663 { 664 byte[] bytes = { 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01 }; 665 CodedInputStream input = new CodedInputStream(bytes); 666 Assert.AreEqual((int)SampleEnum.NegativeValue, input.ReadEnum()); 667 Assert.IsTrue(input.IsAtEnd); 668 } 669 670 //Issue 71: CodedInputStream.ReadBytes go to slow path unnecessarily 671 [Test] TestSlowPathAvoidance()672 public void TestSlowPathAvoidance() 673 { 674 using var ms = new MemoryStream(); 675 var output = new CodedOutputStream(ms); 676 output.WriteTag(1, WireFormat.WireType.LengthDelimited); 677 output.WriteBytes(ByteString.CopyFrom(new byte[100])); 678 output.WriteTag(2, WireFormat.WireType.LengthDelimited); 679 output.WriteBytes(ByteString.CopyFrom(new byte[100])); 680 output.Flush(); 681 682 ms.Position = 0; 683 CodedInputStream input = new CodedInputStream(ms, new byte[ms.Length / 2], 0, 0, false); 684 685 uint tag = input.ReadTag(); 686 Assert.AreEqual(1, WireFormat.GetTagFieldNumber(tag)); 687 Assert.AreEqual(100, input.ReadBytes().Length); 688 689 tag = input.ReadTag(); 690 Assert.AreEqual(2, WireFormat.GetTagFieldNumber(tag)); 691 Assert.AreEqual(100, input.ReadBytes().Length); 692 } 693 694 [Test] MaximumFieldNumber()695 public void MaximumFieldNumber() 696 { 697 MemoryStream ms = new MemoryStream(); 698 CodedOutputStream output = new CodedOutputStream(ms); 699 700 int fieldNumber = 0x1FFFFFFF; 701 uint tag = WireFormat.MakeTag(fieldNumber, WireFormat.WireType.LengthDelimited); 702 output.WriteRawVarint32(tag); 703 output.WriteString("field 1"); 704 output.Flush(); 705 ms.Position = 0; 706 707 CodedInputStream input = new CodedInputStream(ms); 708 709 Assert.AreEqual(tag, input.ReadTag()); 710 Assert.AreEqual(fieldNumber, WireFormat.GetTagFieldNumber(tag)); 711 } 712 713 [Test] Tag0Throws()714 public void Tag0Throws() 715 { 716 var input = new CodedInputStream(new byte[] { 0 }); 717 Assert.Throws<InvalidProtocolBufferException>(() => input.ReadTag()); 718 } 719 720 [Test] SkipGroup()721 public void SkipGroup() 722 { 723 // Create an output stream with a group in: 724 // Field 1: string "field 1" 725 // Field 2: group containing: 726 // Field 1: fixed int32 value 100 727 // Field 2: string "ignore me" 728 // Field 3: nested group containing 729 // Field 1: fixed int64 value 1000 730 // Field 3: string "field 3" 731 var stream = new MemoryStream(); 732 var output = new CodedOutputStream(stream); 733 output.WriteTag(1, WireFormat.WireType.LengthDelimited); 734 output.WriteString("field 1"); 735 736 // The outer group... 737 output.WriteTag(2, WireFormat.WireType.StartGroup); 738 output.WriteTag(1, WireFormat.WireType.Fixed32); 739 output.WriteFixed32(100); 740 output.WriteTag(2, WireFormat.WireType.LengthDelimited); 741 output.WriteString("ignore me"); 742 // The nested group... 743 output.WriteTag(3, WireFormat.WireType.StartGroup); 744 output.WriteTag(1, WireFormat.WireType.Fixed64); 745 output.WriteFixed64(1000); 746 // Note: Not sure the field number is relevant for end group... 747 output.WriteTag(3, WireFormat.WireType.EndGroup); 748 749 // End the outer group 750 output.WriteTag(2, WireFormat.WireType.EndGroup); 751 752 output.WriteTag(3, WireFormat.WireType.LengthDelimited); 753 output.WriteString("field 3"); 754 output.Flush(); 755 stream.Position = 0; 756 757 // Now act like a generated client 758 var input = new CodedInputStream(stream); 759 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited), input.ReadTag()); 760 Assert.AreEqual("field 1", input.ReadString()); 761 Assert.AreEqual(WireFormat.MakeTag(2, WireFormat.WireType.StartGroup), input.ReadTag()); 762 input.SkipLastField(); // Should consume the whole group, including the nested one. 763 Assert.AreEqual(WireFormat.MakeTag(3, WireFormat.WireType.LengthDelimited), input.ReadTag()); 764 Assert.AreEqual("field 3", input.ReadString()); 765 } 766 767 [Test] SkipGroup_WrongEndGroupTag()768 public void SkipGroup_WrongEndGroupTag() 769 { 770 // Create an output stream with: 771 // Field 1: string "field 1" 772 // Start group 2 773 // Field 3: fixed int32 774 // End group 4 (should give an error) 775 var stream = new MemoryStream(); 776 var output = new CodedOutputStream(stream); 777 output.WriteTag(1, WireFormat.WireType.LengthDelimited); 778 output.WriteString("field 1"); 779 780 // The outer group... 781 output.WriteTag(2, WireFormat.WireType.StartGroup); 782 output.WriteTag(3, WireFormat.WireType.Fixed32); 783 output.WriteFixed32(100); 784 output.WriteTag(4, WireFormat.WireType.EndGroup); 785 output.Flush(); 786 stream.Position = 0; 787 788 // Now act like a generated client 789 var input = new CodedInputStream(stream); 790 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited), input.ReadTag()); 791 Assert.AreEqual("field 1", input.ReadString()); 792 Assert.AreEqual(WireFormat.MakeTag(2, WireFormat.WireType.StartGroup), input.ReadTag()); 793 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 794 } 795 796 [Test] RogueEndGroupTag()797 public void RogueEndGroupTag() 798 { 799 // If we have an end-group tag without a leading start-group tag, generated 800 // code will just call SkipLastField... so that should fail. 801 802 var stream = new MemoryStream(); 803 var output = new CodedOutputStream(stream); 804 output.WriteTag(1, WireFormat.WireType.EndGroup); 805 output.Flush(); 806 stream.Position = 0; 807 808 var input = new CodedInputStream(stream); 809 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.EndGroup), input.ReadTag()); 810 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 811 } 812 813 [Test] EndOfStreamReachedWhileSkippingGroup()814 public void EndOfStreamReachedWhileSkippingGroup() 815 { 816 var stream = new MemoryStream(); 817 var output = new CodedOutputStream(stream); 818 output.WriteTag(1, WireFormat.WireType.StartGroup); 819 output.WriteTag(2, WireFormat.WireType.StartGroup); 820 output.WriteTag(2, WireFormat.WireType.EndGroup); 821 822 output.Flush(); 823 stream.Position = 0; 824 825 // Now act like a generated client 826 var input = new CodedInputStream(stream); 827 input.ReadTag(); 828 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 829 } 830 831 [Test] RecursionLimitAppliedWhileSkippingGroup()832 public void RecursionLimitAppliedWhileSkippingGroup() 833 { 834 var stream = new MemoryStream(); 835 var output = new CodedOutputStream(stream); 836 for (int i = 0; i < CodedInputStream.DefaultRecursionLimit + 1; i++) 837 { 838 output.WriteTag(1, WireFormat.WireType.StartGroup); 839 } 840 for (int i = 0; i < CodedInputStream.DefaultRecursionLimit + 1; i++) 841 { 842 output.WriteTag(1, WireFormat.WireType.EndGroup); 843 } 844 output.Flush(); 845 stream.Position = 0; 846 847 // Now act like a generated client 848 var input = new CodedInputStream(stream); 849 Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.StartGroup), input.ReadTag()); 850 Assert.Throws<InvalidProtocolBufferException>(input.SkipLastField); 851 } 852 853 [Test] Construction_Invalid()854 public void Construction_Invalid() 855 { 856 Assert.Throws<ArgumentNullException>(() => new CodedInputStream((byte[]) null)); 857 Assert.Throws<ArgumentNullException>(() => new CodedInputStream(null, 0, 0)); 858 Assert.Throws<ArgumentNullException>(() => new CodedInputStream((Stream) null)); 859 Assert.Throws<ArgumentOutOfRangeException>(() => new CodedInputStream(new byte[10], 100, 0)); 860 Assert.Throws<ArgumentOutOfRangeException>(() => new CodedInputStream(new byte[10], 5, 10)); 861 } 862 863 [Test] CreateWithLimits_InvalidLimits()864 public void CreateWithLimits_InvalidLimits() 865 { 866 var stream = new MemoryStream(); 867 Assert.Throws<ArgumentOutOfRangeException>(() => CodedInputStream.CreateWithLimits(stream, 0, 1)); 868 Assert.Throws<ArgumentOutOfRangeException>(() => CodedInputStream.CreateWithLimits(stream, 1, 0)); 869 } 870 871 [Test] Dispose_DisposesUnderlyingStream()872 public void Dispose_DisposesUnderlyingStream() 873 { 874 var memoryStream = new MemoryStream(); 875 Assert.IsTrue(memoryStream.CanRead); 876 using (var cis = new CodedInputStream(memoryStream)) 877 { 878 } 879 Assert.IsFalse(memoryStream.CanRead); // Disposed 880 } 881 882 [Test] Dispose_WithLeaveOpen()883 public void Dispose_WithLeaveOpen() 884 { 885 var memoryStream = new MemoryStream(); 886 Assert.IsTrue(memoryStream.CanRead); 887 using (var cis = new CodedInputStream(memoryStream, true)) 888 { 889 } 890 Assert.IsTrue(memoryStream.CanRead); // We left the stream open 891 } 892 893 [Test] Dispose_FromByteArray()894 public void Dispose_FromByteArray() 895 { 896 var stream = new CodedInputStream(new byte[10]); 897 stream.Dispose(); 898 } 899 900 [Test] TestParseMessagesCloseTo2G()901 public void TestParseMessagesCloseTo2G() 902 { 903 byte[] serializedMessage = GenerateBigSerializedMessage(); 904 // How many of these big messages do we need to take us near our 2GB limit? 905 int count = int.MaxValue / serializedMessage.Length; 906 // Now make a MemoryStream that will fake a near-2GB stream of messages by returning 907 // our big serialized message 'count' times. 908 using var stream = new RepeatingMemoryStream(serializedMessage, count); 909 Assert.DoesNotThrow(() => TestAllTypes.Parser.ParseFrom(stream)); 910 } 911 912 [Test] TestParseMessagesOver2G()913 public void TestParseMessagesOver2G() 914 { 915 byte[] serializedMessage = GenerateBigSerializedMessage(); 916 // How many of these big messages do we need to take us near our 2GB limit? 917 int count = int.MaxValue / serializedMessage.Length; 918 // Now add one to take us over the 2GB limit 919 count++; 920 // Now make a MemoryStream that will fake a near-2GB stream of messages by returning 921 // our big serialized message 'count' times. 922 using var stream = new RepeatingMemoryStream(serializedMessage, count); 923 Assert.Throws<InvalidProtocolBufferException>(() => TestAllTypes.Parser.ParseFrom(stream), 924 "Protocol message was too large. May be malicious. " + 925 "Use CodedInputStream.SetSizeLimit() to increase the size limit."); 926 } 927 928 /// <returns>A serialized big message</returns> GenerateBigSerializedMessage()929 private static byte[] GenerateBigSerializedMessage() 930 { 931 byte[] value = new byte[16 * 1024 * 1024]; 932 TestAllTypes message = SampleMessages.CreateFullTestAllTypes(); 933 message.SingleBytes = ByteString.CopyFrom(value); 934 return message.ToByteArray(); 935 } 936 937 /// <summary> 938 /// A MemoryStream that repeats a byte arrays' content a number of times. 939 /// Simulates really large input without consuming loads of memory. Used above 940 /// to test the parsing behavior when the input size exceeds 2GB or close to it. 941 /// </summary> 942 private class RepeatingMemoryStream: MemoryStream 943 { 944 private readonly byte[] bytes; 945 private readonly int maxIterations; 946 private int index = 0; 947 RepeatingMemoryStream(byte[] bytes, int maxIterations)948 public RepeatingMemoryStream(byte[] bytes, int maxIterations) 949 { 950 this.bytes = bytes; 951 this.maxIterations = maxIterations; 952 } 953 Read(byte[] buffer, int offset, int count)954 public override int Read(byte[] buffer, int offset, int count) 955 { 956 if (bytes.Length == 0) 957 { 958 return 0; 959 } 960 int numBytesCopiedTotal = 0; 961 while (numBytesCopiedTotal < count && index < maxIterations) 962 { 963 int numBytesToCopy = Math.Min(bytes.Length - (int)Position, count); 964 Array.Copy(bytes, (int)Position, buffer, offset, numBytesToCopy); 965 numBytesCopiedTotal += numBytesToCopy; 966 offset += numBytesToCopy; 967 count -= numBytesCopiedTotal; 968 Position += numBytesToCopy; 969 if (Position >= bytes.Length) 970 { 971 Position = 0; 972 index++; 973 } 974 } 975 return numBytesCopiedTotal; 976 } 977 } 978 } 979 } 980