1# Protocol Buffers - Google's data interchange format 2# Copyright 2008 Google Inc. All rights reserved. 3# https://developers.google.com/protocol-buffers/ 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31"""Code for encoding protocol message primitives. 32 33Contains the logic for encoding every logical protocol field type 34into one of the 5 physical wire types. 35 36This code is designed to push the Python interpreter's performance to the 37limits. 38 39The basic idea is that at startup time, for every field (i.e. every 40FieldDescriptor) we construct two functions: a "sizer" and an "encoder". The 41sizer takes a value of this field's type and computes its byte size. The 42encoder takes a writer function and a value. It encodes the value into byte 43strings and invokes the writer function to write those strings. Typically the 44writer function is the write() method of a BytesIO. 45 46We try to do as much work as possible when constructing the writer and the 47sizer rather than when calling them. In particular: 48* We copy any needed global functions to local variables, so that we do not need 49 to do costly global table lookups at runtime. 50* Similarly, we try to do any attribute lookups at startup time if possible. 51* Every field's tag is encoded to bytes at startup, since it can't change at 52 runtime. 53* Whatever component of the field size we can compute at startup, we do. 54* We *avoid* sharing code if doing so would make the code slower and not sharing 55 does not burden us too much. For example, encoders for repeated fields do 56 not just call the encoders for singular fields in a loop because this would 57 add an extra function call overhead for every loop iteration; instead, we 58 manually inline the single-value encoder into the loop. 59* If a Python function lacks a return statement, Python actually generates 60 instructions to pop the result of the last statement off the stack, push 61 None onto the stack, and then return that. If we really don't care what 62 value is returned, then we can save two instructions by returning the 63 result of the last statement. It looks funny but it helps. 64* We assume that type and bounds checking has happened at a higher level. 65""" 66 67__author__ = 'kenton@google.com (Kenton Varda)' 68 69import struct 70 71import six 72 73from google.protobuf.internal import wire_format 74 75 76# This will overflow and thus become IEEE-754 "infinity". We would use 77# "float('inf')" but it doesn't work on Windows pre-Python-2.6. 78_POS_INF = 1e10000 79_NEG_INF = -_POS_INF 80 81 82def _VarintSize(value): 83 """Compute the size of a varint value.""" 84 if value <= 0x7f: return 1 85 if value <= 0x3fff: return 2 86 if value <= 0x1fffff: return 3 87 if value <= 0xfffffff: return 4 88 if value <= 0x7ffffffff: return 5 89 if value <= 0x3ffffffffff: return 6 90 if value <= 0x1ffffffffffff: return 7 91 if value <= 0xffffffffffffff: return 8 92 if value <= 0x7fffffffffffffff: return 9 93 return 10 94 95 96def _SignedVarintSize(value): 97 """Compute the size of a signed varint value.""" 98 if value < 0: return 10 99 if value <= 0x7f: return 1 100 if value <= 0x3fff: return 2 101 if value <= 0x1fffff: return 3 102 if value <= 0xfffffff: return 4 103 if value <= 0x7ffffffff: return 5 104 if value <= 0x3ffffffffff: return 6 105 if value <= 0x1ffffffffffff: return 7 106 if value <= 0xffffffffffffff: return 8 107 if value <= 0x7fffffffffffffff: return 9 108 return 10 109 110 111def _TagSize(field_number): 112 """Returns the number of bytes required to serialize a tag with this field 113 number.""" 114 # Just pass in type 0, since the type won't affect the tag+type size. 115 return _VarintSize(wire_format.PackTag(field_number, 0)) 116 117 118# -------------------------------------------------------------------- 119# In this section we define some generic sizers. Each of these functions 120# takes parameters specific to a particular field type, e.g. int32 or fixed64. 121# It returns another function which in turn takes parameters specific to a 122# particular field, e.g. the field number and whether it is repeated or packed. 123# Look at the next section to see how these are used. 124 125 126def _SimpleSizer(compute_value_size): 127 """A sizer which uses the function compute_value_size to compute the size of 128 each value. Typically compute_value_size is _VarintSize.""" 129 130 def SpecificSizer(field_number, is_repeated, is_packed): 131 tag_size = _TagSize(field_number) 132 if is_packed: 133 local_VarintSize = _VarintSize 134 def PackedFieldSize(value): 135 result = 0 136 for element in value: 137 result += compute_value_size(element) 138 return result + local_VarintSize(result) + tag_size 139 return PackedFieldSize 140 elif is_repeated: 141 def RepeatedFieldSize(value): 142 result = tag_size * len(value) 143 for element in value: 144 result += compute_value_size(element) 145 return result 146 return RepeatedFieldSize 147 else: 148 def FieldSize(value): 149 return tag_size + compute_value_size(value) 150 return FieldSize 151 152 return SpecificSizer 153 154 155def _ModifiedSizer(compute_value_size, modify_value): 156 """Like SimpleSizer, but modify_value is invoked on each value before it is 157 passed to compute_value_size. modify_value is typically ZigZagEncode.""" 158 159 def SpecificSizer(field_number, is_repeated, is_packed): 160 tag_size = _TagSize(field_number) 161 if is_packed: 162 local_VarintSize = _VarintSize 163 def PackedFieldSize(value): 164 result = 0 165 for element in value: 166 result += compute_value_size(modify_value(element)) 167 return result + local_VarintSize(result) + tag_size 168 return PackedFieldSize 169 elif is_repeated: 170 def RepeatedFieldSize(value): 171 result = tag_size * len(value) 172 for element in value: 173 result += compute_value_size(modify_value(element)) 174 return result 175 return RepeatedFieldSize 176 else: 177 def FieldSize(value): 178 return tag_size + compute_value_size(modify_value(value)) 179 return FieldSize 180 181 return SpecificSizer 182 183 184def _FixedSizer(value_size): 185 """Like _SimpleSizer except for a fixed-size field. The input is the size 186 of one value.""" 187 188 def SpecificSizer(field_number, is_repeated, is_packed): 189 tag_size = _TagSize(field_number) 190 if is_packed: 191 local_VarintSize = _VarintSize 192 def PackedFieldSize(value): 193 result = len(value) * value_size 194 return result + local_VarintSize(result) + tag_size 195 return PackedFieldSize 196 elif is_repeated: 197 element_size = value_size + tag_size 198 def RepeatedFieldSize(value): 199 return len(value) * element_size 200 return RepeatedFieldSize 201 else: 202 field_size = value_size + tag_size 203 def FieldSize(value): 204 return field_size 205 return FieldSize 206 207 return SpecificSizer 208 209 210# ==================================================================== 211# Here we declare a sizer constructor for each field type. Each "sizer 212# constructor" is a function that takes (field_number, is_repeated, is_packed) 213# as parameters and returns a sizer, which in turn takes a field value as 214# a parameter and returns its encoded size. 215 216 217Int32Sizer = Int64Sizer = EnumSizer = _SimpleSizer(_SignedVarintSize) 218 219UInt32Sizer = UInt64Sizer = _SimpleSizer(_VarintSize) 220 221SInt32Sizer = SInt64Sizer = _ModifiedSizer( 222 _SignedVarintSize, wire_format.ZigZagEncode) 223 224Fixed32Sizer = SFixed32Sizer = FloatSizer = _FixedSizer(4) 225Fixed64Sizer = SFixed64Sizer = DoubleSizer = _FixedSizer(8) 226 227BoolSizer = _FixedSizer(1) 228 229 230def StringSizer(field_number, is_repeated, is_packed): 231 """Returns a sizer for a string field.""" 232 233 tag_size = _TagSize(field_number) 234 local_VarintSize = _VarintSize 235 local_len = len 236 assert not is_packed 237 if is_repeated: 238 def RepeatedFieldSize(value): 239 result = tag_size * len(value) 240 for element in value: 241 l = local_len(element.encode('utf-8')) 242 result += local_VarintSize(l) + l 243 return result 244 return RepeatedFieldSize 245 else: 246 def FieldSize(value): 247 l = local_len(value.encode('utf-8')) 248 return tag_size + local_VarintSize(l) + l 249 return FieldSize 250 251 252def BytesSizer(field_number, is_repeated, is_packed): 253 """Returns a sizer for a bytes field.""" 254 255 tag_size = _TagSize(field_number) 256 local_VarintSize = _VarintSize 257 local_len = len 258 assert not is_packed 259 if is_repeated: 260 def RepeatedFieldSize(value): 261 result = tag_size * len(value) 262 for element in value: 263 l = local_len(element) 264 result += local_VarintSize(l) + l 265 return result 266 return RepeatedFieldSize 267 else: 268 def FieldSize(value): 269 l = local_len(value) 270 return tag_size + local_VarintSize(l) + l 271 return FieldSize 272 273 274def GroupSizer(field_number, is_repeated, is_packed): 275 """Returns a sizer for a group field.""" 276 277 tag_size = _TagSize(field_number) * 2 278 assert not is_packed 279 if is_repeated: 280 def RepeatedFieldSize(value): 281 result = tag_size * len(value) 282 for element in value: 283 result += element.ByteSize() 284 return result 285 return RepeatedFieldSize 286 else: 287 def FieldSize(value): 288 return tag_size + value.ByteSize() 289 return FieldSize 290 291 292def MessageSizer(field_number, is_repeated, is_packed): 293 """Returns a sizer for a message field.""" 294 295 tag_size = _TagSize(field_number) 296 local_VarintSize = _VarintSize 297 assert not is_packed 298 if is_repeated: 299 def RepeatedFieldSize(value): 300 result = tag_size * len(value) 301 for element in value: 302 l = element.ByteSize() 303 result += local_VarintSize(l) + l 304 return result 305 return RepeatedFieldSize 306 else: 307 def FieldSize(value): 308 l = value.ByteSize() 309 return tag_size + local_VarintSize(l) + l 310 return FieldSize 311 312 313# -------------------------------------------------------------------- 314# MessageSet is special: it needs custom logic to compute its size properly. 315 316 317def MessageSetItemSizer(field_number): 318 """Returns a sizer for extensions of MessageSet. 319 320 The message set message looks like this: 321 message MessageSet { 322 repeated group Item = 1 { 323 required int32 type_id = 2; 324 required string message = 3; 325 } 326 } 327 """ 328 static_size = (_TagSize(1) * 2 + _TagSize(2) + _VarintSize(field_number) + 329 _TagSize(3)) 330 local_VarintSize = _VarintSize 331 332 def FieldSize(value): 333 l = value.ByteSize() 334 return static_size + local_VarintSize(l) + l 335 336 return FieldSize 337 338 339# -------------------------------------------------------------------- 340# Map is special: it needs custom logic to compute its size properly. 341 342 343def MapSizer(field_descriptor): 344 """Returns a sizer for a map field.""" 345 346 # Can't look at field_descriptor.message_type._concrete_class because it may 347 # not have been initialized yet. 348 message_type = field_descriptor.message_type 349 message_sizer = MessageSizer(field_descriptor.number, False, False) 350 351 def FieldSize(map_value): 352 total = 0 353 for key in map_value: 354 value = map_value[key] 355 # It's wasteful to create the messages and throw them away one second 356 # later since we'll do the same for the actual encode. But there's not an 357 # obvious way to avoid this within the current design without tons of code 358 # duplication. 359 entry_msg = message_type._concrete_class(key=key, value=value) 360 total += message_sizer(entry_msg) 361 return total 362 363 return FieldSize 364 365# ==================================================================== 366# Encoders! 367 368 369def _VarintEncoder(): 370 """Return an encoder for a basic varint value (does not include tag).""" 371 372 def EncodeVarint(write, value): 373 bits = value & 0x7f 374 value >>= 7 375 while value: 376 write(six.int2byte(0x80|bits)) 377 bits = value & 0x7f 378 value >>= 7 379 return write(six.int2byte(bits)) 380 381 return EncodeVarint 382 383 384def _SignedVarintEncoder(): 385 """Return an encoder for a basic signed varint value (does not include 386 tag).""" 387 388 def EncodeSignedVarint(write, value): 389 if value < 0: 390 value += (1 << 64) 391 bits = value & 0x7f 392 value >>= 7 393 while value: 394 write(six.int2byte(0x80|bits)) 395 bits = value & 0x7f 396 value >>= 7 397 return write(six.int2byte(bits)) 398 399 return EncodeSignedVarint 400 401 402_EncodeVarint = _VarintEncoder() 403_EncodeSignedVarint = _SignedVarintEncoder() 404 405 406def _VarintBytes(value): 407 """Encode the given integer as a varint and return the bytes. This is only 408 called at startup time so it doesn't need to be fast.""" 409 410 pieces = [] 411 _EncodeVarint(pieces.append, value) 412 return b"".join(pieces) 413 414 415def TagBytes(field_number, wire_type): 416 """Encode the given tag and return the bytes. Only called at startup.""" 417 418 return _VarintBytes(wire_format.PackTag(field_number, wire_type)) 419 420# -------------------------------------------------------------------- 421# As with sizers (see above), we have a number of common encoder 422# implementations. 423 424 425def _SimpleEncoder(wire_type, encode_value, compute_value_size): 426 """Return a constructor for an encoder for fields of a particular type. 427 428 Args: 429 wire_type: The field's wire type, for encoding tags. 430 encode_value: A function which encodes an individual value, e.g. 431 _EncodeVarint(). 432 compute_value_size: A function which computes the size of an individual 433 value, e.g. _VarintSize(). 434 """ 435 436 def SpecificEncoder(field_number, is_repeated, is_packed): 437 if is_packed: 438 tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 439 local_EncodeVarint = _EncodeVarint 440 def EncodePackedField(write, value): 441 write(tag_bytes) 442 size = 0 443 for element in value: 444 size += compute_value_size(element) 445 local_EncodeVarint(write, size) 446 for element in value: 447 encode_value(write, element) 448 return EncodePackedField 449 elif is_repeated: 450 tag_bytes = TagBytes(field_number, wire_type) 451 def EncodeRepeatedField(write, value): 452 for element in value: 453 write(tag_bytes) 454 encode_value(write, element) 455 return EncodeRepeatedField 456 else: 457 tag_bytes = TagBytes(field_number, wire_type) 458 def EncodeField(write, value): 459 write(tag_bytes) 460 return encode_value(write, value) 461 return EncodeField 462 463 return SpecificEncoder 464 465 466def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value): 467 """Like SimpleEncoder but additionally invokes modify_value on every value 468 before passing it to encode_value. Usually modify_value is ZigZagEncode.""" 469 470 def SpecificEncoder(field_number, is_repeated, is_packed): 471 if is_packed: 472 tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 473 local_EncodeVarint = _EncodeVarint 474 def EncodePackedField(write, value): 475 write(tag_bytes) 476 size = 0 477 for element in value: 478 size += compute_value_size(modify_value(element)) 479 local_EncodeVarint(write, size) 480 for element in value: 481 encode_value(write, modify_value(element)) 482 return EncodePackedField 483 elif is_repeated: 484 tag_bytes = TagBytes(field_number, wire_type) 485 def EncodeRepeatedField(write, value): 486 for element in value: 487 write(tag_bytes) 488 encode_value(write, modify_value(element)) 489 return EncodeRepeatedField 490 else: 491 tag_bytes = TagBytes(field_number, wire_type) 492 def EncodeField(write, value): 493 write(tag_bytes) 494 return encode_value(write, modify_value(value)) 495 return EncodeField 496 497 return SpecificEncoder 498 499 500def _StructPackEncoder(wire_type, format): 501 """Return a constructor for an encoder for a fixed-width field. 502 503 Args: 504 wire_type: The field's wire type, for encoding tags. 505 format: The format string to pass to struct.pack(). 506 """ 507 508 value_size = struct.calcsize(format) 509 510 def SpecificEncoder(field_number, is_repeated, is_packed): 511 local_struct_pack = struct.pack 512 if is_packed: 513 tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 514 local_EncodeVarint = _EncodeVarint 515 def EncodePackedField(write, value): 516 write(tag_bytes) 517 local_EncodeVarint(write, len(value) * value_size) 518 for element in value: 519 write(local_struct_pack(format, element)) 520 return EncodePackedField 521 elif is_repeated: 522 tag_bytes = TagBytes(field_number, wire_type) 523 def EncodeRepeatedField(write, value): 524 for element in value: 525 write(tag_bytes) 526 write(local_struct_pack(format, element)) 527 return EncodeRepeatedField 528 else: 529 tag_bytes = TagBytes(field_number, wire_type) 530 def EncodeField(write, value): 531 write(tag_bytes) 532 return write(local_struct_pack(format, value)) 533 return EncodeField 534 535 return SpecificEncoder 536 537 538def _FloatingPointEncoder(wire_type, format): 539 """Return a constructor for an encoder for float fields. 540 541 This is like StructPackEncoder, but catches errors that may be due to 542 passing non-finite floating-point values to struct.pack, and makes a 543 second attempt to encode those values. 544 545 Args: 546 wire_type: The field's wire type, for encoding tags. 547 format: The format string to pass to struct.pack(). 548 """ 549 550 value_size = struct.calcsize(format) 551 if value_size == 4: 552 def EncodeNonFiniteOrRaise(write, value): 553 # Remember that the serialized form uses little-endian byte order. 554 if value == _POS_INF: 555 write(b'\x00\x00\x80\x7F') 556 elif value == _NEG_INF: 557 write(b'\x00\x00\x80\xFF') 558 elif value != value: # NaN 559 write(b'\x00\x00\xC0\x7F') 560 else: 561 raise 562 elif value_size == 8: 563 def EncodeNonFiniteOrRaise(write, value): 564 if value == _POS_INF: 565 write(b'\x00\x00\x00\x00\x00\x00\xF0\x7F') 566 elif value == _NEG_INF: 567 write(b'\x00\x00\x00\x00\x00\x00\xF0\xFF') 568 elif value != value: # NaN 569 write(b'\x00\x00\x00\x00\x00\x00\xF8\x7F') 570 else: 571 raise 572 else: 573 raise ValueError('Can\'t encode floating-point values that are ' 574 '%d bytes long (only 4 or 8)' % value_size) 575 576 def SpecificEncoder(field_number, is_repeated, is_packed): 577 local_struct_pack = struct.pack 578 if is_packed: 579 tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 580 local_EncodeVarint = _EncodeVarint 581 def EncodePackedField(write, value): 582 write(tag_bytes) 583 local_EncodeVarint(write, len(value) * value_size) 584 for element in value: 585 # This try/except block is going to be faster than any code that 586 # we could write to check whether element is finite. 587 try: 588 write(local_struct_pack(format, element)) 589 except SystemError: 590 EncodeNonFiniteOrRaise(write, element) 591 return EncodePackedField 592 elif is_repeated: 593 tag_bytes = TagBytes(field_number, wire_type) 594 def EncodeRepeatedField(write, value): 595 for element in value: 596 write(tag_bytes) 597 try: 598 write(local_struct_pack(format, element)) 599 except SystemError: 600 EncodeNonFiniteOrRaise(write, element) 601 return EncodeRepeatedField 602 else: 603 tag_bytes = TagBytes(field_number, wire_type) 604 def EncodeField(write, value): 605 write(tag_bytes) 606 try: 607 write(local_struct_pack(format, value)) 608 except SystemError: 609 EncodeNonFiniteOrRaise(write, value) 610 return EncodeField 611 612 return SpecificEncoder 613 614 615# ==================================================================== 616# Here we declare an encoder constructor for each field type. These work 617# very similarly to sizer constructors, described earlier. 618 619 620Int32Encoder = Int64Encoder = EnumEncoder = _SimpleEncoder( 621 wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize) 622 623UInt32Encoder = UInt64Encoder = _SimpleEncoder( 624 wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize) 625 626SInt32Encoder = SInt64Encoder = _ModifiedEncoder( 627 wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize, 628 wire_format.ZigZagEncode) 629 630# Note that Python conveniently guarantees that when using the '<' prefix on 631# formats, they will also have the same size across all platforms (as opposed 632# to without the prefix, where their sizes depend on the C compiler's basic 633# type sizes). 634Fixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<I') 635Fixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<Q') 636SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<i') 637SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<q') 638FloatEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, '<f') 639DoubleEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d') 640 641 642def BoolEncoder(field_number, is_repeated, is_packed): 643 """Returns an encoder for a boolean field.""" 644 645 false_byte = b'\x00' 646 true_byte = b'\x01' 647 if is_packed: 648 tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 649 local_EncodeVarint = _EncodeVarint 650 def EncodePackedField(write, value): 651 write(tag_bytes) 652 local_EncodeVarint(write, len(value)) 653 for element in value: 654 if element: 655 write(true_byte) 656 else: 657 write(false_byte) 658 return EncodePackedField 659 elif is_repeated: 660 tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) 661 def EncodeRepeatedField(write, value): 662 for element in value: 663 write(tag_bytes) 664 if element: 665 write(true_byte) 666 else: 667 write(false_byte) 668 return EncodeRepeatedField 669 else: 670 tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) 671 def EncodeField(write, value): 672 write(tag_bytes) 673 if value: 674 return write(true_byte) 675 return write(false_byte) 676 return EncodeField 677 678 679def StringEncoder(field_number, is_repeated, is_packed): 680 """Returns an encoder for a string field.""" 681 682 tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 683 local_EncodeVarint = _EncodeVarint 684 local_len = len 685 assert not is_packed 686 if is_repeated: 687 def EncodeRepeatedField(write, value): 688 for element in value: 689 encoded = element.encode('utf-8') 690 write(tag) 691 local_EncodeVarint(write, local_len(encoded)) 692 write(encoded) 693 return EncodeRepeatedField 694 else: 695 def EncodeField(write, value): 696 encoded = value.encode('utf-8') 697 write(tag) 698 local_EncodeVarint(write, local_len(encoded)) 699 return write(encoded) 700 return EncodeField 701 702 703def BytesEncoder(field_number, is_repeated, is_packed): 704 """Returns an encoder for a bytes field.""" 705 706 tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 707 local_EncodeVarint = _EncodeVarint 708 local_len = len 709 assert not is_packed 710 if is_repeated: 711 def EncodeRepeatedField(write, value): 712 for element in value: 713 write(tag) 714 local_EncodeVarint(write, local_len(element)) 715 write(element) 716 return EncodeRepeatedField 717 else: 718 def EncodeField(write, value): 719 write(tag) 720 local_EncodeVarint(write, local_len(value)) 721 return write(value) 722 return EncodeField 723 724 725def GroupEncoder(field_number, is_repeated, is_packed): 726 """Returns an encoder for a group field.""" 727 728 start_tag = TagBytes(field_number, wire_format.WIRETYPE_START_GROUP) 729 end_tag = TagBytes(field_number, wire_format.WIRETYPE_END_GROUP) 730 assert not is_packed 731 if is_repeated: 732 def EncodeRepeatedField(write, value): 733 for element in value: 734 write(start_tag) 735 element._InternalSerialize(write) 736 write(end_tag) 737 return EncodeRepeatedField 738 else: 739 def EncodeField(write, value): 740 write(start_tag) 741 value._InternalSerialize(write) 742 return write(end_tag) 743 return EncodeField 744 745 746def MessageEncoder(field_number, is_repeated, is_packed): 747 """Returns an encoder for a message field.""" 748 749 tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) 750 local_EncodeVarint = _EncodeVarint 751 assert not is_packed 752 if is_repeated: 753 def EncodeRepeatedField(write, value): 754 for element in value: 755 write(tag) 756 local_EncodeVarint(write, element.ByteSize()) 757 element._InternalSerialize(write) 758 return EncodeRepeatedField 759 else: 760 def EncodeField(write, value): 761 write(tag) 762 local_EncodeVarint(write, value.ByteSize()) 763 return value._InternalSerialize(write) 764 return EncodeField 765 766 767# -------------------------------------------------------------------- 768# As before, MessageSet is special. 769 770 771def MessageSetItemEncoder(field_number): 772 """Encoder for extensions of MessageSet. 773 774 The message set message looks like this: 775 message MessageSet { 776 repeated group Item = 1 { 777 required int32 type_id = 2; 778 required string message = 3; 779 } 780 } 781 """ 782 start_bytes = b"".join([ 783 TagBytes(1, wire_format.WIRETYPE_START_GROUP), 784 TagBytes(2, wire_format.WIRETYPE_VARINT), 785 _VarintBytes(field_number), 786 TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)]) 787 end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP) 788 local_EncodeVarint = _EncodeVarint 789 790 def EncodeField(write, value): 791 write(start_bytes) 792 local_EncodeVarint(write, value.ByteSize()) 793 value._InternalSerialize(write) 794 return write(end_bytes) 795 796 return EncodeField 797 798 799# -------------------------------------------------------------------- 800# As before, Map is special. 801 802 803def MapEncoder(field_descriptor): 804 """Encoder for extensions of MessageSet. 805 806 Maps always have a wire format like this: 807 message MapEntry { 808 key_type key = 1; 809 value_type value = 2; 810 } 811 repeated MapEntry map = N; 812 """ 813 # Can't look at field_descriptor.message_type._concrete_class because it may 814 # not have been initialized yet. 815 message_type = field_descriptor.message_type 816 encode_message = MessageEncoder(field_descriptor.number, False, False) 817 818 def EncodeField(write, value): 819 for key in value: 820 entry_msg = message_type._concrete_class(key=key, value=value[key]) 821 encode_message(write, entry_msg) 822 823 return EncodeField 824