• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc.  All rights reserved.
3#
4# Use of this source code is governed by a BSD-style
5# license that can be found in the LICENSE file or at
6# https://developers.google.com/open-source/licenses/bsd
7
8"""Constants and static functions to support protocol buffer wire format."""
9
10__author__ = 'robinson@google.com (Will Robinson)'
11
12import struct
13from google.protobuf import descriptor
14from google.protobuf import message
15
16
17TAG_TYPE_BITS = 3  # Number of bits used to hold type info in a proto tag.
18TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1  # 0x7
19
20# These numbers identify the wire type of a protocol buffer value.
21# We use the least-significant TAG_TYPE_BITS bits of the varint-encoded
22# tag-and-type to store one of these WIRETYPE_* constants.
23# These values must match WireType enum in //google/protobuf/wire_format.h.
24WIRETYPE_VARINT = 0
25WIRETYPE_FIXED64 = 1
26WIRETYPE_LENGTH_DELIMITED = 2
27WIRETYPE_START_GROUP = 3
28WIRETYPE_END_GROUP = 4
29WIRETYPE_FIXED32 = 5
30_WIRETYPE_MAX = 5
31
32
33# Bounds for various integer types.
34INT32_MAX = int((1 << 31) - 1)
35INT32_MIN = int(-(1 << 31))
36UINT32_MAX = (1 << 32) - 1
37
38INT64_MAX = (1 << 63) - 1
39INT64_MIN = -(1 << 63)
40UINT64_MAX = (1 << 64) - 1
41
42# "struct" format strings that will encode/decode the specified formats.
43FORMAT_UINT32_LITTLE_ENDIAN = '<I'
44FORMAT_UINT64_LITTLE_ENDIAN = '<Q'
45FORMAT_FLOAT_LITTLE_ENDIAN = '<f'
46FORMAT_DOUBLE_LITTLE_ENDIAN = '<d'
47
48
49# We'll have to provide alternate implementations of AppendLittleEndian*() on
50# any architectures where these checks fail.
51if struct.calcsize(FORMAT_UINT32_LITTLE_ENDIAN) != 4:
52  raise AssertionError('Format "I" is not a 32-bit number.')
53if struct.calcsize(FORMAT_UINT64_LITTLE_ENDIAN) != 8:
54  raise AssertionError('Format "Q" is not a 64-bit number.')
55
56
57def PackTag(field_number, wire_type):
58  """Returns an unsigned 32-bit integer that encodes the field number and
59  wire type information in standard protocol message wire format.
60
61  Args:
62    field_number: Expected to be an integer in the range [1, 1 << 29)
63    wire_type: One of the WIRETYPE_* constants.
64  """
65  if not 0 <= wire_type <= _WIRETYPE_MAX:
66    raise message.EncodeError('Unknown wire type: %d' % wire_type)
67  return (field_number << TAG_TYPE_BITS) | wire_type
68
69
70def UnpackTag(tag):
71  """The inverse of PackTag().  Given an unsigned 32-bit number,
72  returns a (field_number, wire_type) tuple.
73  """
74  return (tag >> TAG_TYPE_BITS), (tag & TAG_TYPE_MASK)
75
76
77def ZigZagEncode(value):
78  """ZigZag Transform:  Encodes signed integers so that they can be
79  effectively used with varint encoding.  See wire_format.h for
80  more details.
81  """
82  if value >= 0:
83    return value << 1
84  return (value << 1) ^ (~0)
85
86
87def ZigZagDecode(value):
88  """Inverse of ZigZagEncode()."""
89  if not value & 0x1:
90    return value >> 1
91  return (value >> 1) ^ (~0)
92
93
94
95# The *ByteSize() functions below return the number of bytes required to
96# serialize "field number + type" information and then serialize the value.
97
98
99def Int32ByteSize(field_number, int32):
100  return Int64ByteSize(field_number, int32)
101
102
103def Int32ByteSizeNoTag(int32):
104  return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32)
105
106
107def Int64ByteSize(field_number, int64):
108  # Have to convert to uint before calling UInt64ByteSize().
109  return UInt64ByteSize(field_number, 0xffffffffffffffff & int64)
110
111
112def UInt32ByteSize(field_number, uint32):
113  return UInt64ByteSize(field_number, uint32)
114
115
116def UInt64ByteSize(field_number, uint64):
117  return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64)
118
119
120def SInt32ByteSize(field_number, int32):
121  return UInt32ByteSize(field_number, ZigZagEncode(int32))
122
123
124def SInt64ByteSize(field_number, int64):
125  return UInt64ByteSize(field_number, ZigZagEncode(int64))
126
127
128def Fixed32ByteSize(field_number, fixed32):
129  return TagByteSize(field_number) + 4
130
131
132def Fixed64ByteSize(field_number, fixed64):
133  return TagByteSize(field_number) + 8
134
135
136def SFixed32ByteSize(field_number, sfixed32):
137  return TagByteSize(field_number) + 4
138
139
140def SFixed64ByteSize(field_number, sfixed64):
141  return TagByteSize(field_number) + 8
142
143
144def FloatByteSize(field_number, flt):
145  return TagByteSize(field_number) + 4
146
147
148def DoubleByteSize(field_number, double):
149  return TagByteSize(field_number) + 8
150
151
152def BoolByteSize(field_number, b):
153  return TagByteSize(field_number) + 1
154
155
156def EnumByteSize(field_number, enum):
157  return UInt32ByteSize(field_number, enum)
158
159
160def StringByteSize(field_number, string):
161  return BytesByteSize(field_number, string.encode('utf-8'))
162
163
164def BytesByteSize(field_number, b):
165  return (TagByteSize(field_number)
166          + _VarUInt64ByteSizeNoTag(len(b))
167          + len(b))
168
169
170def GroupByteSize(field_number, message):
171  return (2 * TagByteSize(field_number)  # START and END group.
172          + message.ByteSize())
173
174
175def MessageByteSize(field_number, message):
176  return (TagByteSize(field_number)
177          + _VarUInt64ByteSizeNoTag(message.ByteSize())
178          + message.ByteSize())
179
180
181def MessageSetItemByteSize(field_number, msg):
182  # First compute the sizes of the tags.
183  # There are 2 tags for the beginning and ending of the repeated group, that
184  # is field number 1, one with field number 2 (type_id) and one with field
185  # number 3 (message).
186  total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3))
187
188  # Add the number of bytes for type_id.
189  total_size += _VarUInt64ByteSizeNoTag(field_number)
190
191  message_size = msg.ByteSize()
192
193  # The number of bytes for encoding the length of the message.
194  total_size += _VarUInt64ByteSizeNoTag(message_size)
195
196  # The size of the message.
197  total_size += message_size
198  return total_size
199
200
201def TagByteSize(field_number):
202  """Returns the bytes required to serialize a tag with this field number."""
203  # Just pass in type 0, since the type won't affect the tag+type size.
204  return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0))
205
206
207# Private helper function for the *ByteSize() functions above.
208
209def _VarUInt64ByteSizeNoTag(uint64):
210  """Returns the number of bytes required to serialize a single varint
211  using boundary value comparisons. (unrolled loop optimization -WPierce)
212  uint64 must be unsigned.
213  """
214  if uint64 <= 0x7f: return 1
215  if uint64 <= 0x3fff: return 2
216  if uint64 <= 0x1fffff: return 3
217  if uint64 <= 0xfffffff: return 4
218  if uint64 <= 0x7ffffffff: return 5
219  if uint64 <= 0x3ffffffffff: return 6
220  if uint64 <= 0x1ffffffffffff: return 7
221  if uint64 <= 0xffffffffffffff: return 8
222  if uint64 <= 0x7fffffffffffffff: return 9
223  if uint64 > UINT64_MAX:
224    raise message.EncodeError('Value out of range: %d' % uint64)
225  return 10
226
227
228NON_PACKABLE_TYPES = (
229  descriptor.FieldDescriptor.TYPE_STRING,
230  descriptor.FieldDescriptor.TYPE_GROUP,
231  descriptor.FieldDescriptor.TYPE_MESSAGE,
232  descriptor.FieldDescriptor.TYPE_BYTES
233)
234
235
236def IsTypePackable(field_type):
237  """Return true iff packable = true is valid for fields of this type.
238
239  Args:
240    field_type: a FieldDescriptor::Type value.
241
242  Returns:
243    True iff fields of this type are packable.
244  """
245  return field_type not in NON_PACKABLE_TYPES
246