1# Protocol Buffers - Google's data interchange format 2# Copyright 2008 Google Inc. All rights reserved. 3# https://developers.google.com/protocol-buffers/ 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31"""Contains routines for printing protocol messages in JSON format. 32 33Simple usage example: 34 35 # Create a proto object and serialize it to a json format string. 36 message = my_proto_pb2.MyMessage(foo='bar') 37 json_string = json_format.MessageToJson(message) 38 39 # Parse a json format string to proto object. 40 message = json_format.Parse(json_string, my_proto_pb2.MyMessage()) 41""" 42 43__author__ = 'jieluo@google.com (Jie Luo)' 44 45 46import base64 47from collections import OrderedDict 48import json 49import math 50from operator import methodcaller 51import re 52import sys 53 54from google.protobuf.internal import type_checkers 55from google.protobuf import descriptor 56from google.protobuf import symbol_database 57 58 59_TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S' 60_INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32, 61 descriptor.FieldDescriptor.CPPTYPE_UINT32, 62 descriptor.FieldDescriptor.CPPTYPE_INT64, 63 descriptor.FieldDescriptor.CPPTYPE_UINT64]) 64_INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64, 65 descriptor.FieldDescriptor.CPPTYPE_UINT64]) 66_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, 67 descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) 68_INFINITY = 'Infinity' 69_NEG_INFINITY = '-Infinity' 70_NAN = 'NaN' 71 72_UNPAIRED_SURROGATE_PATTERN = re.compile( 73 u'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]') 74 75_VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$') 76 77 78class Error(Exception): 79 """Top-level module error for json_format.""" 80 81 82class SerializeToJsonError(Error): 83 """Thrown if serialization to JSON fails.""" 84 85 86class ParseError(Error): 87 """Thrown in case of parsing error.""" 88 89 90def MessageToJson( 91 message, 92 including_default_value_fields=False, 93 preserving_proto_field_name=False, 94 indent=2, 95 sort_keys=False, 96 use_integers_for_enums=False, 97 descriptor_pool=None, 98 float_precision=None, 99 ensure_ascii=True): 100 """Converts protobuf message to JSON format. 101 102 Args: 103 message: The protocol buffers message instance to serialize. 104 including_default_value_fields: If True, singular primitive fields, 105 repeated fields, and map fields will always be serialized. If 106 False, only serialize non-empty fields. Singular message fields 107 and oneof fields are not affected by this option. 108 preserving_proto_field_name: If True, use the original proto field 109 names as defined in the .proto file. If False, convert the field 110 names to lowerCamelCase. 111 indent: The JSON object will be pretty-printed with this indent level. 112 An indent level of 0 or negative will only insert newlines. 113 sort_keys: If True, then the output will be sorted by field names. 114 use_integers_for_enums: If true, print integers instead of enum names. 115 descriptor_pool: A Descriptor Pool for resolving types. If None use the 116 default. 117 float_precision: If set, use this to specify float field valid digits. 118 ensure_ascii: If True, strings with non-ASCII characters are escaped. 119 If False, Unicode strings are returned unchanged. 120 121 Returns: 122 A string containing the JSON formatted protocol buffer message. 123 """ 124 printer = _Printer( 125 including_default_value_fields, 126 preserving_proto_field_name, 127 use_integers_for_enums, 128 descriptor_pool, 129 float_precision=float_precision) 130 return printer.ToJsonString(message, indent, sort_keys, ensure_ascii) 131 132 133def MessageToDict( 134 message, 135 including_default_value_fields=False, 136 preserving_proto_field_name=False, 137 use_integers_for_enums=False, 138 descriptor_pool=None, 139 float_precision=None): 140 """Converts protobuf message to a dictionary. 141 142 When the dictionary is encoded to JSON, it conforms to proto3 JSON spec. 143 144 Args: 145 message: The protocol buffers message instance to serialize. 146 including_default_value_fields: If True, singular primitive fields, 147 repeated fields, and map fields will always be serialized. If 148 False, only serialize non-empty fields. Singular message fields 149 and oneof fields are not affected by this option. 150 preserving_proto_field_name: If True, use the original proto field 151 names as defined in the .proto file. If False, convert the field 152 names to lowerCamelCase. 153 use_integers_for_enums: If true, print integers instead of enum names. 154 descriptor_pool: A Descriptor Pool for resolving types. If None use the 155 default. 156 float_precision: If set, use this to specify float field valid digits. 157 158 Returns: 159 A dict representation of the protocol buffer message. 160 """ 161 printer = _Printer( 162 including_default_value_fields, 163 preserving_proto_field_name, 164 use_integers_for_enums, 165 descriptor_pool, 166 float_precision=float_precision) 167 # pylint: disable=protected-access 168 return printer._MessageToJsonObject(message) 169 170 171def _IsMapEntry(field): 172 return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and 173 field.message_type.has_options and 174 field.message_type.GetOptions().map_entry) 175 176 177class _Printer(object): 178 """JSON format printer for protocol message.""" 179 180 def __init__( 181 self, 182 including_default_value_fields=False, 183 preserving_proto_field_name=False, 184 use_integers_for_enums=False, 185 descriptor_pool=None, 186 float_precision=None): 187 self.including_default_value_fields = including_default_value_fields 188 self.preserving_proto_field_name = preserving_proto_field_name 189 self.use_integers_for_enums = use_integers_for_enums 190 self.descriptor_pool = descriptor_pool 191 if float_precision: 192 self.float_format = '.{}g'.format(float_precision) 193 else: 194 self.float_format = None 195 196 def ToJsonString(self, message, indent, sort_keys, ensure_ascii): 197 js = self._MessageToJsonObject(message) 198 return json.dumps( 199 js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii) 200 201 def _MessageToJsonObject(self, message): 202 """Converts message to an object according to Proto3 JSON Specification.""" 203 message_descriptor = message.DESCRIPTOR 204 full_name = message_descriptor.full_name 205 if _IsWrapperMessage(message_descriptor): 206 return self._WrapperMessageToJsonObject(message) 207 if full_name in _WKTJSONMETHODS: 208 return methodcaller(_WKTJSONMETHODS[full_name][0], message)(self) 209 js = {} 210 return self._RegularMessageToJsonObject(message, js) 211 212 def _RegularMessageToJsonObject(self, message, js): 213 """Converts normal message according to Proto3 JSON Specification.""" 214 fields = message.ListFields() 215 216 try: 217 for field, value in fields: 218 if self.preserving_proto_field_name: 219 name = field.name 220 else: 221 name = field.json_name 222 if _IsMapEntry(field): 223 # Convert a map field. 224 v_field = field.message_type.fields_by_name['value'] 225 js_map = {} 226 for key in value: 227 if isinstance(key, bool): 228 if key: 229 recorded_key = 'true' 230 else: 231 recorded_key = 'false' 232 else: 233 recorded_key = str(key) 234 js_map[recorded_key] = self._FieldToJsonObject( 235 v_field, value[key]) 236 js[name] = js_map 237 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: 238 # Convert a repeated field. 239 js[name] = [self._FieldToJsonObject(field, k) 240 for k in value] 241 elif field.is_extension: 242 name = '[%s]' % field.full_name 243 js[name] = self._FieldToJsonObject(field, value) 244 else: 245 js[name] = self._FieldToJsonObject(field, value) 246 247 # Serialize default value if including_default_value_fields is True. 248 if self.including_default_value_fields: 249 message_descriptor = message.DESCRIPTOR 250 for field in message_descriptor.fields: 251 # Singular message fields and oneof fields will not be affected. 252 if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and 253 field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or 254 field.containing_oneof): 255 continue 256 if self.preserving_proto_field_name: 257 name = field.name 258 else: 259 name = field.json_name 260 if name in js: 261 # Skip the field which has been serialized already. 262 continue 263 if _IsMapEntry(field): 264 js[name] = {} 265 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: 266 js[name] = [] 267 else: 268 js[name] = self._FieldToJsonObject(field, field.default_value) 269 270 except ValueError as e: 271 raise SerializeToJsonError( 272 'Failed to serialize {0} field: {1}.'.format(field.name, e)) 273 274 return js 275 276 def _FieldToJsonObject(self, field, value): 277 """Converts field value according to Proto3 JSON Specification.""" 278 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 279 return self._MessageToJsonObject(value) 280 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: 281 if self.use_integers_for_enums: 282 return value 283 if field.enum_type.full_name == 'google.protobuf.NullValue': 284 return None 285 enum_value = field.enum_type.values_by_number.get(value, None) 286 if enum_value is not None: 287 return enum_value.name 288 else: 289 if field.file.syntax == 'proto3': 290 return value 291 raise SerializeToJsonError('Enum field contains an integer value ' 292 'which can not mapped to an enum value.') 293 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: 294 if field.type == descriptor.FieldDescriptor.TYPE_BYTES: 295 # Use base64 Data encoding for bytes 296 return base64.b64encode(value).decode('utf-8') 297 else: 298 return value 299 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: 300 return bool(value) 301 elif field.cpp_type in _INT64_TYPES: 302 return str(value) 303 elif field.cpp_type in _FLOAT_TYPES: 304 if math.isinf(value): 305 if value < 0.0: 306 return _NEG_INFINITY 307 else: 308 return _INFINITY 309 if math.isnan(value): 310 return _NAN 311 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT: 312 if self.float_format: 313 return float(format(value, self.float_format)) 314 else: 315 return type_checkers.ToShortestFloat(value) 316 317 return value 318 319 def _AnyMessageToJsonObject(self, message): 320 """Converts Any message according to Proto3 JSON Specification.""" 321 if not message.ListFields(): 322 return {} 323 # Must print @type first, use OrderedDict instead of {} 324 js = OrderedDict() 325 type_url = message.type_url 326 js['@type'] = type_url 327 sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool) 328 sub_message.ParseFromString(message.value) 329 message_descriptor = sub_message.DESCRIPTOR 330 full_name = message_descriptor.full_name 331 if _IsWrapperMessage(message_descriptor): 332 js['value'] = self._WrapperMessageToJsonObject(sub_message) 333 return js 334 if full_name in _WKTJSONMETHODS: 335 js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], 336 sub_message)(self) 337 return js 338 return self._RegularMessageToJsonObject(sub_message, js) 339 340 def _GenericMessageToJsonObject(self, message): 341 """Converts message according to Proto3 JSON Specification.""" 342 # Duration, Timestamp and FieldMask have ToJsonString method to do the 343 # convert. Users can also call the method directly. 344 return message.ToJsonString() 345 346 def _ValueMessageToJsonObject(self, message): 347 """Converts Value message according to Proto3 JSON Specification.""" 348 which = message.WhichOneof('kind') 349 # If the Value message is not set treat as null_value when serialize 350 # to JSON. The parse back result will be different from original message. 351 if which is None or which == 'null_value': 352 return None 353 if which == 'list_value': 354 return self._ListValueMessageToJsonObject(message.list_value) 355 if which == 'struct_value': 356 value = message.struct_value 357 else: 358 value = getattr(message, which) 359 oneof_descriptor = message.DESCRIPTOR.fields_by_name[which] 360 return self._FieldToJsonObject(oneof_descriptor, value) 361 362 def _ListValueMessageToJsonObject(self, message): 363 """Converts ListValue message according to Proto3 JSON Specification.""" 364 return [self._ValueMessageToJsonObject(value) 365 for value in message.values] 366 367 def _StructMessageToJsonObject(self, message): 368 """Converts Struct message according to Proto3 JSON Specification.""" 369 fields = message.fields 370 ret = {} 371 for key in fields: 372 ret[key] = self._ValueMessageToJsonObject(fields[key]) 373 return ret 374 375 def _WrapperMessageToJsonObject(self, message): 376 return self._FieldToJsonObject( 377 message.DESCRIPTOR.fields_by_name['value'], message.value) 378 379 380def _IsWrapperMessage(message_descriptor): 381 return message_descriptor.file.name == 'google/protobuf/wrappers.proto' 382 383 384def _DuplicateChecker(js): 385 result = {} 386 for name, value in js: 387 if name in result: 388 raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name)) 389 result[name] = value 390 return result 391 392 393def _CreateMessageFromTypeUrl(type_url, descriptor_pool): 394 """Creates a message from a type URL.""" 395 db = symbol_database.Default() 396 pool = db.pool if descriptor_pool is None else descriptor_pool 397 type_name = type_url.split('/')[-1] 398 try: 399 message_descriptor = pool.FindMessageTypeByName(type_name) 400 except KeyError: 401 raise TypeError( 402 'Can not find message descriptor by type_url: {0}'.format(type_url)) 403 message_class = db.GetPrototype(message_descriptor) 404 return message_class() 405 406 407def Parse(text, 408 message, 409 ignore_unknown_fields=False, 410 descriptor_pool=None, 411 max_recursion_depth=100): 412 """Parses a JSON representation of a protocol message into a message. 413 414 Args: 415 text: Message JSON representation. 416 message: A protocol buffer message to merge into. 417 ignore_unknown_fields: If True, do not raise errors for unknown fields. 418 descriptor_pool: A Descriptor Pool for resolving types. If None use the 419 default. 420 max_recursion_depth: max recursion depth of JSON message to be 421 deserialized. JSON messages over this depth will fail to be 422 deserialized. Default value is 100. 423 424 Returns: 425 The same message passed as argument. 426 427 Raises:: 428 ParseError: On JSON parsing problems. 429 """ 430 if not isinstance(text, str): 431 text = text.decode('utf-8') 432 try: 433 js = json.loads(text, object_pairs_hook=_DuplicateChecker) 434 except ValueError as e: 435 raise ParseError('Failed to load JSON: {0}.'.format(str(e))) 436 return ParseDict(js, message, ignore_unknown_fields, descriptor_pool, 437 max_recursion_depth) 438 439 440def ParseDict(js_dict, 441 message, 442 ignore_unknown_fields=False, 443 descriptor_pool=None, 444 max_recursion_depth=100): 445 """Parses a JSON dictionary representation into a message. 446 447 Args: 448 js_dict: Dict representation of a JSON message. 449 message: A protocol buffer message to merge into. 450 ignore_unknown_fields: If True, do not raise errors for unknown fields. 451 descriptor_pool: A Descriptor Pool for resolving types. If None use the 452 default. 453 max_recursion_depth: max recursion depth of JSON message to be 454 deserialized. JSON messages over this depth will fail to be 455 deserialized. Default value is 100. 456 457 Returns: 458 The same message passed as argument. 459 """ 460 parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth) 461 parser.ConvertMessage(js_dict, message, '') 462 return message 463 464 465_INT_OR_FLOAT = (int, float) 466 467 468class _Parser(object): 469 """JSON format parser for protocol message.""" 470 471 def __init__(self, ignore_unknown_fields, descriptor_pool, 472 max_recursion_depth): 473 self.ignore_unknown_fields = ignore_unknown_fields 474 self.descriptor_pool = descriptor_pool 475 self.max_recursion_depth = max_recursion_depth 476 self.recursion_depth = 0 477 478 def ConvertMessage(self, value, message, path): 479 """Convert a JSON object into a message. 480 481 Args: 482 value: A JSON object. 483 message: A WKT or regular protocol message to record the data. 484 path: parent path to log parse error info. 485 486 Raises: 487 ParseError: In case of convert problems. 488 """ 489 self.recursion_depth += 1 490 if self.recursion_depth > self.max_recursion_depth: 491 raise ParseError('Message too deep. Max recursion depth is {0}'.format( 492 self.max_recursion_depth)) 493 message_descriptor = message.DESCRIPTOR 494 full_name = message_descriptor.full_name 495 if not path: 496 path = message_descriptor.name 497 if _IsWrapperMessage(message_descriptor): 498 self._ConvertWrapperMessage(value, message, path) 499 elif full_name in _WKTJSONMETHODS: 500 methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self) 501 else: 502 self._ConvertFieldValuePair(value, message, path) 503 self.recursion_depth -= 1 504 505 def _ConvertFieldValuePair(self, js, message, path): 506 """Convert field value pairs into regular message. 507 508 Args: 509 js: A JSON object to convert the field value pairs. 510 message: A regular protocol message to record the data. 511 path: parent path to log parse error info. 512 513 Raises: 514 ParseError: In case of problems converting. 515 """ 516 names = [] 517 message_descriptor = message.DESCRIPTOR 518 fields_by_json_name = dict((f.json_name, f) 519 for f in message_descriptor.fields) 520 for name in js: 521 try: 522 field = fields_by_json_name.get(name, None) 523 if not field: 524 field = message_descriptor.fields_by_name.get(name, None) 525 if not field and _VALID_EXTENSION_NAME.match(name): 526 if not message_descriptor.is_extendable: 527 raise ParseError( 528 'Message type {0} does not have extensions at {1}'.format( 529 message_descriptor.full_name, path)) 530 identifier = name[1:-1] # strip [] brackets 531 # pylint: disable=protected-access 532 field = message.Extensions._FindExtensionByName(identifier) 533 # pylint: enable=protected-access 534 if not field: 535 # Try looking for extension by the message type name, dropping the 536 # field name following the final . separator in full_name. 537 identifier = '.'.join(identifier.split('.')[:-1]) 538 # pylint: disable=protected-access 539 field = message.Extensions._FindExtensionByName(identifier) 540 # pylint: enable=protected-access 541 if not field: 542 if self.ignore_unknown_fields: 543 continue 544 raise ParseError( 545 ('Message type "{0}" has no field named "{1}" at "{2}".\n' 546 ' Available Fields(except extensions): "{3}"').format( 547 message_descriptor.full_name, name, path, 548 [f.json_name for f in message_descriptor.fields])) 549 if name in names: 550 raise ParseError('Message type "{0}" should not have multiple ' 551 '"{1}" fields at "{2}".'.format( 552 message.DESCRIPTOR.full_name, name, path)) 553 names.append(name) 554 value = js[name] 555 # Check no other oneof field is parsed. 556 if field.containing_oneof is not None and value is not None: 557 oneof_name = field.containing_oneof.name 558 if oneof_name in names: 559 raise ParseError('Message type "{0}" should not have multiple ' 560 '"{1}" oneof fields at "{2}".'.format( 561 message.DESCRIPTOR.full_name, oneof_name, 562 path)) 563 names.append(oneof_name) 564 565 if value is None: 566 if (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE 567 and field.message_type.full_name == 'google.protobuf.Value'): 568 sub_message = getattr(message, field.name) 569 sub_message.null_value = 0 570 elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM 571 and field.enum_type.full_name == 'google.protobuf.NullValue'): 572 setattr(message, field.name, 0) 573 else: 574 message.ClearField(field.name) 575 continue 576 577 # Parse field value. 578 if _IsMapEntry(field): 579 message.ClearField(field.name) 580 self._ConvertMapFieldValue(value, message, field, 581 '{0}.{1}'.format(path, name)) 582 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: 583 message.ClearField(field.name) 584 if not isinstance(value, list): 585 raise ParseError('repeated field {0} must be in [] which is ' 586 '{1} at {2}'.format(name, value, path)) 587 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 588 # Repeated message field. 589 for index, item in enumerate(value): 590 sub_message = getattr(message, field.name).add() 591 # None is a null_value in Value. 592 if (item is None and 593 sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'): 594 raise ParseError('null is not allowed to be used as an element' 595 ' in a repeated field at {0}.{1}[{2}]'.format( 596 path, name, index)) 597 self.ConvertMessage(item, sub_message, 598 '{0}.{1}[{2}]'.format(path, name, index)) 599 else: 600 # Repeated scalar field. 601 for index, item in enumerate(value): 602 if item is None: 603 raise ParseError('null is not allowed to be used as an element' 604 ' in a repeated field at {0}.{1}[{2}]'.format( 605 path, name, index)) 606 getattr(message, field.name).append( 607 _ConvertScalarFieldValue( 608 item, field, '{0}.{1}[{2}]'.format(path, name, index))) 609 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 610 if field.is_extension: 611 sub_message = message.Extensions[field] 612 else: 613 sub_message = getattr(message, field.name) 614 sub_message.SetInParent() 615 self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name)) 616 else: 617 if field.is_extension: 618 message.Extensions[field] = _ConvertScalarFieldValue( 619 value, field, '{0}.{1}'.format(path, name)) 620 else: 621 setattr( 622 message, field.name, 623 _ConvertScalarFieldValue(value, field, 624 '{0}.{1}'.format(path, name))) 625 except ParseError as e: 626 if field and field.containing_oneof is None: 627 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) 628 else: 629 raise ParseError(str(e)) 630 except ValueError as e: 631 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) 632 except TypeError as e: 633 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) 634 635 def _ConvertAnyMessage(self, value, message, path): 636 """Convert a JSON representation into Any message.""" 637 if isinstance(value, dict) and not value: 638 return 639 try: 640 type_url = value['@type'] 641 except KeyError: 642 raise ParseError( 643 '@type is missing when parsing any message at {0}'.format(path)) 644 645 try: 646 sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool) 647 except TypeError as e: 648 raise ParseError('{0} at {1}'.format(e, path)) 649 message_descriptor = sub_message.DESCRIPTOR 650 full_name = message_descriptor.full_name 651 if _IsWrapperMessage(message_descriptor): 652 self._ConvertWrapperMessage(value['value'], sub_message, 653 '{0}.value'.format(path)) 654 elif full_name in _WKTJSONMETHODS: 655 methodcaller(_WKTJSONMETHODS[full_name][1], value['value'], sub_message, 656 '{0}.value'.format(path))( 657 self) 658 else: 659 del value['@type'] 660 self._ConvertFieldValuePair(value, sub_message, path) 661 value['@type'] = type_url 662 # Sets Any message 663 message.value = sub_message.SerializeToString() 664 message.type_url = type_url 665 666 def _ConvertGenericMessage(self, value, message, path): 667 """Convert a JSON representation into message with FromJsonString.""" 668 # Duration, Timestamp, FieldMask have a FromJsonString method to do the 669 # conversion. Users can also call the method directly. 670 try: 671 message.FromJsonString(value) 672 except ValueError as e: 673 raise ParseError('{0} at {1}'.format(e, path)) 674 675 def _ConvertValueMessage(self, value, message, path): 676 """Convert a JSON representation into Value message.""" 677 if isinstance(value, dict): 678 self._ConvertStructMessage(value, message.struct_value, path) 679 elif isinstance(value, list): 680 self._ConvertListValueMessage(value, message.list_value, path) 681 elif value is None: 682 message.null_value = 0 683 elif isinstance(value, bool): 684 message.bool_value = value 685 elif isinstance(value, str): 686 message.string_value = value 687 elif isinstance(value, _INT_OR_FLOAT): 688 message.number_value = value 689 else: 690 raise ParseError('Value {0} has unexpected type {1} at {2}'.format( 691 value, type(value), path)) 692 693 def _ConvertListValueMessage(self, value, message, path): 694 """Convert a JSON representation into ListValue message.""" 695 if not isinstance(value, list): 696 raise ParseError('ListValue must be in [] which is {0} at {1}'.format( 697 value, path)) 698 message.ClearField('values') 699 for index, item in enumerate(value): 700 self._ConvertValueMessage(item, message.values.add(), 701 '{0}[{1}]'.format(path, index)) 702 703 def _ConvertStructMessage(self, value, message, path): 704 """Convert a JSON representation into Struct message.""" 705 if not isinstance(value, dict): 706 raise ParseError('Struct must be in a dict which is {0} at {1}'.format( 707 value, path)) 708 # Clear will mark the struct as modified so it will be created even if 709 # there are no values. 710 message.Clear() 711 for key in value: 712 self._ConvertValueMessage(value[key], message.fields[key], 713 '{0}.{1}'.format(path, key)) 714 return 715 716 def _ConvertWrapperMessage(self, value, message, path): 717 """Convert a JSON representation into Wrapper message.""" 718 field = message.DESCRIPTOR.fields_by_name['value'] 719 setattr( 720 message, 'value', 721 _ConvertScalarFieldValue(value, field, path='{0}.value'.format(path))) 722 723 def _ConvertMapFieldValue(self, value, message, field, path): 724 """Convert map field value for a message map field. 725 726 Args: 727 value: A JSON object to convert the map field value. 728 message: A protocol message to record the converted data. 729 field: The descriptor of the map field to be converted. 730 path: parent path to log parse error info. 731 732 Raises: 733 ParseError: In case of convert problems. 734 """ 735 if not isinstance(value, dict): 736 raise ParseError( 737 'Map field {0} must be in a dict which is {1} at {2}'.format( 738 field.name, value, path)) 739 key_field = field.message_type.fields_by_name['key'] 740 value_field = field.message_type.fields_by_name['value'] 741 for key in value: 742 key_value = _ConvertScalarFieldValue(key, key_field, 743 '{0}.key'.format(path), True) 744 if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 745 self.ConvertMessage(value[key], 746 getattr(message, field.name)[key_value], 747 '{0}[{1}]'.format(path, key_value)) 748 else: 749 getattr(message, field.name)[key_value] = _ConvertScalarFieldValue( 750 value[key], value_field, path='{0}[{1}]'.format(path, key_value)) 751 752 753def _ConvertScalarFieldValue(value, field, path, require_str=False): 754 """Convert a single scalar field value. 755 756 Args: 757 value: A scalar value to convert the scalar field value. 758 field: The descriptor of the field to convert. 759 path: parent path to log parse error info. 760 require_str: If True, the field value must be a str. 761 762 Returns: 763 The converted scalar field value 764 765 Raises: 766 ParseError: In case of convert problems. 767 """ 768 try: 769 if field.cpp_type in _INT_TYPES: 770 return _ConvertInteger(value) 771 elif field.cpp_type in _FLOAT_TYPES: 772 return _ConvertFloat(value, field) 773 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: 774 return _ConvertBool(value, require_str) 775 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: 776 if field.type == descriptor.FieldDescriptor.TYPE_BYTES: 777 if isinstance(value, str): 778 encoded = value.encode('utf-8') 779 else: 780 encoded = value 781 # Add extra padding '=' 782 padded_value = encoded + b'=' * (4 - len(encoded) % 4) 783 return base64.urlsafe_b64decode(padded_value) 784 else: 785 # Checking for unpaired surrogates appears to be unreliable, 786 # depending on the specific Python version, so we check manually. 787 if _UNPAIRED_SURROGATE_PATTERN.search(value): 788 raise ParseError('Unpaired surrogate') 789 return value 790 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: 791 # Convert an enum value. 792 enum_value = field.enum_type.values_by_name.get(value, None) 793 if enum_value is None: 794 try: 795 number = int(value) 796 enum_value = field.enum_type.values_by_number.get(number, None) 797 except ValueError: 798 raise ParseError('Invalid enum value {0} for enum type {1}'.format( 799 value, field.enum_type.full_name)) 800 if enum_value is None: 801 if field.file.syntax == 'proto3': 802 # Proto3 accepts unknown enums. 803 return number 804 raise ParseError('Invalid enum value {0} for enum type {1}'.format( 805 value, field.enum_type.full_name)) 806 return enum_value.number 807 except ParseError as e: 808 raise ParseError('{0} at {1}'.format(e, path)) 809 810 811def _ConvertInteger(value): 812 """Convert an integer. 813 814 Args: 815 value: A scalar value to convert. 816 817 Returns: 818 The integer value. 819 820 Raises: 821 ParseError: If an integer couldn't be consumed. 822 """ 823 if isinstance(value, float) and not value.is_integer(): 824 raise ParseError('Couldn\'t parse integer: {0}'.format(value)) 825 826 if isinstance(value, str) and value.find(' ') != -1: 827 raise ParseError('Couldn\'t parse integer: "{0}"'.format(value)) 828 829 if isinstance(value, bool): 830 raise ParseError('Bool value {0} is not acceptable for ' 831 'integer field'.format(value)) 832 833 return int(value) 834 835 836def _ConvertFloat(value, field): 837 """Convert an floating point number.""" 838 if isinstance(value, float): 839 if math.isnan(value): 840 raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead') 841 if math.isinf(value): 842 if value > 0: 843 raise ParseError('Couldn\'t parse Infinity or value too large, ' 844 'use quoted "Infinity" instead') 845 else: 846 raise ParseError('Couldn\'t parse -Infinity or value too small, ' 847 'use quoted "-Infinity" instead') 848 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT: 849 # pylint: disable=protected-access 850 if value > type_checkers._FLOAT_MAX: 851 raise ParseError('Float value too large') 852 # pylint: disable=protected-access 853 if value < type_checkers._FLOAT_MIN: 854 raise ParseError('Float value too small') 855 if value == 'nan': 856 raise ParseError('Couldn\'t parse float "nan", use "NaN" instead') 857 try: 858 # Assume Python compatible syntax. 859 return float(value) 860 except ValueError: 861 # Check alternative spellings. 862 if value == _NEG_INFINITY: 863 return float('-inf') 864 elif value == _INFINITY: 865 return float('inf') 866 elif value == _NAN: 867 return float('nan') 868 else: 869 raise ParseError('Couldn\'t parse float: {0}'.format(value)) 870 871 872def _ConvertBool(value, require_str): 873 """Convert a boolean value. 874 875 Args: 876 value: A scalar value to convert. 877 require_str: If True, value must be a str. 878 879 Returns: 880 The bool parsed. 881 882 Raises: 883 ParseError: If a boolean value couldn't be consumed. 884 """ 885 if require_str: 886 if value == 'true': 887 return True 888 elif value == 'false': 889 return False 890 else: 891 raise ParseError('Expected "true" or "false", not {0}'.format(value)) 892 893 if not isinstance(value, bool): 894 raise ParseError('Expected true or false without quotes') 895 return value 896 897_WKTJSONMETHODS = { 898 'google.protobuf.Any': ['_AnyMessageToJsonObject', 899 '_ConvertAnyMessage'], 900 'google.protobuf.Duration': ['_GenericMessageToJsonObject', 901 '_ConvertGenericMessage'], 902 'google.protobuf.FieldMask': ['_GenericMessageToJsonObject', 903 '_ConvertGenericMessage'], 904 'google.protobuf.ListValue': ['_ListValueMessageToJsonObject', 905 '_ConvertListValueMessage'], 906 'google.protobuf.Struct': ['_StructMessageToJsonObject', 907 '_ConvertStructMessage'], 908 'google.protobuf.Timestamp': ['_GenericMessageToJsonObject', 909 '_ConvertGenericMessage'], 910 'google.protobuf.Value': ['_ValueMessageToJsonObject', 911 '_ConvertValueMessage'] 912} 913