1# Protocol Buffers - Google's data interchange format 2# Copyright 2008 Google Inc. All rights reserved. 3# https://developers.google.com/protocol-buffers/ 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31"""Contains routines for printing protocol messages in JSON format. 32 33Simple usage example: 34 35 # Create a proto object and serialize it to a json format string. 36 message = my_proto_pb2.MyMessage(foo='bar') 37 json_string = json_format.MessageToJson(message) 38 39 # Parse a json format string to proto object. 40 message = json_format.Parse(json_string, my_proto_pb2.MyMessage()) 41""" 42 43__author__ = 'jieluo@google.com (Jie Luo)' 44 45import base64 46import json 47import math 48import six 49import sys 50 51from google.protobuf import descriptor 52from google.protobuf import symbol_database 53 54_TIMESTAMPFOMAT = '%Y-%m-%dT%H:%M:%S' 55_INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32, 56 descriptor.FieldDescriptor.CPPTYPE_UINT32, 57 descriptor.FieldDescriptor.CPPTYPE_INT64, 58 descriptor.FieldDescriptor.CPPTYPE_UINT64]) 59_INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64, 60 descriptor.FieldDescriptor.CPPTYPE_UINT64]) 61_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, 62 descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) 63_INFINITY = 'Infinity' 64_NEG_INFINITY = '-Infinity' 65_NAN = 'NaN' 66 67 68class Error(Exception): 69 """Top-level module error for json_format.""" 70 71 72class SerializeToJsonError(Error): 73 """Thrown if serialization to JSON fails.""" 74 75 76class ParseError(Error): 77 """Thrown in case of parsing error.""" 78 79 80def MessageToJson(message, including_default_value_fields=False): 81 """Converts protobuf message to JSON format. 82 83 Args: 84 message: The protocol buffers message instance to serialize. 85 including_default_value_fields: If True, singular primitive fields, 86 repeated fields, and map fields will always be serialized. If 87 False, only serialize non-empty fields. Singular message fields 88 and oneof fields are not affected by this option. 89 90 Returns: 91 A string containing the JSON formatted protocol buffer message. 92 """ 93 js = _MessageToJsonObject(message, including_default_value_fields) 94 return json.dumps(js, indent=2) 95 96 97def _MessageToJsonObject(message, including_default_value_fields): 98 """Converts message to an object according to Proto3 JSON Specification.""" 99 message_descriptor = message.DESCRIPTOR 100 full_name = message_descriptor.full_name 101 if _IsWrapperMessage(message_descriptor): 102 return _WrapperMessageToJsonObject(message) 103 if full_name in _WKTJSONMETHODS: 104 return _WKTJSONMETHODS[full_name][0]( 105 message, including_default_value_fields) 106 js = {} 107 return _RegularMessageToJsonObject( 108 message, js, including_default_value_fields) 109 110 111def _IsMapEntry(field): 112 return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and 113 field.message_type.has_options and 114 field.message_type.GetOptions().map_entry) 115 116 117def _RegularMessageToJsonObject(message, js, including_default_value_fields): 118 """Converts normal message according to Proto3 JSON Specification.""" 119 fields = message.ListFields() 120 include_default = including_default_value_fields 121 122 try: 123 for field, value in fields: 124 name = field.camelcase_name 125 if _IsMapEntry(field): 126 # Convert a map field. 127 v_field = field.message_type.fields_by_name['value'] 128 js_map = {} 129 for key in value: 130 if isinstance(key, bool): 131 if key: 132 recorded_key = 'true' 133 else: 134 recorded_key = 'false' 135 else: 136 recorded_key = key 137 js_map[recorded_key] = _FieldToJsonObject( 138 v_field, value[key], including_default_value_fields) 139 js[name] = js_map 140 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: 141 # Convert a repeated field. 142 js[name] = [_FieldToJsonObject(field, k, include_default) 143 for k in value] 144 else: 145 js[name] = _FieldToJsonObject(field, value, include_default) 146 147 # Serialize default value if including_default_value_fields is True. 148 if including_default_value_fields: 149 message_descriptor = message.DESCRIPTOR 150 for field in message_descriptor.fields: 151 # Singular message fields and oneof fields will not be affected. 152 if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and 153 field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or 154 field.containing_oneof): 155 continue 156 name = field.camelcase_name 157 if name in js: 158 # Skip the field which has been serailized already. 159 continue 160 if _IsMapEntry(field): 161 js[name] = {} 162 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: 163 js[name] = [] 164 else: 165 js[name] = _FieldToJsonObject(field, field.default_value) 166 167 except ValueError as e: 168 raise SerializeToJsonError( 169 'Failed to serialize {0} field: {1}.'.format(field.name, e)) 170 171 return js 172 173 174def _FieldToJsonObject( 175 field, value, including_default_value_fields=False): 176 """Converts field value according to Proto3 JSON Specification.""" 177 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 178 return _MessageToJsonObject(value, including_default_value_fields) 179 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: 180 enum_value = field.enum_type.values_by_number.get(value, None) 181 if enum_value is not None: 182 return enum_value.name 183 else: 184 raise SerializeToJsonError('Enum field contains an integer value ' 185 'which can not mapped to an enum value.') 186 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: 187 if field.type == descriptor.FieldDescriptor.TYPE_BYTES: 188 # Use base64 Data encoding for bytes 189 return base64.b64encode(value).decode('utf-8') 190 else: 191 return value 192 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: 193 return bool(value) 194 elif field.cpp_type in _INT64_TYPES: 195 return str(value) 196 elif field.cpp_type in _FLOAT_TYPES: 197 if math.isinf(value): 198 if value < 0.0: 199 return _NEG_INFINITY 200 else: 201 return _INFINITY 202 if math.isnan(value): 203 return _NAN 204 return value 205 206 207def _AnyMessageToJsonObject(message, including_default): 208 """Converts Any message according to Proto3 JSON Specification.""" 209 if not message.ListFields(): 210 return {} 211 js = {} 212 type_url = message.type_url 213 js['@type'] = type_url 214 sub_message = _CreateMessageFromTypeUrl(type_url) 215 sub_message.ParseFromString(message.value) 216 message_descriptor = sub_message.DESCRIPTOR 217 full_name = message_descriptor.full_name 218 if _IsWrapperMessage(message_descriptor): 219 js['value'] = _WrapperMessageToJsonObject(sub_message) 220 return js 221 if full_name in _WKTJSONMETHODS: 222 js['value'] = _WKTJSONMETHODS[full_name][0](sub_message, including_default) 223 return js 224 return _RegularMessageToJsonObject(sub_message, js, including_default) 225 226 227def _CreateMessageFromTypeUrl(type_url): 228 # TODO(jieluo): Should add a way that users can register the type resolver 229 # instead of the default one. 230 db = symbol_database.Default() 231 type_name = type_url.split('/')[-1] 232 try: 233 message_descriptor = db.pool.FindMessageTypeByName(type_name) 234 except KeyError: 235 raise TypeError( 236 'Can not find message descriptor by type_url: {0}.'.format(type_url)) 237 message_class = db.GetPrototype(message_descriptor) 238 return message_class() 239 240 241def _GenericMessageToJsonObject(message, unused_including_default): 242 """Converts message by ToJsonString according to Proto3 JSON Specification.""" 243 # Duration, Timestamp and FieldMask have ToJsonString method to do the 244 # convert. Users can also call the method directly. 245 return message.ToJsonString() 246 247 248def _ValueMessageToJsonObject(message, unused_including_default=False): 249 """Converts Value message according to Proto3 JSON Specification.""" 250 which = message.WhichOneof('kind') 251 # If the Value message is not set treat as null_value when serialize 252 # to JSON. The parse back result will be different from original message. 253 if which is None or which == 'null_value': 254 return None 255 if which == 'list_value': 256 return _ListValueMessageToJsonObject(message.list_value) 257 if which == 'struct_value': 258 value = message.struct_value 259 else: 260 value = getattr(message, which) 261 oneof_descriptor = message.DESCRIPTOR.fields_by_name[which] 262 return _FieldToJsonObject(oneof_descriptor, value) 263 264 265def _ListValueMessageToJsonObject(message, unused_including_default=False): 266 """Converts ListValue message according to Proto3 JSON Specification.""" 267 return [_ValueMessageToJsonObject(value) 268 for value in message.values] 269 270 271def _StructMessageToJsonObject(message, unused_including_default=False): 272 """Converts Struct message according to Proto3 JSON Specification.""" 273 fields = message.fields 274 ret = {} 275 for key in fields: 276 ret[key] = _ValueMessageToJsonObject(fields[key]) 277 return ret 278 279 280def _IsWrapperMessage(message_descriptor): 281 return message_descriptor.file.name == 'google/protobuf/wrappers.proto' 282 283 284def _WrapperMessageToJsonObject(message): 285 return _FieldToJsonObject( 286 message.DESCRIPTOR.fields_by_name['value'], message.value) 287 288 289def _DuplicateChecker(js): 290 result = {} 291 for name, value in js: 292 if name in result: 293 raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name)) 294 result[name] = value 295 return result 296 297 298def Parse(text, message): 299 """Parses a JSON representation of a protocol message into a message. 300 301 Args: 302 text: Message JSON representation. 303 message: A protocol beffer message to merge into. 304 305 Returns: 306 The same message passed as argument. 307 308 Raises:: 309 ParseError: On JSON parsing problems. 310 """ 311 if not isinstance(text, six.text_type): text = text.decode('utf-8') 312 try: 313 if sys.version_info < (2, 7): 314 # object_pair_hook is not supported before python2.7 315 js = json.loads(text) 316 else: 317 js = json.loads(text, object_pairs_hook=_DuplicateChecker) 318 except ValueError as e: 319 raise ParseError('Failed to load JSON: {0}.'.format(str(e))) 320 _ConvertMessage(js, message) 321 return message 322 323 324def _ConvertFieldValuePair(js, message): 325 """Convert field value pairs into regular message. 326 327 Args: 328 js: A JSON object to convert the field value pairs. 329 message: A regular protocol message to record the data. 330 331 Raises: 332 ParseError: In case of problems converting. 333 """ 334 names = [] 335 message_descriptor = message.DESCRIPTOR 336 for name in js: 337 try: 338 field = message_descriptor.fields_by_camelcase_name.get(name, None) 339 if not field: 340 raise ParseError( 341 'Message type "{0}" has no field named "{1}".'.format( 342 message_descriptor.full_name, name)) 343 if name in names: 344 raise ParseError( 345 'Message type "{0}" should not have multiple "{1}" fields.'.format( 346 message.DESCRIPTOR.full_name, name)) 347 names.append(name) 348 # Check no other oneof field is parsed. 349 if field.containing_oneof is not None: 350 oneof_name = field.containing_oneof.name 351 if oneof_name in names: 352 raise ParseError('Message type "{0}" should not have multiple "{1}" ' 353 'oneof fields.'.format( 354 message.DESCRIPTOR.full_name, oneof_name)) 355 names.append(oneof_name) 356 357 value = js[name] 358 if value is None: 359 message.ClearField(field.name) 360 continue 361 362 # Parse field value. 363 if _IsMapEntry(field): 364 message.ClearField(field.name) 365 _ConvertMapFieldValue(value, message, field) 366 elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: 367 message.ClearField(field.name) 368 if not isinstance(value, list): 369 raise ParseError('repeated field {0} must be in [] which is ' 370 '{1}.'.format(name, value)) 371 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 372 # Repeated message field. 373 for item in value: 374 sub_message = getattr(message, field.name).add() 375 # None is a null_value in Value. 376 if (item is None and 377 sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'): 378 raise ParseError('null is not allowed to be used as an element' 379 ' in a repeated field.') 380 _ConvertMessage(item, sub_message) 381 else: 382 # Repeated scalar field. 383 for item in value: 384 if item is None: 385 raise ParseError('null is not allowed to be used as an element' 386 ' in a repeated field.') 387 getattr(message, field.name).append( 388 _ConvertScalarFieldValue(item, field)) 389 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 390 sub_message = getattr(message, field.name) 391 _ConvertMessage(value, sub_message) 392 else: 393 setattr(message, field.name, _ConvertScalarFieldValue(value, field)) 394 except ParseError as e: 395 if field and field.containing_oneof is None: 396 raise ParseError('Failed to parse {0} field: {1}'.format(name, e)) 397 else: 398 raise ParseError(str(e)) 399 except ValueError as e: 400 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) 401 except TypeError as e: 402 raise ParseError('Failed to parse {0} field: {1}.'.format(name, e)) 403 404 405def _ConvertMessage(value, message): 406 """Convert a JSON object into a message. 407 408 Args: 409 value: A JSON object. 410 message: A WKT or regular protocol message to record the data. 411 412 Raises: 413 ParseError: In case of convert problems. 414 """ 415 message_descriptor = message.DESCRIPTOR 416 full_name = message_descriptor.full_name 417 if _IsWrapperMessage(message_descriptor): 418 _ConvertWrapperMessage(value, message) 419 elif full_name in _WKTJSONMETHODS: 420 _WKTJSONMETHODS[full_name][1](value, message) 421 else: 422 _ConvertFieldValuePair(value, message) 423 424 425def _ConvertAnyMessage(value, message): 426 """Convert a JSON representation into Any message.""" 427 if isinstance(value, dict) and not value: 428 return 429 try: 430 type_url = value['@type'] 431 except KeyError: 432 raise ParseError('@type is missing when parsing any message.') 433 434 sub_message = _CreateMessageFromTypeUrl(type_url) 435 message_descriptor = sub_message.DESCRIPTOR 436 full_name = message_descriptor.full_name 437 if _IsWrapperMessage(message_descriptor): 438 _ConvertWrapperMessage(value['value'], sub_message) 439 elif full_name in _WKTJSONMETHODS: 440 _WKTJSONMETHODS[full_name][1](value['value'], sub_message) 441 else: 442 del value['@type'] 443 _ConvertFieldValuePair(value, sub_message) 444 # Sets Any message 445 message.value = sub_message.SerializeToString() 446 message.type_url = type_url 447 448 449def _ConvertGenericMessage(value, message): 450 """Convert a JSON representation into message with FromJsonString.""" 451 # Durantion, Timestamp, FieldMask have FromJsonString method to do the 452 # convert. Users can also call the method directly. 453 message.FromJsonString(value) 454 455 456_INT_OR_FLOAT = six.integer_types + (float,) 457 458 459def _ConvertValueMessage(value, message): 460 """Convert a JSON representation into Value message.""" 461 if isinstance(value, dict): 462 _ConvertStructMessage(value, message.struct_value) 463 elif isinstance(value, list): 464 _ConvertListValueMessage(value, message.list_value) 465 elif value is None: 466 message.null_value = 0 467 elif isinstance(value, bool): 468 message.bool_value = value 469 elif isinstance(value, six.string_types): 470 message.string_value = value 471 elif isinstance(value, _INT_OR_FLOAT): 472 message.number_value = value 473 else: 474 raise ParseError('Unexpected type for Value message.') 475 476 477def _ConvertListValueMessage(value, message): 478 """Convert a JSON representation into ListValue message.""" 479 if not isinstance(value, list): 480 raise ParseError( 481 'ListValue must be in [] which is {0}.'.format(value)) 482 message.ClearField('values') 483 for item in value: 484 _ConvertValueMessage(item, message.values.add()) 485 486 487def _ConvertStructMessage(value, message): 488 """Convert a JSON representation into Struct message.""" 489 if not isinstance(value, dict): 490 raise ParseError( 491 'Struct must be in a dict which is {0}.'.format(value)) 492 for key in value: 493 _ConvertValueMessage(value[key], message.fields[key]) 494 return 495 496 497def _ConvertWrapperMessage(value, message): 498 """Convert a JSON representation into Wrapper message.""" 499 field = message.DESCRIPTOR.fields_by_name['value'] 500 setattr(message, 'value', _ConvertScalarFieldValue(value, field)) 501 502 503def _ConvertMapFieldValue(value, message, field): 504 """Convert map field value for a message map field. 505 506 Args: 507 value: A JSON object to convert the map field value. 508 message: A protocol message to record the converted data. 509 field: The descriptor of the map field to be converted. 510 511 Raises: 512 ParseError: In case of convert problems. 513 """ 514 if not isinstance(value, dict): 515 raise ParseError( 516 'Map field {0} must be in a dict which is {1}.'.format( 517 field.name, value)) 518 key_field = field.message_type.fields_by_name['key'] 519 value_field = field.message_type.fields_by_name['value'] 520 for key in value: 521 key_value = _ConvertScalarFieldValue(key, key_field, True) 522 if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: 523 _ConvertMessage(value[key], getattr(message, field.name)[key_value]) 524 else: 525 getattr(message, field.name)[key_value] = _ConvertScalarFieldValue( 526 value[key], value_field) 527 528 529def _ConvertScalarFieldValue(value, field, require_str=False): 530 """Convert a single scalar field value. 531 532 Args: 533 value: A scalar value to convert the scalar field value. 534 field: The descriptor of the field to convert. 535 require_str: If True, the field value must be a str. 536 537 Returns: 538 The converted scalar field value 539 540 Raises: 541 ParseError: In case of convert problems. 542 """ 543 if field.cpp_type in _INT_TYPES: 544 return _ConvertInteger(value) 545 elif field.cpp_type in _FLOAT_TYPES: 546 return _ConvertFloat(value) 547 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: 548 return _ConvertBool(value, require_str) 549 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: 550 if field.type == descriptor.FieldDescriptor.TYPE_BYTES: 551 return base64.b64decode(value) 552 else: 553 return value 554 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: 555 # Convert an enum value. 556 enum_value = field.enum_type.values_by_name.get(value, None) 557 if enum_value is None: 558 raise ParseError( 559 'Enum value must be a string literal with double quotes. ' 560 'Type "{0}" has no value named {1}.'.format( 561 field.enum_type.full_name, value)) 562 return enum_value.number 563 564 565def _ConvertInteger(value): 566 """Convert an integer. 567 568 Args: 569 value: A scalar value to convert. 570 571 Returns: 572 The integer value. 573 574 Raises: 575 ParseError: If an integer couldn't be consumed. 576 """ 577 if isinstance(value, float): 578 raise ParseError('Couldn\'t parse integer: {0}.'.format(value)) 579 580 if isinstance(value, six.text_type) and value.find(' ') != -1: 581 raise ParseError('Couldn\'t parse integer: "{0}".'.format(value)) 582 583 return int(value) 584 585 586def _ConvertFloat(value): 587 """Convert an floating point number.""" 588 if value == 'nan': 589 raise ParseError('Couldn\'t parse float "nan", use "NaN" instead.') 590 try: 591 # Assume Python compatible syntax. 592 return float(value) 593 except ValueError: 594 # Check alternative spellings. 595 if value == _NEG_INFINITY: 596 return float('-inf') 597 elif value == _INFINITY: 598 return float('inf') 599 elif value == _NAN: 600 return float('nan') 601 else: 602 raise ParseError('Couldn\'t parse float: {0}.'.format(value)) 603 604 605def _ConvertBool(value, require_str): 606 """Convert a boolean value. 607 608 Args: 609 value: A scalar value to convert. 610 require_str: If True, value must be a str. 611 612 Returns: 613 The bool parsed. 614 615 Raises: 616 ParseError: If a boolean value couldn't be consumed. 617 """ 618 if require_str: 619 if value == 'true': 620 return True 621 elif value == 'false': 622 return False 623 else: 624 raise ParseError('Expected "true" or "false", not {0}.'.format(value)) 625 626 if not isinstance(value, bool): 627 raise ParseError('Expected true or false without quotes.') 628 return value 629 630_WKTJSONMETHODS = { 631 'google.protobuf.Any': [_AnyMessageToJsonObject, 632 _ConvertAnyMessage], 633 'google.protobuf.Duration': [_GenericMessageToJsonObject, 634 _ConvertGenericMessage], 635 'google.protobuf.FieldMask': [_GenericMessageToJsonObject, 636 _ConvertGenericMessage], 637 'google.protobuf.ListValue': [_ListValueMessageToJsonObject, 638 _ConvertListValueMessage], 639 'google.protobuf.Struct': [_StructMessageToJsonObject, 640 _ConvertStructMessage], 641 'google.protobuf.Timestamp': [_GenericMessageToJsonObject, 642 _ConvertGenericMessage], 643 'google.protobuf.Value': [_ValueMessageToJsonObject, 644 _ConvertValueMessage] 645} 646