1#!/usr/bin/env python 2# 3# Copyright 2010 Google Inc. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18"""Services descriptor definitions. 19 20Contains message definitions and functions for converting 21service classes into transmittable message format. 22 23Describing an Enum instance, Enum class, Field class or Message class will 24generate an appropriate descriptor object that describes that class. 25This message can itself be used to transmit information to clients wishing 26to know the description of an enum value, enum, field or message without 27needing to download the source code. This format is also compatible with 28other, non-Python languages. 29 30The descriptors are modeled to be binary compatible with: 31 32 http://code.google.com/p/protobuf/source/browse/trunk/src/google/protobuf/descriptor.proto 33 34NOTE: The names of types and fields are not always the same between these 35descriptors and the ones defined in descriptor.proto. This was done in order 36to make source code files that use these descriptors easier to read. For 37example, it is not necessary to prefix TYPE to all the values in 38FieldDescriptor.Variant as is done in descriptor.proto FieldDescriptorProto.Type. 39 40Example: 41 42 class Pixel(messages.Message): 43 44 x = messages.IntegerField(1, required=True) 45 y = messages.IntegerField(2, required=True) 46 47 color = messages.BytesField(3) 48 49 # Describe Pixel class using message descriptor. 50 fields = [] 51 52 field = FieldDescriptor() 53 field.name = 'x' 54 field.number = 1 55 field.label = FieldDescriptor.Label.REQUIRED 56 field.variant = FieldDescriptor.Variant.INT64 57 fields.append(field) 58 59 field = FieldDescriptor() 60 field.name = 'y' 61 field.number = 2 62 field.label = FieldDescriptor.Label.REQUIRED 63 field.variant = FieldDescriptor.Variant.INT64 64 fields.append(field) 65 66 field = FieldDescriptor() 67 field.name = 'color' 68 field.number = 3 69 field.label = FieldDescriptor.Label.OPTIONAL 70 field.variant = FieldDescriptor.Variant.BYTES 71 fields.append(field) 72 73 message = MessageDescriptor() 74 message.name = 'Pixel' 75 message.fields = fields 76 77 # Describing is the equivalent of building the above message. 78 message == describe_message(Pixel) 79 80Public Classes: 81 EnumValueDescriptor: Describes Enum values. 82 EnumDescriptor: Describes Enum classes. 83 FieldDescriptor: Describes field instances. 84 FileDescriptor: Describes a single 'file' unit. 85 FileSet: Describes a collection of file descriptors. 86 MessageDescriptor: Describes Message classes. 87 MethodDescriptor: Describes a method of a service. 88 ServiceDescriptor: Describes a services. 89 90Public Functions: 91 describe_enum_value: Describe an individual enum-value. 92 describe_enum: Describe an Enum class. 93 describe_field: Describe a Field definition. 94 describe_file: Describe a 'file' unit from a Python module or object. 95 describe_file_set: Describe a file set from a list of modules or objects. 96 describe_message: Describe a Message definition. 97 describe_method: Describe a Method definition. 98 describe_service: Describe a Service definition. 99""" 100import six 101 102__author__ = 'rafek@google.com (Rafe Kaplan)' 103 104import codecs 105import types 106 107from . import messages 108from . import util 109 110 111__all__ = ['EnumDescriptor', 112 'EnumValueDescriptor', 113 'FieldDescriptor', 114 'MessageDescriptor', 115 'MethodDescriptor', 116 'FileDescriptor', 117 'FileSet', 118 'ServiceDescriptor', 119 'DescriptorLibrary', 120 121 'describe_enum', 122 'describe_enum_value', 123 'describe_field', 124 'describe_message', 125 'describe_method', 126 'describe_file', 127 'describe_file_set', 128 'describe_service', 129 'describe', 130 'import_descriptor_loader', 131 ] 132 133 134# NOTE: MessageField is missing because message fields cannot have 135# a default value at this time. 136# TODO(rafek): Support default message values. 137# 138# Map to functions that convert default values of fields of a given type 139# to a string. The function must return a value that is compatible with 140# FieldDescriptor.default_value and therefore a unicode string. 141_DEFAULT_TO_STRING_MAP = { 142 messages.IntegerField: six.text_type, 143 messages.FloatField: six.text_type, 144 messages.BooleanField: lambda value: value and u'true' or u'false', 145 messages.BytesField: lambda value: codecs.escape_encode(value)[0], 146 messages.StringField: lambda value: value, 147 messages.EnumField: lambda value: six.text_type(value.number), 148} 149 150_DEFAULT_FROM_STRING_MAP = { 151 messages.IntegerField: int, 152 messages.FloatField: float, 153 messages.BooleanField: lambda value: value == u'true', 154 messages.BytesField: lambda value: codecs.escape_decode(value)[0], 155 messages.StringField: lambda value: value, 156 messages.EnumField: int, 157} 158 159 160class EnumValueDescriptor(messages.Message): 161 """Enum value descriptor. 162 163 Fields: 164 name: Name of enumeration value. 165 number: Number of enumeration value. 166 """ 167 168 # TODO(rafek): Why are these listed as optional in descriptor.proto. 169 # Harmonize? 170 name = messages.StringField(1, required=True) 171 number = messages.IntegerField(2, 172 required=True, 173 variant=messages.Variant.INT32) 174 175 176class EnumDescriptor(messages.Message): 177 """Enum class descriptor. 178 179 Fields: 180 name: Name of Enum without any qualification. 181 values: Values defined by Enum class. 182 """ 183 184 name = messages.StringField(1) 185 values = messages.MessageField(EnumValueDescriptor, 2, repeated=True) 186 187 188class FieldDescriptor(messages.Message): 189 """Field definition descriptor. 190 191 Enums: 192 Variant: Wire format hint sub-types for field. 193 Label: Values for optional, required and repeated fields. 194 195 Fields: 196 name: Name of field. 197 number: Number of field. 198 variant: Variant of field. 199 type_name: Type name for message and enum fields. 200 default_value: String representation of default value. 201 """ 202 203 Variant = messages.Variant 204 205 class Label(messages.Enum): 206 """Field label.""" 207 208 OPTIONAL = 1 209 REQUIRED = 2 210 REPEATED = 3 211 212 name = messages.StringField(1, required=True) 213 number = messages.IntegerField(3, 214 required=True, 215 variant=messages.Variant.INT32) 216 label = messages.EnumField(Label, 4, default=Label.OPTIONAL) 217 variant = messages.EnumField(Variant, 5) 218 type_name = messages.StringField(6) 219 220 # For numeric types, contains the original text representation of the value. 221 # For booleans, "true" or "false". 222 # For strings, contains the default text contents (not escaped in any way). 223 # For bytes, contains the C escaped value. All bytes < 128 are that are 224 # traditionally considered unprintable are also escaped. 225 default_value = messages.StringField(7) 226 227 228class MessageDescriptor(messages.Message): 229 """Message definition descriptor. 230 231 Fields: 232 name: Name of Message without any qualification. 233 fields: Fields defined for message. 234 message_types: Nested Message classes defined on message. 235 enum_types: Nested Enum classes defined on message. 236 """ 237 238 name = messages.StringField(1) 239 fields = messages.MessageField(FieldDescriptor, 2, repeated=True) 240 241 message_types = messages.MessageField( 242 'protorpc.descriptor.MessageDescriptor', 3, repeated=True) 243 enum_types = messages.MessageField(EnumDescriptor, 4, repeated=True) 244 245 246class MethodDescriptor(messages.Message): 247 """Service method definition descriptor. 248 249 Fields: 250 name: Name of service method. 251 request_type: Fully qualified or relative name of request message type. 252 response_type: Fully qualified or relative name of response message type. 253 """ 254 255 name = messages.StringField(1) 256 257 request_type = messages.StringField(2) 258 response_type = messages.StringField(3) 259 260 261class ServiceDescriptor(messages.Message): 262 """Service definition descriptor. 263 264 Fields: 265 name: Name of Service without any qualification. 266 methods: Remote methods of Service. 267 """ 268 269 name = messages.StringField(1) 270 271 methods = messages.MessageField(MethodDescriptor, 2, repeated=True) 272 273 274class FileDescriptor(messages.Message): 275 """Description of file containing protobuf definitions. 276 277 Fields: 278 package: Fully qualified name of package that definitions belong to. 279 message_types: Message definitions contained in file. 280 enum_types: Enum definitions contained in file. 281 service_types: Service definitions contained in file. 282 """ 283 284 package = messages.StringField(2) 285 286 # TODO(rafek): Add dependency field 287 288 message_types = messages.MessageField(MessageDescriptor, 4, repeated=True) 289 enum_types = messages.MessageField(EnumDescriptor, 5, repeated=True) 290 service_types = messages.MessageField(ServiceDescriptor, 6, repeated=True) 291 292 293class FileSet(messages.Message): 294 """A collection of FileDescriptors. 295 296 Fields: 297 files: Files in file-set. 298 """ 299 300 files = messages.MessageField(FileDescriptor, 1, repeated=True) 301 302 303def describe_enum_value(enum_value): 304 """Build descriptor for Enum instance. 305 306 Args: 307 enum_value: Enum value to provide descriptor for. 308 309 Returns: 310 Initialized EnumValueDescriptor instance describing the Enum instance. 311 """ 312 enum_value_descriptor = EnumValueDescriptor() 313 enum_value_descriptor.name = six.text_type(enum_value.name) 314 enum_value_descriptor.number = enum_value.number 315 return enum_value_descriptor 316 317 318def describe_enum(enum_definition): 319 """Build descriptor for Enum class. 320 321 Args: 322 enum_definition: Enum class to provide descriptor for. 323 324 Returns: 325 Initialized EnumDescriptor instance describing the Enum class. 326 """ 327 enum_descriptor = EnumDescriptor() 328 enum_descriptor.name = enum_definition.definition_name().split('.')[-1] 329 330 values = [] 331 for number in enum_definition.numbers(): 332 value = enum_definition.lookup_by_number(number) 333 values.append(describe_enum_value(value)) 334 335 if values: 336 enum_descriptor.values = values 337 338 return enum_descriptor 339 340 341def describe_field(field_definition): 342 """Build descriptor for Field instance. 343 344 Args: 345 field_definition: Field instance to provide descriptor for. 346 347 Returns: 348 Initialized FieldDescriptor instance describing the Field instance. 349 """ 350 field_descriptor = FieldDescriptor() 351 field_descriptor.name = field_definition.name 352 field_descriptor.number = field_definition.number 353 field_descriptor.variant = field_definition.variant 354 355 if isinstance(field_definition, messages.EnumField): 356 field_descriptor.type_name = field_definition.type.definition_name() 357 358 if isinstance(field_definition, messages.MessageField): 359 field_descriptor.type_name = field_definition.message_type.definition_name() 360 361 if field_definition.default is not None: 362 field_descriptor.default_value = _DEFAULT_TO_STRING_MAP[ 363 type(field_definition)](field_definition.default) 364 365 # Set label. 366 if field_definition.repeated: 367 field_descriptor.label = FieldDescriptor.Label.REPEATED 368 elif field_definition.required: 369 field_descriptor.label = FieldDescriptor.Label.REQUIRED 370 else: 371 field_descriptor.label = FieldDescriptor.Label.OPTIONAL 372 373 return field_descriptor 374 375 376def describe_message(message_definition): 377 """Build descriptor for Message class. 378 379 Args: 380 message_definition: Message class to provide descriptor for. 381 382 Returns: 383 Initialized MessageDescriptor instance describing the Message class. 384 """ 385 message_descriptor = MessageDescriptor() 386 message_descriptor.name = message_definition.definition_name().split('.')[-1] 387 388 fields = sorted(message_definition.all_fields(), 389 key=lambda v: v.number) 390 if fields: 391 message_descriptor.fields = [describe_field(field) for field in fields] 392 393 try: 394 nested_messages = message_definition.__messages__ 395 except AttributeError: 396 pass 397 else: 398 message_descriptors = [] 399 for name in nested_messages: 400 value = getattr(message_definition, name) 401 message_descriptors.append(describe_message(value)) 402 403 message_descriptor.message_types = message_descriptors 404 405 try: 406 nested_enums = message_definition.__enums__ 407 except AttributeError: 408 pass 409 else: 410 enum_descriptors = [] 411 for name in nested_enums: 412 value = getattr(message_definition, name) 413 enum_descriptors.append(describe_enum(value)) 414 415 message_descriptor.enum_types = enum_descriptors 416 417 return message_descriptor 418 419 420def describe_method(method): 421 """Build descriptor for service method. 422 423 Args: 424 method: Remote service method to describe. 425 426 Returns: 427 Initialized MethodDescriptor instance describing the service method. 428 """ 429 method_info = method.remote 430 descriptor = MethodDescriptor() 431 descriptor.name = method_info.method.__name__ 432 descriptor.request_type = method_info.request_type.definition_name() 433 descriptor.response_type = method_info.response_type.definition_name() 434 435 return descriptor 436 437 438def describe_service(service_class): 439 """Build descriptor for service. 440 441 Args: 442 service_class: Service class to describe. 443 444 Returns: 445 Initialized ServiceDescriptor instance describing the service. 446 """ 447 descriptor = ServiceDescriptor() 448 descriptor.name = service_class.__name__ 449 methods = [] 450 remote_methods = service_class.all_remote_methods() 451 for name in sorted(remote_methods.keys()): 452 if name == 'get_descriptor': 453 continue 454 455 method = remote_methods[name] 456 methods.append(describe_method(method)) 457 if methods: 458 descriptor.methods = methods 459 460 return descriptor 461 462 463def describe_file(module): 464 """Build a file from a specified Python module. 465 466 Args: 467 module: Python module to describe. 468 469 Returns: 470 Initialized FileDescriptor instance describing the module. 471 """ 472 # May not import remote at top of file because remote depends on this 473 # file 474 # TODO(rafek): Straighten out this dependency. Possibly move these functions 475 # from descriptor to their own module. 476 from . import remote 477 478 descriptor = FileDescriptor() 479 descriptor.package = util.get_package_for_module(module) 480 481 if not descriptor.package: 482 descriptor.package = None 483 484 message_descriptors = [] 485 enum_descriptors = [] 486 service_descriptors = [] 487 488 # Need to iterate over all top level attributes of the module looking for 489 # message, enum and service definitions. Each definition must be itself 490 # described. 491 for name in sorted(dir(module)): 492 value = getattr(module, name) 493 494 if isinstance(value, type): 495 if issubclass(value, messages.Message): 496 message_descriptors.append(describe_message(value)) 497 498 elif issubclass(value, messages.Enum): 499 enum_descriptors.append(describe_enum(value)) 500 501 elif issubclass(value, remote.Service): 502 service_descriptors.append(describe_service(value)) 503 504 if message_descriptors: 505 descriptor.message_types = message_descriptors 506 507 if enum_descriptors: 508 descriptor.enum_types = enum_descriptors 509 510 if service_descriptors: 511 descriptor.service_types = service_descriptors 512 513 return descriptor 514 515 516def describe_file_set(modules): 517 """Build a file set from a specified Python modules. 518 519 Args: 520 modules: Iterable of Python module to describe. 521 522 Returns: 523 Initialized FileSet instance describing the modules. 524 """ 525 descriptor = FileSet() 526 file_descriptors = [] 527 for module in modules: 528 file_descriptors.append(describe_file(module)) 529 530 if file_descriptors: 531 descriptor.files = file_descriptors 532 533 return descriptor 534 535 536def describe(value): 537 """Describe any value as a descriptor. 538 539 Helper function for describing any object with an appropriate descriptor 540 object. 541 542 Args: 543 value: Value to describe as a descriptor. 544 545 Returns: 546 Descriptor message class if object is describable as a descriptor, else 547 None. 548 """ 549 from . import remote 550 if isinstance(value, types.ModuleType): 551 return describe_file(value) 552 elif callable(value) and hasattr(value, 'remote'): 553 return describe_method(value) 554 elif isinstance(value, messages.Field): 555 return describe_field(value) 556 elif isinstance(value, messages.Enum): 557 return describe_enum_value(value) 558 elif isinstance(value, type): 559 if issubclass(value, messages.Message): 560 return describe_message(value) 561 elif issubclass(value, messages.Enum): 562 return describe_enum(value) 563 elif issubclass(value, remote.Service): 564 return describe_service(value) 565 return None 566 567 568@util.positional(1) 569def import_descriptor_loader(definition_name, importer=__import__): 570 """Find objects by importing modules as needed. 571 572 A definition loader is a function that resolves a definition name to a 573 descriptor. 574 575 The import finder resolves definitions to their names by importing modules 576 when necessary. 577 578 Args: 579 definition_name: Name of definition to find. 580 importer: Import function used for importing new modules. 581 582 Returns: 583 Appropriate descriptor for any describable type located by name. 584 585 Raises: 586 DefinitionNotFoundError when a name does not refer to either a definition 587 or a module. 588 """ 589 # Attempt to import descriptor as a module. 590 if definition_name.startswith('.'): 591 definition_name = definition_name[1:] 592 if not definition_name.startswith('.'): 593 leaf = definition_name.split('.')[-1] 594 if definition_name: 595 try: 596 module = importer(definition_name, '', '', [leaf]) 597 except ImportError: 598 pass 599 else: 600 return describe(module) 601 602 try: 603 # Attempt to use messages.find_definition to find item. 604 return describe(messages.find_definition(definition_name, 605 importer=__import__)) 606 except messages.DefinitionNotFoundError as err: 607 # There are things that find_definition will not find, but if the parent 608 # is loaded, its children can be searched for a match. 609 split_name = definition_name.rsplit('.', 1) 610 if len(split_name) > 1: 611 parent, child = split_name 612 try: 613 parent_definition = import_descriptor_loader(parent, importer=importer) 614 except messages.DefinitionNotFoundError: 615 # Fall through to original error. 616 pass 617 else: 618 # Check the parent definition for a matching descriptor. 619 if isinstance(parent_definition, FileDescriptor): 620 search_list = parent_definition.service_types or [] 621 elif isinstance(parent_definition, ServiceDescriptor): 622 search_list = parent_definition.methods or [] 623 elif isinstance(parent_definition, EnumDescriptor): 624 search_list = parent_definition.values or [] 625 elif isinstance(parent_definition, MessageDescriptor): 626 search_list = parent_definition.fields or [] 627 else: 628 search_list = [] 629 630 for definition in search_list: 631 if definition.name == child: 632 return definition 633 634 # Still didn't find. Reraise original exception. 635 raise err 636 637 638class DescriptorLibrary(object): 639 """A descriptor library is an object that contains known definitions. 640 641 A descriptor library contains a cache of descriptor objects mapped by 642 definition name. It contains all types of descriptors except for 643 file sets. 644 645 When a definition name is requested that the library does not know about 646 it can be provided with a descriptor loader which attempt to resolve the 647 missing descriptor. 648 """ 649 650 @util.positional(1) 651 def __init__(self, 652 descriptors=None, 653 descriptor_loader=import_descriptor_loader): 654 """Constructor. 655 656 Args: 657 descriptors: A dictionary or dictionary-like object that can be used 658 to store and cache descriptors by definition name. 659 definition_loader: A function used for resolving missing descriptors. 660 The function takes a definition name as its parameter and returns 661 an appropriate descriptor. It may raise DefinitionNotFoundError. 662 """ 663 self.__descriptor_loader = descriptor_loader 664 self.__descriptors = descriptors or {} 665 666 def lookup_descriptor(self, definition_name): 667 """Lookup descriptor by name. 668 669 Get descriptor from library by name. If descriptor is not found will 670 attempt to find via descriptor loader if provided. 671 672 Args: 673 definition_name: Definition name to find. 674 675 Returns: 676 Descriptor that describes definition name. 677 678 Raises: 679 DefinitionNotFoundError if not descriptor exists for definition name. 680 """ 681 try: 682 return self.__descriptors[definition_name] 683 except KeyError: 684 pass 685 686 if self.__descriptor_loader: 687 definition = self.__descriptor_loader(definition_name) 688 self.__descriptors[definition_name] = definition 689 return definition 690 else: 691 raise messages.DefinitionNotFoundError( 692 'Could not find definition for %s' % definition_name) 693 694 def lookup_package(self, definition_name): 695 """Determines the package name for any definition. 696 697 Determine the package that any definition name belongs to. May check 698 parent for package name and will resolve missing descriptors if provided 699 descriptor loader. 700 701 Args: 702 definition_name: Definition name to find package for. 703 """ 704 while True: 705 descriptor = self.lookup_descriptor(definition_name) 706 if isinstance(descriptor, FileDescriptor): 707 return descriptor.package 708 else: 709 index = definition_name.rfind('.') 710 if index < 0: 711 return None 712 definition_name = definition_name[:index] 713