• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2015 Google Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Assorted utilities shared between parts of apitools."""
18from __future__ import print_function
19from __future__ import unicode_literals
20
21import collections
22import contextlib
23import gzip
24import json
25import keyword
26import logging
27import os
28import re
29import tempfile
30
31import six
32from six.moves import urllib_parse
33import six.moves.urllib.error as urllib_error
34import six.moves.urllib.request as urllib_request
35
36
37class Error(Exception):
38
39    """Base error for apitools generation."""
40
41
42class CommunicationError(Error):
43
44    """Error in network communication."""
45
46
47def _SortLengthFirstKey(a):
48    return -len(a), a
49
50
51class Names(object):
52
53    """Utility class for cleaning and normalizing names in a fixed style."""
54    DEFAULT_NAME_CONVENTION = 'LOWER_CAMEL'
55    NAME_CONVENTIONS = ['LOWER_CAMEL', 'LOWER_WITH_UNDER', 'NONE']
56
57    def __init__(self, strip_prefixes,
58                 name_convention=None,
59                 capitalize_enums=False):
60        self.__strip_prefixes = sorted(strip_prefixes, key=_SortLengthFirstKey)
61        self.__name_convention = (
62            name_convention or self.DEFAULT_NAME_CONVENTION)
63        self.__capitalize_enums = capitalize_enums
64
65    @staticmethod
66    def __FromCamel(name, separator='_'):
67        name = re.sub(r'([a-z0-9])([A-Z])', r'\1%s\2' % separator, name)
68        return name.lower()
69
70    @staticmethod
71    def __ToCamel(name, separator='_'):
72        # TODO(craigcitro): Consider what to do about leading or trailing
73        # underscores (such as `_refValue` in discovery).
74        return ''.join(s[0:1].upper() + s[1:] for s in name.split(separator))
75
76    @staticmethod
77    def __ToLowerCamel(name, separator='_'):
78        name = Names.__ToCamel(name, separator=separator)
79        return name[0].lower() + name[1:]
80
81    def __StripName(self, name):
82        """Strip strip_prefix entries from name."""
83        if not name:
84            return name
85        for prefix in self.__strip_prefixes:
86            if name.startswith(prefix):
87                return name[len(prefix):]
88        return name
89
90    @staticmethod
91    def CleanName(name):
92        """Perform generic name cleaning."""
93        name = re.sub('[^_A-Za-z0-9]', '_', name)
94        if name[0].isdigit():
95            name = '_%s' % name
96        while keyword.iskeyword(name):
97            name = '%s_' % name
98        # If we end up with __ as a prefix, we'll run afoul of python
99        # field renaming, so we manually correct for it.
100        if name.startswith('__'):
101            name = 'f%s' % name
102        return name
103
104    @staticmethod
105    def NormalizeRelativePath(path):
106        """Normalize camelCase entries in path."""
107        path_components = path.split('/')
108        normalized_components = []
109        for component in path_components:
110            if re.match(r'{[A-Za-z0-9_]+}$', component):
111                normalized_components.append(
112                    '{%s}' % Names.CleanName(component[1:-1]))
113            else:
114                normalized_components.append(component)
115        return '/'.join(normalized_components)
116
117    def NormalizeEnumName(self, enum_name):
118        if self.__capitalize_enums:
119            enum_name = enum_name.upper()
120        return self.CleanName(enum_name)
121
122    def ClassName(self, name, separator='_'):
123        """Generate a valid class name from name."""
124        # TODO(craigcitro): Get rid of this case here and in MethodName.
125        if name is None:
126            return name
127        # TODO(craigcitro): This is a hack to handle the case of specific
128        # protorpc class names; clean this up.
129        if name.startswith(('protorpc.', 'message_types.',
130                            'apitools.base.protorpclite.',
131                            'apitools.base.protorpclite.message_types.')):
132            return name
133        name = self.__StripName(name)
134        name = self.__ToCamel(name, separator=separator)
135        return self.CleanName(name)
136
137    def MethodName(self, name, separator='_'):
138        """Generate a valid method name from name."""
139        if name is None:
140            return None
141        name = Names.__ToCamel(name, separator=separator)
142        return Names.CleanName(name)
143
144    def FieldName(self, name):
145        """Generate a valid field name from name."""
146        # TODO(craigcitro): We shouldn't need to strip this name, but some
147        # of the service names here are excessive. Fix the API and then
148        # remove this.
149        name = self.__StripName(name)
150        if self.__name_convention == 'LOWER_CAMEL':
151            name = Names.__ToLowerCamel(name)
152        elif self.__name_convention == 'LOWER_WITH_UNDER':
153            name = Names.__FromCamel(name)
154        return Names.CleanName(name)
155
156
157@contextlib.contextmanager
158def Chdir(dirname, create=True):
159    if not os.path.exists(dirname):
160        if not create:
161            raise OSError('Cannot find directory %s' % dirname)
162        else:
163            os.mkdir(dirname)
164    previous_directory = os.getcwd()
165    try:
166        os.chdir(dirname)
167        yield
168    finally:
169        os.chdir(previous_directory)
170
171
172def NormalizeVersion(version):
173    # Currently, '.' is the only character that might cause us trouble.
174    return version.replace('.', '_')
175
176
177def _ComputePaths(package, version, discovery_doc):
178    full_path = urllib_parse.urljoin(
179        discovery_doc['rootUrl'], discovery_doc['servicePath'])
180    api_path_component = '/'.join((package, version, ''))
181    if api_path_component not in full_path:
182        return full_path, ''
183    prefix, _, suffix = full_path.rpartition(api_path_component)
184    return prefix + api_path_component, suffix
185
186
187class ClientInfo(collections.namedtuple('ClientInfo', (
188        'package', 'scopes', 'version', 'client_id', 'client_secret',
189        'user_agent', 'client_class_name', 'url_version', 'api_key',
190        'base_url', 'base_path'))):
191
192    """Container for client-related info and names."""
193
194    @classmethod
195    def Create(cls, discovery_doc,
196               scope_ls, client_id, client_secret, user_agent, names, api_key):
197        """Create a new ClientInfo object from a discovery document."""
198        scopes = set(
199            discovery_doc.get('auth', {}).get('oauth2', {}).get('scopes', {}))
200        scopes.update(scope_ls)
201        package = discovery_doc['name']
202        url_version = discovery_doc['version']
203        base_url, base_path = _ComputePaths(package, url_version,
204                                            discovery_doc)
205
206        client_info = {
207            'package': package,
208            'version': NormalizeVersion(discovery_doc['version']),
209            'url_version': url_version,
210            'scopes': sorted(list(scopes)),
211            'client_id': client_id,
212            'client_secret': client_secret,
213            'user_agent': user_agent,
214            'api_key': api_key,
215            'base_url': base_url,
216            'base_path': base_path,
217        }
218        client_class_name = '%s%s' % (
219            names.ClassName(client_info['package']),
220            names.ClassName(client_info['version']))
221        client_info['client_class_name'] = client_class_name
222        return cls(**client_info)
223
224    @property
225    def default_directory(self):
226        return self.package
227
228    @property
229    def client_rule_name(self):
230        return '%s_%s_client' % (self.package, self.version)
231
232    @property
233    def client_file_name(self):
234        return '%s.py' % self.client_rule_name
235
236    @property
237    def messages_rule_name(self):
238        return '%s_%s_messages' % (self.package, self.version)
239
240    @property
241    def services_rule_name(self):
242        return '%s_%s_services' % (self.package, self.version)
243
244    @property
245    def messages_file_name(self):
246        return '%s.py' % self.messages_rule_name
247
248    @property
249    def messages_proto_file_name(self):
250        return '%s.proto' % self.messages_rule_name
251
252    @property
253    def services_proto_file_name(self):
254        return '%s.proto' % self.services_rule_name
255
256
257def ReplaceHomoglyphs(s):
258    """Returns s with unicode homoglyphs replaced by ascii equivalents."""
259    homoglyphs = {
260        '\xa0': ' ',  #   ?
261        '\u00e3': '',  # TODO(gsfowler) drop after .proto spurious char elided
262        '\u00a0': ' ',  #   ?
263        '\u00a9': '(C)',  # COPYRIGHT SIGN (would you believe "asciiglyph"?)
264        '\u00ae': '(R)',  # REGISTERED SIGN (would you believe "asciiglyph"?)
265        '\u2014': '-',  # EM DASH
266        '\u2018': "'",  # LEFT SINGLE QUOTATION MARK
267        '\u2019': "'",  # RIGHT SINGLE QUOTATION MARK
268        '\u201c': '"',  # LEFT DOUBLE QUOTATION MARK
269        '\u201d': '"',  # RIGHT DOUBLE QUOTATION MARK
270        '\u2026': '...',  # HORIZONTAL ELLIPSIS
271        '\u2e3a': '-',  # TWO-EM DASH
272    }
273
274    def _ReplaceOne(c):
275        """Returns the homoglyph or escaped replacement for c."""
276        equiv = homoglyphs.get(c)
277        if equiv is not None:
278            return equiv
279        try:
280            c.encode('ascii')
281            return c
282        except UnicodeError:
283            pass
284        try:
285            return c.encode('unicode-escape').decode('ascii')
286        except UnicodeError:
287            return '?'
288
289    return ''.join([_ReplaceOne(c) for c in s])
290
291
292def CleanDescription(description):
293    """Return a version of description safe for printing in a docstring."""
294    if not isinstance(description, six.string_types):
295        return description
296    if six.PY3:
297        # https://docs.python.org/3/reference/lexical_analysis.html#index-18
298        description = description.replace('\\N', '\\\\N')
299        description = description.replace('\\u', '\\\\u')
300        description = description.replace('\\U', '\\\\U')
301    description = ReplaceHomoglyphs(description)
302    return description.replace('"""', '" " "')
303
304
305class SimplePrettyPrinter(object):
306
307    """Simple pretty-printer that supports an indent contextmanager."""
308
309    def __init__(self, out):
310        self.__out = out
311        self.__indent = ''
312        self.__skip = False
313        self.__comment_context = False
314
315    @property
316    def indent(self):
317        return self.__indent
318
319    def CalculateWidth(self, max_width=78):
320        return max_width - len(self.indent)
321
322    @contextlib.contextmanager
323    def Indent(self, indent='  '):
324        previous_indent = self.__indent
325        self.__indent = '%s%s' % (previous_indent, indent)
326        yield
327        self.__indent = previous_indent
328
329    @contextlib.contextmanager
330    def CommentContext(self):
331        """Print without any argument formatting."""
332        old_context = self.__comment_context
333        self.__comment_context = True
334        yield
335        self.__comment_context = old_context
336
337    def __call__(self, *args):
338        if self.__comment_context and args[1:]:
339            raise Error('Cannot do string interpolation in comment context')
340        if args and args[0]:
341            if not self.__comment_context:
342                line = (args[0] % args[1:]).rstrip()
343            else:
344                line = args[0].rstrip()
345            line = ReplaceHomoglyphs(line)
346            try:
347                print('%s%s' % (self.__indent, line), file=self.__out)
348            except UnicodeEncodeError:
349                line = line.encode('ascii', 'backslashreplace').decode('ascii')
350                print('%s%s' % (self.__indent, line), file=self.__out)
351        else:
352            print('', file=self.__out)
353
354
355def _NormalizeDiscoveryUrls(discovery_url):
356    """Expands a few abbreviations into full discovery urls."""
357    if discovery_url.startswith('http'):
358        return [discovery_url]
359    elif '.' not in discovery_url:
360        raise ValueError('Unrecognized value "%s" for discovery url')
361    api_name, _, api_version = discovery_url.partition('.')
362    return [
363        'https://www.googleapis.com/discovery/v1/apis/%s/%s/rest' % (
364            api_name, api_version),
365        'https://%s.googleapis.com/$discovery/rest?version=%s' % (
366            api_name, api_version),
367    ]
368
369
370def _Gunzip(gzipped_content):
371    """Returns gunzipped content from gzipped contents."""
372    f = tempfile.NamedTemporaryFile(suffix='gz', mode='w+b', delete=False)
373    try:
374        f.write(gzipped_content)
375        f.close()  # force file synchronization
376        with gzip.open(f.name, 'rb') as h:
377            decompressed_content = h.read()
378        return decompressed_content
379    finally:
380        os.unlink(f.name)
381
382
383def _GetURLContent(url):
384    """Download and return the content of URL."""
385    response = urllib_request.urlopen(url)
386    encoding = response.info().get('Content-Encoding')
387    if encoding == 'gzip':
388        content = _Gunzip(response.read())
389    else:
390        content = response.read()
391    return content
392
393
394def FetchDiscoveryDoc(discovery_url, retries=5):
395    """Fetch the discovery document at the given url."""
396    discovery_urls = _NormalizeDiscoveryUrls(discovery_url)
397    discovery_doc = None
398    last_exception = None
399    for url in discovery_urls:
400        for _ in range(retries):
401            try:
402                content = _GetURLContent(url)
403                if isinstance(content, bytes):
404                    content = content.decode('utf8')
405                discovery_doc = json.loads(content)
406                break
407            except (urllib_error.HTTPError, urllib_error.URLError) as e:
408                logging.info(
409                    'Attempting to fetch discovery doc again after "%s"', e)
410                last_exception = e
411    if discovery_doc is None:
412        raise CommunicationError(
413            'Could not find discovery doc at any of %s: %s' % (
414                discovery_urls, last_exception))
415    return discovery_doc
416