• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Internationalization and localization support.
2
3This module provides internationalization (I18N) and localization (L10N)
4support for your Python programs by providing an interface to the GNU gettext
5message catalog library.
6
7I18N refers to the operation by which a program is made aware of multiple
8languages.  L10N refers to the adaptation of your program, once
9internationalized, to the local language and cultural habits.
10
11"""
12
13# This module represents the integration of work, contributions, feedback, and
14# suggestions from the following people:
15#
16# Martin von Loewis, who wrote the initial implementation of the underlying
17# C-based libintlmodule (later renamed _gettext), along with a skeletal
18# gettext.py implementation.
19#
20# Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
21# which also included a pure-Python implementation to read .mo files if
22# intlmodule wasn't available.
23#
24# James Henstridge, who also wrote a gettext.py module, which has some
25# interesting, but currently unsupported experimental features: the notion of
26# a Catalog class and instances, and the ability to add to a catalog file via
27# a Python API.
28#
29# Barry Warsaw integrated these modules, wrote the .install() API and code,
30# and conformed all C and Python code to Python's coding standards.
31#
32# Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
33# module.
34#
35# J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs.
36#
37# TODO:
38# - Lazy loading of .mo files.  Currently the entire catalog is loaded into
39#   memory, but that's probably bad for large translated programs.  Instead,
40#   the lexical sort of original strings in GNU .mo files should be exploited
41#   to do binary searches and lazy initializations.  Or you might want to use
42#   the undocumented double-hash algorithm for .mo files with hash tables, but
43#   you'll need to study the GNU gettext code to do this.
44#
45# - Support Solaris .mo file formats.  Unfortunately, we've been unable to
46#   find this format documented anywhere.
47
48
49import locale, copy, io, os, re, struct, sys
50from errno import ENOENT
51
52
53__all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
54           'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
55           'bind_textdomain_codeset',
56           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
57           'ldngettext', 'lngettext', 'ngettext',
58           ]
59
60_default_localedir = os.path.join(sys.base_prefix, 'share', 'locale')
61
62# Expression parsing for plural form selection.
63#
64# The gettext library supports a small subset of C syntax.  The only
65# incompatible difference is that integer literals starting with zero are
66# decimal.
67#
68# https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms
69# http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y
70
71_token_pattern = re.compile(r"""
72        (?P<WHITESPACES>[ \t]+)                    | # spaces and horizontal tabs
73        (?P<NUMBER>[0-9]+\b)                       | # decimal integer
74        (?P<NAME>n\b)                              | # only n is allowed
75        (?P<PARENTHESIS>[()])                      |
76        (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >,
77                                                     # <=, >=, ==, !=, &&, ||,
78                                                     # ? :
79                                                     # unary and bitwise ops
80                                                     # not allowed
81        (?P<INVALID>\w+|.)                           # invalid token
82    """, re.VERBOSE|re.DOTALL)
83
84def _tokenize(plural):
85    for mo in re.finditer(_token_pattern, plural):
86        kind = mo.lastgroup
87        if kind == 'WHITESPACES':
88            continue
89        value = mo.group(kind)
90        if kind == 'INVALID':
91            raise ValueError('invalid token in plural form: %s' % value)
92        yield value
93    yield ''
94
95def _error(value):
96    if value:
97        return ValueError('unexpected token in plural form: %s' % value)
98    else:
99        return ValueError('unexpected end of plural form')
100
101_binary_ops = (
102    ('||',),
103    ('&&',),
104    ('==', '!='),
105    ('<', '>', '<=', '>='),
106    ('+', '-'),
107    ('*', '/', '%'),
108)
109_binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops}
110_c2py_ops = {'||': 'or', '&&': 'and', '/': '//'}
111
112def _parse(tokens, priority=-1):
113    result = ''
114    nexttok = next(tokens)
115    while nexttok == '!':
116        result += 'not '
117        nexttok = next(tokens)
118
119    if nexttok == '(':
120        sub, nexttok = _parse(tokens)
121        result = '%s(%s)' % (result, sub)
122        if nexttok != ')':
123            raise ValueError('unbalanced parenthesis in plural form')
124    elif nexttok == 'n':
125        result = '%s%s' % (result, nexttok)
126    else:
127        try:
128            value = int(nexttok, 10)
129        except ValueError:
130            raise _error(nexttok) from None
131        result = '%s%d' % (result, value)
132    nexttok = next(tokens)
133
134    j = 100
135    while nexttok in _binary_ops:
136        i = _binary_ops[nexttok]
137        if i < priority:
138            break
139        # Break chained comparisons
140        if i in (3, 4) and j in (3, 4):  # '==', '!=', '<', '>', '<=', '>='
141            result = '(%s)' % result
142        # Replace some C operators by their Python equivalents
143        op = _c2py_ops.get(nexttok, nexttok)
144        right, nexttok = _parse(tokens, i + 1)
145        result = '%s %s %s' % (result, op, right)
146        j = i
147    if j == priority == 4:  # '<', '>', '<=', '>='
148        result = '(%s)' % result
149
150    if nexttok == '?' and priority <= 0:
151        if_true, nexttok = _parse(tokens, 0)
152        if nexttok != ':':
153            raise _error(nexttok)
154        if_false, nexttok = _parse(tokens)
155        result = '%s if %s else %s' % (if_true, result, if_false)
156        if priority == 0:
157            result = '(%s)' % result
158
159    return result, nexttok
160
161def _as_int(n):
162    try:
163        i = round(n)
164    except TypeError:
165        raise TypeError('Plural value must be an integer, got %s' %
166                        (n.__class__.__name__,)) from None
167    return n
168
169def c2py(plural):
170    """Gets a C expression as used in PO files for plural forms and returns a
171    Python function that implements an equivalent expression.
172    """
173
174    if len(plural) > 1000:
175        raise ValueError('plural form expression is too long')
176    try:
177        result, nexttok = _parse(_tokenize(plural))
178        if nexttok:
179            raise _error(nexttok)
180
181        depth = 0
182        for c in result:
183            if c == '(':
184                depth += 1
185                if depth > 20:
186                    # Python compiler limit is about 90.
187                    # The most complex example has 2.
188                    raise ValueError('plural form expression is too complex')
189            elif c == ')':
190                depth -= 1
191
192        ns = {'_as_int': _as_int}
193        exec('''if True:
194            def func(n):
195                if not isinstance(n, int):
196                    n = _as_int(n)
197                return int(%s)
198            ''' % result, ns)
199        return ns['func']
200    except RecursionError:
201        # Recursion error can be raised in _parse() or exec().
202        raise ValueError('plural form expression is too complex')
203
204
205def _expand_lang(loc):
206    loc = locale.normalize(loc)
207    COMPONENT_CODESET   = 1 << 0
208    COMPONENT_TERRITORY = 1 << 1
209    COMPONENT_MODIFIER  = 1 << 2
210    # split up the locale into its base components
211    mask = 0
212    pos = loc.find('@')
213    if pos >= 0:
214        modifier = loc[pos:]
215        loc = loc[:pos]
216        mask |= COMPONENT_MODIFIER
217    else:
218        modifier = ''
219    pos = loc.find('.')
220    if pos >= 0:
221        codeset = loc[pos:]
222        loc = loc[:pos]
223        mask |= COMPONENT_CODESET
224    else:
225        codeset = ''
226    pos = loc.find('_')
227    if pos >= 0:
228        territory = loc[pos:]
229        loc = loc[:pos]
230        mask |= COMPONENT_TERRITORY
231    else:
232        territory = ''
233    language = loc
234    ret = []
235    for i in range(mask+1):
236        if not (i & ~mask):  # if all components for this combo exist ...
237            val = language
238            if i & COMPONENT_TERRITORY: val += territory
239            if i & COMPONENT_CODESET:   val += codeset
240            if i & COMPONENT_MODIFIER:  val += modifier
241            ret.append(val)
242    ret.reverse()
243    return ret
244
245
246
247class NullTranslations:
248    def __init__(self, fp=None):
249        self._info = {}
250        self._charset = None
251        self._output_charset = None
252        self._fallback = None
253        if fp is not None:
254            self._parse(fp)
255
256    def _parse(self, fp):
257        pass
258
259    def add_fallback(self, fallback):
260        if self._fallback:
261            self._fallback.add_fallback(fallback)
262        else:
263            self._fallback = fallback
264
265    def gettext(self, message):
266        if self._fallback:
267            return self._fallback.gettext(message)
268        return message
269
270    def lgettext(self, message):
271        if self._fallback:
272            return self._fallback.lgettext(message)
273        return message
274
275    def ngettext(self, msgid1, msgid2, n):
276        if self._fallback:
277            return self._fallback.ngettext(msgid1, msgid2, n)
278        if n == 1:
279            return msgid1
280        else:
281            return msgid2
282
283    def lngettext(self, msgid1, msgid2, n):
284        if self._fallback:
285            return self._fallback.lngettext(msgid1, msgid2, n)
286        if n == 1:
287            return msgid1
288        else:
289            return msgid2
290
291    def info(self):
292        return self._info
293
294    def charset(self):
295        return self._charset
296
297    def output_charset(self):
298        return self._output_charset
299
300    def set_output_charset(self, charset):
301        self._output_charset = charset
302
303    def install(self, names=None):
304        import builtins
305        builtins.__dict__['_'] = self.gettext
306        if hasattr(names, "__contains__"):
307            if "gettext" in names:
308                builtins.__dict__['gettext'] = builtins.__dict__['_']
309            if "ngettext" in names:
310                builtins.__dict__['ngettext'] = self.ngettext
311            if "lgettext" in names:
312                builtins.__dict__['lgettext'] = self.lgettext
313            if "lngettext" in names:
314                builtins.__dict__['lngettext'] = self.lngettext
315
316
317class GNUTranslations(NullTranslations):
318    # Magic number of .mo files
319    LE_MAGIC = 0x950412de
320    BE_MAGIC = 0xde120495
321
322    # Acceptable .mo versions
323    VERSIONS = (0, 1)
324
325    def _get_versions(self, version):
326        """Returns a tuple of major version, minor version"""
327        return (version >> 16, version & 0xffff)
328
329    def _parse(self, fp):
330        """Override this method to support alternative .mo formats."""
331        unpack = struct.unpack
332        filename = getattr(fp, 'name', '')
333        # Parse the .mo file header, which consists of 5 little endian 32
334        # bit words.
335        self._catalog = catalog = {}
336        self.plural = lambda n: int(n != 1) # germanic plural by default
337        buf = fp.read()
338        buflen = len(buf)
339        # Are we big endian or little endian?
340        magic = unpack('<I', buf[:4])[0]
341        if magic == self.LE_MAGIC:
342            version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])
343            ii = '<II'
344        elif magic == self.BE_MAGIC:
345            version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
346            ii = '>II'
347        else:
348            raise OSError(0, 'Bad magic number', filename)
349
350        major_version, minor_version = self._get_versions(version)
351
352        if major_version not in self.VERSIONS:
353            raise OSError(0, 'Bad version number ' + str(major_version), filename)
354
355        # Now put all messages from the .mo file buffer into the catalog
356        # dictionary.
357        for i in range(0, msgcount):
358            mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
359            mend = moff + mlen
360            tlen, toff = unpack(ii, buf[transidx:transidx+8])
361            tend = toff + tlen
362            if mend < buflen and tend < buflen:
363                msg = buf[moff:mend]
364                tmsg = buf[toff:tend]
365            else:
366                raise OSError(0, 'File is corrupt', filename)
367            # See if we're looking at GNU .mo conventions for metadata
368            if mlen == 0:
369                # Catalog description
370                lastk = None
371                for b_item in tmsg.split('\n'.encode("ascii")):
372                    item = b_item.decode().strip()
373                    if not item:
374                        continue
375                    k = v = None
376                    if ':' in item:
377                        k, v = item.split(':', 1)
378                        k = k.strip().lower()
379                        v = v.strip()
380                        self._info[k] = v
381                        lastk = k
382                    elif lastk:
383                        self._info[lastk] += '\n' + item
384                    if k == 'content-type':
385                        self._charset = v.split('charset=')[1]
386                    elif k == 'plural-forms':
387                        v = v.split(';')
388                        plural = v[1].split('plural=')[1]
389                        self.plural = c2py(plural)
390            # Note: we unconditionally convert both msgids and msgstrs to
391            # Unicode using the character encoding specified in the charset
392            # parameter of the Content-Type header.  The gettext documentation
393            # strongly encourages msgids to be us-ascii, but some applications
394            # require alternative encodings (e.g. Zope's ZCML and ZPT).  For
395            # traditional gettext applications, the msgid conversion will
396            # cause no problems since us-ascii should always be a subset of
397            # the charset encoding.  We may want to fall back to 8-bit msgids
398            # if the Unicode conversion fails.
399            charset = self._charset or 'ascii'
400            if b'\x00' in msg:
401                # Plural forms
402                msgid1, msgid2 = msg.split(b'\x00')
403                tmsg = tmsg.split(b'\x00')
404                msgid1 = str(msgid1, charset)
405                for i, x in enumerate(tmsg):
406                    catalog[(msgid1, i)] = str(x, charset)
407            else:
408                catalog[str(msg, charset)] = str(tmsg, charset)
409            # advance to next entry in the seek tables
410            masteridx += 8
411            transidx += 8
412
413    def lgettext(self, message):
414        missing = object()
415        tmsg = self._catalog.get(message, missing)
416        if tmsg is missing:
417            if self._fallback:
418                return self._fallback.lgettext(message)
419            return message
420        if self._output_charset:
421            return tmsg.encode(self._output_charset)
422        return tmsg.encode(locale.getpreferredencoding())
423
424    def lngettext(self, msgid1, msgid2, n):
425        try:
426            tmsg = self._catalog[(msgid1, self.plural(n))]
427            if self._output_charset:
428                return tmsg.encode(self._output_charset)
429            return tmsg.encode(locale.getpreferredencoding())
430        except KeyError:
431            if self._fallback:
432                return self._fallback.lngettext(msgid1, msgid2, n)
433            if n == 1:
434                return msgid1
435            else:
436                return msgid2
437
438    def gettext(self, message):
439        missing = object()
440        tmsg = self._catalog.get(message, missing)
441        if tmsg is missing:
442            if self._fallback:
443                return self._fallback.gettext(message)
444            return message
445        return tmsg
446
447    def ngettext(self, msgid1, msgid2, n):
448        try:
449            tmsg = self._catalog[(msgid1, self.plural(n))]
450        except KeyError:
451            if self._fallback:
452                return self._fallback.ngettext(msgid1, msgid2, n)
453            if n == 1:
454                tmsg = msgid1
455            else:
456                tmsg = msgid2
457        return tmsg
458
459
460# Locate a .mo file using the gettext strategy
461def find(domain, localedir=None, languages=None, all=False):
462    # Get some reasonable defaults for arguments that were not supplied
463    if localedir is None:
464        localedir = _default_localedir
465    if languages is None:
466        languages = []
467        for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
468            val = os.environ.get(envar)
469            if val:
470                languages = val.split(':')
471                break
472        if 'C' not in languages:
473            languages.append('C')
474    # now normalize and expand the languages
475    nelangs = []
476    for lang in languages:
477        for nelang in _expand_lang(lang):
478            if nelang not in nelangs:
479                nelangs.append(nelang)
480    # select a language
481    if all:
482        result = []
483    else:
484        result = None
485    for lang in nelangs:
486        if lang == 'C':
487            break
488        mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
489        if os.path.exists(mofile):
490            if all:
491                result.append(mofile)
492            else:
493                return mofile
494    return result
495
496
497
498# a mapping between absolute .mo file path and Translation object
499_translations = {}
500
501def translation(domain, localedir=None, languages=None,
502                class_=None, fallback=False, codeset=None):
503    if class_ is None:
504        class_ = GNUTranslations
505    mofiles = find(domain, localedir, languages, all=True)
506    if not mofiles:
507        if fallback:
508            return NullTranslations()
509        raise OSError(ENOENT, 'No translation file found for domain', domain)
510    # Avoid opening, reading, and parsing the .mo file after it's been done
511    # once.
512    result = None
513    for mofile in mofiles:
514        key = (class_, os.path.abspath(mofile))
515        t = _translations.get(key)
516        if t is None:
517            with open(mofile, 'rb') as fp:
518                t = _translations.setdefault(key, class_(fp))
519        # Copy the translation object to allow setting fallbacks and
520        # output charset. All other instance data is shared with the
521        # cached object.
522        t = copy.copy(t)
523        if codeset:
524            t.set_output_charset(codeset)
525        if result is None:
526            result = t
527        else:
528            result.add_fallback(t)
529    return result
530
531
532def install(domain, localedir=None, codeset=None, names=None):
533    t = translation(domain, localedir, fallback=True, codeset=codeset)
534    t.install(names)
535
536
537
538# a mapping b/w domains and locale directories
539_localedirs = {}
540# a mapping b/w domains and codesets
541_localecodesets = {}
542# current global domain, `messages' used for compatibility w/ GNU gettext
543_current_domain = 'messages'
544
545
546def textdomain(domain=None):
547    global _current_domain
548    if domain is not None:
549        _current_domain = domain
550    return _current_domain
551
552
553def bindtextdomain(domain, localedir=None):
554    global _localedirs
555    if localedir is not None:
556        _localedirs[domain] = localedir
557    return _localedirs.get(domain, _default_localedir)
558
559
560def bind_textdomain_codeset(domain, codeset=None):
561    global _localecodesets
562    if codeset is not None:
563        _localecodesets[domain] = codeset
564    return _localecodesets.get(domain)
565
566
567def dgettext(domain, message):
568    try:
569        t = translation(domain, _localedirs.get(domain, None),
570                        codeset=_localecodesets.get(domain))
571    except OSError:
572        return message
573    return t.gettext(message)
574
575def ldgettext(domain, message):
576    try:
577        t = translation(domain, _localedirs.get(domain, None),
578                        codeset=_localecodesets.get(domain))
579    except OSError:
580        return message
581    return t.lgettext(message)
582
583def dngettext(domain, msgid1, msgid2, n):
584    try:
585        t = translation(domain, _localedirs.get(domain, None),
586                        codeset=_localecodesets.get(domain))
587    except OSError:
588        if n == 1:
589            return msgid1
590        else:
591            return msgid2
592    return t.ngettext(msgid1, msgid2, n)
593
594def ldngettext(domain, msgid1, msgid2, n):
595    try:
596        t = translation(domain, _localedirs.get(domain, None),
597                        codeset=_localecodesets.get(domain))
598    except OSError:
599        if n == 1:
600            return msgid1
601        else:
602            return msgid2
603    return t.lngettext(msgid1, msgid2, n)
604
605def gettext(message):
606    return dgettext(_current_domain, message)
607
608def lgettext(message):
609    return ldgettext(_current_domain, message)
610
611def ngettext(msgid1, msgid2, n):
612    return dngettext(_current_domain, msgid1, msgid2, n)
613
614def lngettext(msgid1, msgid2, n):
615    return ldngettext(_current_domain, msgid1, msgid2, n)
616
617# dcgettext() has been deemed unnecessary and is not implemented.
618
619# James Henstridge's Catalog constructor from GNOME gettext.  Documented usage
620# was:
621#
622#    import gettext
623#    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
624#    _ = cat.gettext
625#    print _('Hello World')
626
627# The resulting catalog object currently don't support access through a
628# dictionary API, which was supported (but apparently unused) in GNOME
629# gettext.
630
631Catalog = translation
632