1"""Internationalization and localization support. 2 3This module provides internationalization (I18N) and localization (L10N) 4support for your Python programs by providing an interface to the GNU gettext 5message catalog library. 6 7I18N refers to the operation by which a program is made aware of multiple 8languages. L10N refers to the adaptation of your program, once 9internationalized, to the local language and cultural habits. 10 11""" 12 13# This module represents the integration of work, contributions, feedback, and 14# suggestions from the following people: 15# 16# Martin von Loewis, who wrote the initial implementation of the underlying 17# C-based libintlmodule (later renamed _gettext), along with a skeletal 18# gettext.py implementation. 19# 20# Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule, 21# which also included a pure-Python implementation to read .mo files if 22# intlmodule wasn't available. 23# 24# James Henstridge, who also wrote a gettext.py module, which has some 25# interesting, but currently unsupported experimental features: the notion of 26# a Catalog class and instances, and the ability to add to a catalog file via 27# a Python API. 28# 29# Barry Warsaw integrated these modules, wrote the .install() API and code, 30# and conformed all C and Python code to Python's coding standards. 31# 32# Francois Pinard and Marc-Andre Lemburg also contributed valuably to this 33# module. 34# 35# J. David Ibanez implemented plural forms. Bruno Haible fixed some bugs. 36# 37# TODO: 38# - Lazy loading of .mo files. Currently the entire catalog is loaded into 39# memory, but that's probably bad for large translated programs. Instead, 40# the lexical sort of original strings in GNU .mo files should be exploited 41# to do binary searches and lazy initializations. Or you might want to use 42# the undocumented double-hash algorithm for .mo files with hash tables, but 43# you'll need to study the GNU gettext code to do this. 44# 45# - Support Solaris .mo file formats. Unfortunately, we've been unable to 46# find this format documented anywhere. 47 48 49import locale, copy, io, os, re, struct, sys 50from errno import ENOENT 51 52 53__all__ = ['NullTranslations', 'GNUTranslations', 'Catalog', 54 'find', 'translation', 'install', 'textdomain', 'bindtextdomain', 55 'bind_textdomain_codeset', 56 'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext', 57 'ldngettext', 'lngettext', 'ngettext', 58 ] 59 60_default_localedir = os.path.join(sys.base_prefix, 'share', 'locale') 61 62# Expression parsing for plural form selection. 63# 64# The gettext library supports a small subset of C syntax. The only 65# incompatible difference is that integer literals starting with zero are 66# decimal. 67# 68# https://www.gnu.org/software/gettext/manual/gettext.html#Plural-forms 69# http://git.savannah.gnu.org/cgit/gettext.git/tree/gettext-runtime/intl/plural.y 70 71_token_pattern = re.compile(r""" 72 (?P<WHITESPACES>[ \t]+) | # spaces and horizontal tabs 73 (?P<NUMBER>[0-9]+\b) | # decimal integer 74 (?P<NAME>n\b) | # only n is allowed 75 (?P<PARENTHESIS>[()]) | 76 (?P<OPERATOR>[-*/%+?:]|[><!]=?|==|&&|\|\|) | # !, *, /, %, +, -, <, >, 77 # <=, >=, ==, !=, &&, ||, 78 # ? : 79 # unary and bitwise ops 80 # not allowed 81 (?P<INVALID>\w+|.) # invalid token 82 """, re.VERBOSE|re.DOTALL) 83 84def _tokenize(plural): 85 for mo in re.finditer(_token_pattern, plural): 86 kind = mo.lastgroup 87 if kind == 'WHITESPACES': 88 continue 89 value = mo.group(kind) 90 if kind == 'INVALID': 91 raise ValueError('invalid token in plural form: %s' % value) 92 yield value 93 yield '' 94 95def _error(value): 96 if value: 97 return ValueError('unexpected token in plural form: %s' % value) 98 else: 99 return ValueError('unexpected end of plural form') 100 101_binary_ops = ( 102 ('||',), 103 ('&&',), 104 ('==', '!='), 105 ('<', '>', '<=', '>='), 106 ('+', '-'), 107 ('*', '/', '%'), 108) 109_binary_ops = {op: i for i, ops in enumerate(_binary_ops, 1) for op in ops} 110_c2py_ops = {'||': 'or', '&&': 'and', '/': '//'} 111 112def _parse(tokens, priority=-1): 113 result = '' 114 nexttok = next(tokens) 115 while nexttok == '!': 116 result += 'not ' 117 nexttok = next(tokens) 118 119 if nexttok == '(': 120 sub, nexttok = _parse(tokens) 121 result = '%s(%s)' % (result, sub) 122 if nexttok != ')': 123 raise ValueError('unbalanced parenthesis in plural form') 124 elif nexttok == 'n': 125 result = '%s%s' % (result, nexttok) 126 else: 127 try: 128 value = int(nexttok, 10) 129 except ValueError: 130 raise _error(nexttok) from None 131 result = '%s%d' % (result, value) 132 nexttok = next(tokens) 133 134 j = 100 135 while nexttok in _binary_ops: 136 i = _binary_ops[nexttok] 137 if i < priority: 138 break 139 # Break chained comparisons 140 if i in (3, 4) and j in (3, 4): # '==', '!=', '<', '>', '<=', '>=' 141 result = '(%s)' % result 142 # Replace some C operators by their Python equivalents 143 op = _c2py_ops.get(nexttok, nexttok) 144 right, nexttok = _parse(tokens, i + 1) 145 result = '%s %s %s' % (result, op, right) 146 j = i 147 if j == priority == 4: # '<', '>', '<=', '>=' 148 result = '(%s)' % result 149 150 if nexttok == '?' and priority <= 0: 151 if_true, nexttok = _parse(tokens, 0) 152 if nexttok != ':': 153 raise _error(nexttok) 154 if_false, nexttok = _parse(tokens) 155 result = '%s if %s else %s' % (if_true, result, if_false) 156 if priority == 0: 157 result = '(%s)' % result 158 159 return result, nexttok 160 161def _as_int(n): 162 try: 163 i = round(n) 164 except TypeError: 165 raise TypeError('Plural value must be an integer, got %s' % 166 (n.__class__.__name__,)) from None 167 return n 168 169def c2py(plural): 170 """Gets a C expression as used in PO files for plural forms and returns a 171 Python function that implements an equivalent expression. 172 """ 173 174 if len(plural) > 1000: 175 raise ValueError('plural form expression is too long') 176 try: 177 result, nexttok = _parse(_tokenize(plural)) 178 if nexttok: 179 raise _error(nexttok) 180 181 depth = 0 182 for c in result: 183 if c == '(': 184 depth += 1 185 if depth > 20: 186 # Python compiler limit is about 90. 187 # The most complex example has 2. 188 raise ValueError('plural form expression is too complex') 189 elif c == ')': 190 depth -= 1 191 192 ns = {'_as_int': _as_int} 193 exec('''if True: 194 def func(n): 195 if not isinstance(n, int): 196 n = _as_int(n) 197 return int(%s) 198 ''' % result, ns) 199 return ns['func'] 200 except RecursionError: 201 # Recursion error can be raised in _parse() or exec(). 202 raise ValueError('plural form expression is too complex') 203 204 205def _expand_lang(loc): 206 loc = locale.normalize(loc) 207 COMPONENT_CODESET = 1 << 0 208 COMPONENT_TERRITORY = 1 << 1 209 COMPONENT_MODIFIER = 1 << 2 210 # split up the locale into its base components 211 mask = 0 212 pos = loc.find('@') 213 if pos >= 0: 214 modifier = loc[pos:] 215 loc = loc[:pos] 216 mask |= COMPONENT_MODIFIER 217 else: 218 modifier = '' 219 pos = loc.find('.') 220 if pos >= 0: 221 codeset = loc[pos:] 222 loc = loc[:pos] 223 mask |= COMPONENT_CODESET 224 else: 225 codeset = '' 226 pos = loc.find('_') 227 if pos >= 0: 228 territory = loc[pos:] 229 loc = loc[:pos] 230 mask |= COMPONENT_TERRITORY 231 else: 232 territory = '' 233 language = loc 234 ret = [] 235 for i in range(mask+1): 236 if not (i & ~mask): # if all components for this combo exist ... 237 val = language 238 if i & COMPONENT_TERRITORY: val += territory 239 if i & COMPONENT_CODESET: val += codeset 240 if i & COMPONENT_MODIFIER: val += modifier 241 ret.append(val) 242 ret.reverse() 243 return ret 244 245 246 247class NullTranslations: 248 def __init__(self, fp=None): 249 self._info = {} 250 self._charset = None 251 self._output_charset = None 252 self._fallback = None 253 if fp is not None: 254 self._parse(fp) 255 256 def _parse(self, fp): 257 pass 258 259 def add_fallback(self, fallback): 260 if self._fallback: 261 self._fallback.add_fallback(fallback) 262 else: 263 self._fallback = fallback 264 265 def gettext(self, message): 266 if self._fallback: 267 return self._fallback.gettext(message) 268 return message 269 270 def lgettext(self, message): 271 if self._fallback: 272 return self._fallback.lgettext(message) 273 return message 274 275 def ngettext(self, msgid1, msgid2, n): 276 if self._fallback: 277 return self._fallback.ngettext(msgid1, msgid2, n) 278 if n == 1: 279 return msgid1 280 else: 281 return msgid2 282 283 def lngettext(self, msgid1, msgid2, n): 284 if self._fallback: 285 return self._fallback.lngettext(msgid1, msgid2, n) 286 if n == 1: 287 return msgid1 288 else: 289 return msgid2 290 291 def info(self): 292 return self._info 293 294 def charset(self): 295 return self._charset 296 297 def output_charset(self): 298 return self._output_charset 299 300 def set_output_charset(self, charset): 301 self._output_charset = charset 302 303 def install(self, names=None): 304 import builtins 305 builtins.__dict__['_'] = self.gettext 306 if hasattr(names, "__contains__"): 307 if "gettext" in names: 308 builtins.__dict__['gettext'] = builtins.__dict__['_'] 309 if "ngettext" in names: 310 builtins.__dict__['ngettext'] = self.ngettext 311 if "lgettext" in names: 312 builtins.__dict__['lgettext'] = self.lgettext 313 if "lngettext" in names: 314 builtins.__dict__['lngettext'] = self.lngettext 315 316 317class GNUTranslations(NullTranslations): 318 # Magic number of .mo files 319 LE_MAGIC = 0x950412de 320 BE_MAGIC = 0xde120495 321 322 # Acceptable .mo versions 323 VERSIONS = (0, 1) 324 325 def _get_versions(self, version): 326 """Returns a tuple of major version, minor version""" 327 return (version >> 16, version & 0xffff) 328 329 def _parse(self, fp): 330 """Override this method to support alternative .mo formats.""" 331 unpack = struct.unpack 332 filename = getattr(fp, 'name', '') 333 # Parse the .mo file header, which consists of 5 little endian 32 334 # bit words. 335 self._catalog = catalog = {} 336 self.plural = lambda n: int(n != 1) # germanic plural by default 337 buf = fp.read() 338 buflen = len(buf) 339 # Are we big endian or little endian? 340 magic = unpack('<I', buf[:4])[0] 341 if magic == self.LE_MAGIC: 342 version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20]) 343 ii = '<II' 344 elif magic == self.BE_MAGIC: 345 version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20]) 346 ii = '>II' 347 else: 348 raise OSError(0, 'Bad magic number', filename) 349 350 major_version, minor_version = self._get_versions(version) 351 352 if major_version not in self.VERSIONS: 353 raise OSError(0, 'Bad version number ' + str(major_version), filename) 354 355 # Now put all messages from the .mo file buffer into the catalog 356 # dictionary. 357 for i in range(0, msgcount): 358 mlen, moff = unpack(ii, buf[masteridx:masteridx+8]) 359 mend = moff + mlen 360 tlen, toff = unpack(ii, buf[transidx:transidx+8]) 361 tend = toff + tlen 362 if mend < buflen and tend < buflen: 363 msg = buf[moff:mend] 364 tmsg = buf[toff:tend] 365 else: 366 raise OSError(0, 'File is corrupt', filename) 367 # See if we're looking at GNU .mo conventions for metadata 368 if mlen == 0: 369 # Catalog description 370 lastk = None 371 for b_item in tmsg.split('\n'.encode("ascii")): 372 item = b_item.decode().strip() 373 if not item: 374 continue 375 k = v = None 376 if ':' in item: 377 k, v = item.split(':', 1) 378 k = k.strip().lower() 379 v = v.strip() 380 self._info[k] = v 381 lastk = k 382 elif lastk: 383 self._info[lastk] += '\n' + item 384 if k == 'content-type': 385 self._charset = v.split('charset=')[1] 386 elif k == 'plural-forms': 387 v = v.split(';') 388 plural = v[1].split('plural=')[1] 389 self.plural = c2py(plural) 390 # Note: we unconditionally convert both msgids and msgstrs to 391 # Unicode using the character encoding specified in the charset 392 # parameter of the Content-Type header. The gettext documentation 393 # strongly encourages msgids to be us-ascii, but some applications 394 # require alternative encodings (e.g. Zope's ZCML and ZPT). For 395 # traditional gettext applications, the msgid conversion will 396 # cause no problems since us-ascii should always be a subset of 397 # the charset encoding. We may want to fall back to 8-bit msgids 398 # if the Unicode conversion fails. 399 charset = self._charset or 'ascii' 400 if b'\x00' in msg: 401 # Plural forms 402 msgid1, msgid2 = msg.split(b'\x00') 403 tmsg = tmsg.split(b'\x00') 404 msgid1 = str(msgid1, charset) 405 for i, x in enumerate(tmsg): 406 catalog[(msgid1, i)] = str(x, charset) 407 else: 408 catalog[str(msg, charset)] = str(tmsg, charset) 409 # advance to next entry in the seek tables 410 masteridx += 8 411 transidx += 8 412 413 def lgettext(self, message): 414 missing = object() 415 tmsg = self._catalog.get(message, missing) 416 if tmsg is missing: 417 if self._fallback: 418 return self._fallback.lgettext(message) 419 return message 420 if self._output_charset: 421 return tmsg.encode(self._output_charset) 422 return tmsg.encode(locale.getpreferredencoding()) 423 424 def lngettext(self, msgid1, msgid2, n): 425 try: 426 tmsg = self._catalog[(msgid1, self.plural(n))] 427 if self._output_charset: 428 return tmsg.encode(self._output_charset) 429 return tmsg.encode(locale.getpreferredencoding()) 430 except KeyError: 431 if self._fallback: 432 return self._fallback.lngettext(msgid1, msgid2, n) 433 if n == 1: 434 return msgid1 435 else: 436 return msgid2 437 438 def gettext(self, message): 439 missing = object() 440 tmsg = self._catalog.get(message, missing) 441 if tmsg is missing: 442 if self._fallback: 443 return self._fallback.gettext(message) 444 return message 445 return tmsg 446 447 def ngettext(self, msgid1, msgid2, n): 448 try: 449 tmsg = self._catalog[(msgid1, self.plural(n))] 450 except KeyError: 451 if self._fallback: 452 return self._fallback.ngettext(msgid1, msgid2, n) 453 if n == 1: 454 tmsg = msgid1 455 else: 456 tmsg = msgid2 457 return tmsg 458 459 460# Locate a .mo file using the gettext strategy 461def find(domain, localedir=None, languages=None, all=False): 462 # Get some reasonable defaults for arguments that were not supplied 463 if localedir is None: 464 localedir = _default_localedir 465 if languages is None: 466 languages = [] 467 for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'): 468 val = os.environ.get(envar) 469 if val: 470 languages = val.split(':') 471 break 472 if 'C' not in languages: 473 languages.append('C') 474 # now normalize and expand the languages 475 nelangs = [] 476 for lang in languages: 477 for nelang in _expand_lang(lang): 478 if nelang not in nelangs: 479 nelangs.append(nelang) 480 # select a language 481 if all: 482 result = [] 483 else: 484 result = None 485 for lang in nelangs: 486 if lang == 'C': 487 break 488 mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain) 489 if os.path.exists(mofile): 490 if all: 491 result.append(mofile) 492 else: 493 return mofile 494 return result 495 496 497 498# a mapping between absolute .mo file path and Translation object 499_translations = {} 500 501def translation(domain, localedir=None, languages=None, 502 class_=None, fallback=False, codeset=None): 503 if class_ is None: 504 class_ = GNUTranslations 505 mofiles = find(domain, localedir, languages, all=True) 506 if not mofiles: 507 if fallback: 508 return NullTranslations() 509 raise OSError(ENOENT, 'No translation file found for domain', domain) 510 # Avoid opening, reading, and parsing the .mo file after it's been done 511 # once. 512 result = None 513 for mofile in mofiles: 514 key = (class_, os.path.abspath(mofile)) 515 t = _translations.get(key) 516 if t is None: 517 with open(mofile, 'rb') as fp: 518 t = _translations.setdefault(key, class_(fp)) 519 # Copy the translation object to allow setting fallbacks and 520 # output charset. All other instance data is shared with the 521 # cached object. 522 t = copy.copy(t) 523 if codeset: 524 t.set_output_charset(codeset) 525 if result is None: 526 result = t 527 else: 528 result.add_fallback(t) 529 return result 530 531 532def install(domain, localedir=None, codeset=None, names=None): 533 t = translation(domain, localedir, fallback=True, codeset=codeset) 534 t.install(names) 535 536 537 538# a mapping b/w domains and locale directories 539_localedirs = {} 540# a mapping b/w domains and codesets 541_localecodesets = {} 542# current global domain, `messages' used for compatibility w/ GNU gettext 543_current_domain = 'messages' 544 545 546def textdomain(domain=None): 547 global _current_domain 548 if domain is not None: 549 _current_domain = domain 550 return _current_domain 551 552 553def bindtextdomain(domain, localedir=None): 554 global _localedirs 555 if localedir is not None: 556 _localedirs[domain] = localedir 557 return _localedirs.get(domain, _default_localedir) 558 559 560def bind_textdomain_codeset(domain, codeset=None): 561 global _localecodesets 562 if codeset is not None: 563 _localecodesets[domain] = codeset 564 return _localecodesets.get(domain) 565 566 567def dgettext(domain, message): 568 try: 569 t = translation(domain, _localedirs.get(domain, None), 570 codeset=_localecodesets.get(domain)) 571 except OSError: 572 return message 573 return t.gettext(message) 574 575def ldgettext(domain, message): 576 try: 577 t = translation(domain, _localedirs.get(domain, None), 578 codeset=_localecodesets.get(domain)) 579 except OSError: 580 return message 581 return t.lgettext(message) 582 583def dngettext(domain, msgid1, msgid2, n): 584 try: 585 t = translation(domain, _localedirs.get(domain, None), 586 codeset=_localecodesets.get(domain)) 587 except OSError: 588 if n == 1: 589 return msgid1 590 else: 591 return msgid2 592 return t.ngettext(msgid1, msgid2, n) 593 594def ldngettext(domain, msgid1, msgid2, n): 595 try: 596 t = translation(domain, _localedirs.get(domain, None), 597 codeset=_localecodesets.get(domain)) 598 except OSError: 599 if n == 1: 600 return msgid1 601 else: 602 return msgid2 603 return t.lngettext(msgid1, msgid2, n) 604 605def gettext(message): 606 return dgettext(_current_domain, message) 607 608def lgettext(message): 609 return ldgettext(_current_domain, message) 610 611def ngettext(msgid1, msgid2, n): 612 return dngettext(_current_domain, msgid1, msgid2, n) 613 614def lngettext(msgid1, msgid2, n): 615 return ldngettext(_current_domain, msgid1, msgid2, n) 616 617# dcgettext() has been deemed unnecessary and is not implemented. 618 619# James Henstridge's Catalog constructor from GNOME gettext. Documented usage 620# was: 621# 622# import gettext 623# cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR) 624# _ = cat.gettext 625# print _('Hello World') 626 627# The resulting catalog object currently don't support access through a 628# dictionary API, which was supported (but apparently unused) in GNOME 629# gettext. 630 631Catalog = translation 632