1"""Extension API for adding custom tags and behavior.""" 2import pprint 3import re 4import typing as t 5 6from markupsafe import Markup 7 8from . import defaults 9from . import nodes 10from .environment import Environment 11from .exceptions import TemplateAssertionError 12from .exceptions import TemplateSyntaxError 13from .runtime import concat # type: ignore 14from .runtime import Context 15from .runtime import Undefined 16from .utils import import_string 17from .utils import pass_context 18 19if t.TYPE_CHECKING: 20 import typing_extensions as te 21 from .lexer import Token 22 from .lexer import TokenStream 23 from .parser import Parser 24 25 class _TranslationsBasic(te.Protocol): 26 def gettext(self, message: str) -> str: 27 ... 28 29 def ngettext(self, singular: str, plural: str, n: int) -> str: 30 pass 31 32 class _TranslationsContext(_TranslationsBasic): 33 def pgettext(self, context: str, message: str) -> str: 34 ... 35 36 def npgettext(self, context: str, singular: str, plural: str, n: int) -> str: 37 ... 38 39 _SupportedTranslations = t.Union[_TranslationsBasic, _TranslationsContext] 40 41 42# I18N functions available in Jinja templates. If the I18N library 43# provides ugettext, it will be assigned to gettext. 44GETTEXT_FUNCTIONS: t.Tuple[str, ...] = ( 45 "_", 46 "gettext", 47 "ngettext", 48 "pgettext", 49 "npgettext", 50) 51_ws_re = re.compile(r"\s*\n\s*") 52 53 54class Extension: 55 """Extensions can be used to add extra functionality to the Jinja template 56 system at the parser level. Custom extensions are bound to an environment 57 but may not store environment specific data on `self`. The reason for 58 this is that an extension can be bound to another environment (for 59 overlays) by creating a copy and reassigning the `environment` attribute. 60 61 As extensions are created by the environment they cannot accept any 62 arguments for configuration. One may want to work around that by using 63 a factory function, but that is not possible as extensions are identified 64 by their import name. The correct way to configure the extension is 65 storing the configuration values on the environment. Because this way the 66 environment ends up acting as central configuration storage the 67 attributes may clash which is why extensions have to ensure that the names 68 they choose for configuration are not too generic. ``prefix`` for example 69 is a terrible name, ``fragment_cache_prefix`` on the other hand is a good 70 name as includes the name of the extension (fragment cache). 71 """ 72 73 identifier: t.ClassVar[str] 74 75 def __init_subclass__(cls) -> None: 76 cls.identifier = f"{cls.__module__}.{cls.__name__}" 77 78 #: if this extension parses this is the list of tags it's listening to. 79 tags: t.Set[str] = set() 80 81 #: the priority of that extension. This is especially useful for 82 #: extensions that preprocess values. A lower value means higher 83 #: priority. 84 #: 85 #: .. versionadded:: 2.4 86 priority = 100 87 88 def __init__(self, environment: Environment) -> None: 89 self.environment = environment 90 91 def bind(self, environment: Environment) -> "Extension": 92 """Create a copy of this extension bound to another environment.""" 93 rv = object.__new__(self.__class__) 94 rv.__dict__.update(self.__dict__) 95 rv.environment = environment 96 return rv 97 98 def preprocess( 99 self, source: str, name: t.Optional[str], filename: t.Optional[str] = None 100 ) -> str: 101 """This method is called before the actual lexing and can be used to 102 preprocess the source. The `filename` is optional. The return value 103 must be the preprocessed source. 104 """ 105 return source 106 107 def filter_stream( 108 self, stream: "TokenStream" 109 ) -> t.Union["TokenStream", t.Iterable["Token"]]: 110 """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used 111 to filter tokens returned. This method has to return an iterable of 112 :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a 113 :class:`~jinja2.lexer.TokenStream`. 114 """ 115 return stream 116 117 def parse(self, parser: "Parser") -> t.Union[nodes.Node, t.List[nodes.Node]]: 118 """If any of the :attr:`tags` matched this method is called with the 119 parser as first argument. The token the parser stream is pointing at 120 is the name token that matched. This method has to return one or a 121 list of multiple nodes. 122 """ 123 raise NotImplementedError() 124 125 def attr( 126 self, name: str, lineno: t.Optional[int] = None 127 ) -> nodes.ExtensionAttribute: 128 """Return an attribute node for the current extension. This is useful 129 to pass constants on extensions to generated template code. 130 131 :: 132 133 self.attr('_my_attribute', lineno=lineno) 134 """ 135 return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) 136 137 def call_method( 138 self, 139 name: str, 140 args: t.Optional[t.List[nodes.Expr]] = None, 141 kwargs: t.Optional[t.List[nodes.Keyword]] = None, 142 dyn_args: t.Optional[nodes.Expr] = None, 143 dyn_kwargs: t.Optional[nodes.Expr] = None, 144 lineno: t.Optional[int] = None, 145 ) -> nodes.Call: 146 """Call a method of the extension. This is a shortcut for 147 :meth:`attr` + :class:`jinja2.nodes.Call`. 148 """ 149 if args is None: 150 args = [] 151 if kwargs is None: 152 kwargs = [] 153 return nodes.Call( 154 self.attr(name, lineno=lineno), 155 args, 156 kwargs, 157 dyn_args, 158 dyn_kwargs, 159 lineno=lineno, 160 ) 161 162 163@pass_context 164def _gettext_alias( 165 __context: Context, *args: t.Any, **kwargs: t.Any 166) -> t.Union[t.Any, Undefined]: 167 return __context.call(__context.resolve("gettext"), *args, **kwargs) 168 169 170def _make_new_gettext(func: t.Callable[[str], str]) -> t.Callable[..., str]: 171 @pass_context 172 def gettext(__context: Context, __string: str, **variables: t.Any) -> str: 173 rv = __context.call(func, __string) 174 if __context.eval_ctx.autoescape: 175 rv = Markup(rv) 176 # Always treat as a format string, even if there are no 177 # variables. This makes translation strings more consistent 178 # and predictable. This requires escaping 179 return rv % variables # type: ignore 180 181 return gettext 182 183 184def _make_new_ngettext(func: t.Callable[[str, str, int], str]) -> t.Callable[..., str]: 185 @pass_context 186 def ngettext( 187 __context: Context, 188 __singular: str, 189 __plural: str, 190 __num: int, 191 **variables: t.Any, 192 ) -> str: 193 variables.setdefault("num", __num) 194 rv = __context.call(func, __singular, __plural, __num) 195 if __context.eval_ctx.autoescape: 196 rv = Markup(rv) 197 # Always treat as a format string, see gettext comment above. 198 return rv % variables # type: ignore 199 200 return ngettext 201 202 203def _make_new_pgettext(func: t.Callable[[str, str], str]) -> t.Callable[..., str]: 204 @pass_context 205 def pgettext( 206 __context: Context, __string_ctx: str, __string: str, **variables: t.Any 207 ) -> str: 208 variables.setdefault("context", __string_ctx) 209 rv = __context.call(func, __string_ctx, __string) 210 211 if __context.eval_ctx.autoescape: 212 rv = Markup(rv) 213 214 # Always treat as a format string, see gettext comment above. 215 return rv % variables # type: ignore 216 217 return pgettext 218 219 220def _make_new_npgettext( 221 func: t.Callable[[str, str, str, int], str] 222) -> t.Callable[..., str]: 223 @pass_context 224 def npgettext( 225 __context: Context, 226 __string_ctx: str, 227 __singular: str, 228 __plural: str, 229 __num: int, 230 **variables: t.Any, 231 ) -> str: 232 variables.setdefault("context", __string_ctx) 233 variables.setdefault("num", __num) 234 rv = __context.call(func, __string_ctx, __singular, __plural, __num) 235 236 if __context.eval_ctx.autoescape: 237 rv = Markup(rv) 238 239 # Always treat as a format string, see gettext comment above. 240 return rv % variables # type: ignore 241 242 return npgettext 243 244 245class InternationalizationExtension(Extension): 246 """This extension adds gettext support to Jinja.""" 247 248 tags = {"trans"} 249 250 # TODO: the i18n extension is currently reevaluating values in a few 251 # situations. Take this example: 252 # {% trans count=something() %}{{ count }} foo{% pluralize 253 # %}{{ count }} fooss{% endtrans %} 254 # something is called twice here. One time for the gettext value and 255 # the other time for the n-parameter of the ngettext function. 256 257 def __init__(self, environment: Environment) -> None: 258 super().__init__(environment) 259 environment.globals["_"] = _gettext_alias 260 environment.extend( 261 install_gettext_translations=self._install, 262 install_null_translations=self._install_null, 263 install_gettext_callables=self._install_callables, 264 uninstall_gettext_translations=self._uninstall, 265 extract_translations=self._extract, 266 newstyle_gettext=False, 267 ) 268 269 def _install( 270 self, translations: "_SupportedTranslations", newstyle: t.Optional[bool] = None 271 ) -> None: 272 # ugettext and ungettext are preferred in case the I18N library 273 # is providing compatibility with older Python versions. 274 gettext = getattr(translations, "ugettext", None) 275 if gettext is None: 276 gettext = translations.gettext 277 ngettext = getattr(translations, "ungettext", None) 278 if ngettext is None: 279 ngettext = translations.ngettext 280 281 pgettext = getattr(translations, "pgettext", None) 282 npgettext = getattr(translations, "npgettext", None) 283 self._install_callables( 284 gettext, ngettext, newstyle=newstyle, pgettext=pgettext, npgettext=npgettext 285 ) 286 287 def _install_null(self, newstyle: t.Optional[bool] = None) -> None: 288 import gettext 289 290 translations = gettext.NullTranslations() 291 292 if hasattr(translations, "pgettext"): 293 # Python < 3.8 294 pgettext = translations.pgettext # type: ignore 295 else: 296 297 def pgettext(c: str, s: str) -> str: 298 return s 299 300 if hasattr(translations, "npgettext"): 301 npgettext = translations.npgettext # type: ignore 302 else: 303 304 def npgettext(c: str, s: str, p: str, n: int) -> str: 305 return s if n == 1 else p 306 307 self._install_callables( 308 gettext=translations.gettext, 309 ngettext=translations.ngettext, 310 newstyle=newstyle, 311 pgettext=pgettext, 312 npgettext=npgettext, 313 ) 314 315 def _install_callables( 316 self, 317 gettext: t.Callable[[str], str], 318 ngettext: t.Callable[[str, str, int], str], 319 newstyle: t.Optional[bool] = None, 320 pgettext: t.Optional[t.Callable[[str, str], str]] = None, 321 npgettext: t.Optional[t.Callable[[str, str, str, int], str]] = None, 322 ) -> None: 323 if newstyle is not None: 324 self.environment.newstyle_gettext = newstyle # type: ignore 325 if self.environment.newstyle_gettext: # type: ignore 326 gettext = _make_new_gettext(gettext) 327 ngettext = _make_new_ngettext(ngettext) 328 329 if pgettext is not None: 330 pgettext = _make_new_pgettext(pgettext) 331 332 if npgettext is not None: 333 npgettext = _make_new_npgettext(npgettext) 334 335 self.environment.globals.update( 336 gettext=gettext, ngettext=ngettext, pgettext=pgettext, npgettext=npgettext 337 ) 338 339 def _uninstall(self, translations: "_SupportedTranslations") -> None: 340 for key in ("gettext", "ngettext", "pgettext", "npgettext"): 341 self.environment.globals.pop(key, None) 342 343 def _extract( 344 self, 345 source: t.Union[str, nodes.Template], 346 gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS, 347 ) -> t.Iterator[ 348 t.Tuple[int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]] 349 ]: 350 if isinstance(source, str): 351 source = self.environment.parse(source) 352 return extract_from_ast(source, gettext_functions) 353 354 def parse(self, parser: "Parser") -> t.Union[nodes.Node, t.List[nodes.Node]]: 355 """Parse a translatable tag.""" 356 lineno = next(parser.stream).lineno 357 358 context = None 359 context_token = parser.stream.next_if("string") 360 361 if context_token is not None: 362 context = context_token.value 363 364 # find all the variables referenced. Additionally a variable can be 365 # defined in the body of the trans block too, but this is checked at 366 # a later state. 367 plural_expr: t.Optional[nodes.Expr] = None 368 plural_expr_assignment: t.Optional[nodes.Assign] = None 369 num_called_num = False 370 variables: t.Dict[str, nodes.Expr] = {} 371 trimmed = None 372 while parser.stream.current.type != "block_end": 373 if variables: 374 parser.stream.expect("comma") 375 376 # skip colon for python compatibility 377 if parser.stream.skip_if("colon"): 378 break 379 380 token = parser.stream.expect("name") 381 if token.value in variables: 382 parser.fail( 383 f"translatable variable {token.value!r} defined twice.", 384 token.lineno, 385 exc=TemplateAssertionError, 386 ) 387 388 # expressions 389 if parser.stream.current.type == "assign": 390 next(parser.stream) 391 variables[token.value] = var = parser.parse_expression() 392 elif trimmed is None and token.value in ("trimmed", "notrimmed"): 393 trimmed = token.value == "trimmed" 394 continue 395 else: 396 variables[token.value] = var = nodes.Name(token.value, "load") 397 398 if plural_expr is None: 399 if isinstance(var, nodes.Call): 400 plural_expr = nodes.Name("_trans", "load") 401 variables[token.value] = plural_expr 402 plural_expr_assignment = nodes.Assign( 403 nodes.Name("_trans", "store"), var 404 ) 405 else: 406 plural_expr = var 407 num_called_num = token.value == "num" 408 409 parser.stream.expect("block_end") 410 411 plural = None 412 have_plural = False 413 referenced = set() 414 415 # now parse until endtrans or pluralize 416 singular_names, singular = self._parse_block(parser, True) 417 if singular_names: 418 referenced.update(singular_names) 419 if plural_expr is None: 420 plural_expr = nodes.Name(singular_names[0], "load") 421 num_called_num = singular_names[0] == "num" 422 423 # if we have a pluralize block, we parse that too 424 if parser.stream.current.test("name:pluralize"): 425 have_plural = True 426 next(parser.stream) 427 if parser.stream.current.type != "block_end": 428 token = parser.stream.expect("name") 429 if token.value not in variables: 430 parser.fail( 431 f"unknown variable {token.value!r} for pluralization", 432 token.lineno, 433 exc=TemplateAssertionError, 434 ) 435 plural_expr = variables[token.value] 436 num_called_num = token.value == "num" 437 parser.stream.expect("block_end") 438 plural_names, plural = self._parse_block(parser, False) 439 next(parser.stream) 440 referenced.update(plural_names) 441 else: 442 next(parser.stream) 443 444 # register free names as simple name expressions 445 for name in referenced: 446 if name not in variables: 447 variables[name] = nodes.Name(name, "load") 448 449 if not have_plural: 450 plural_expr = None 451 elif plural_expr is None: 452 parser.fail("pluralize without variables", lineno) 453 454 if trimmed is None: 455 trimmed = self.environment.policies["ext.i18n.trimmed"] 456 if trimmed: 457 singular = self._trim_whitespace(singular) 458 if plural: 459 plural = self._trim_whitespace(plural) 460 461 node = self._make_node( 462 singular, 463 plural, 464 context, 465 variables, 466 plural_expr, 467 bool(referenced), 468 num_called_num and have_plural, 469 ) 470 node.set_lineno(lineno) 471 if plural_expr_assignment is not None: 472 return [plural_expr_assignment, node] 473 else: 474 return node 475 476 def _trim_whitespace(self, string: str, _ws_re: t.Pattern[str] = _ws_re) -> str: 477 return _ws_re.sub(" ", string.strip()) 478 479 def _parse_block( 480 self, parser: "Parser", allow_pluralize: bool 481 ) -> t.Tuple[t.List[str], str]: 482 """Parse until the next block tag with a given name.""" 483 referenced = [] 484 buf = [] 485 486 while True: 487 if parser.stream.current.type == "data": 488 buf.append(parser.stream.current.value.replace("%", "%%")) 489 next(parser.stream) 490 elif parser.stream.current.type == "variable_begin": 491 next(parser.stream) 492 name = parser.stream.expect("name").value 493 referenced.append(name) 494 buf.append(f"%({name})s") 495 parser.stream.expect("variable_end") 496 elif parser.stream.current.type == "block_begin": 497 next(parser.stream) 498 if parser.stream.current.test("name:endtrans"): 499 break 500 elif parser.stream.current.test("name:pluralize"): 501 if allow_pluralize: 502 break 503 parser.fail( 504 "a translatable section can have only one pluralize section" 505 ) 506 parser.fail( 507 "control structures in translatable sections are not allowed" 508 ) 509 elif parser.stream.eos: 510 parser.fail("unclosed translation block") 511 else: 512 raise RuntimeError("internal parser error") 513 514 return referenced, concat(buf) 515 516 def _make_node( 517 self, 518 singular: str, 519 plural: t.Optional[str], 520 context: t.Optional[str], 521 variables: t.Dict[str, nodes.Expr], 522 plural_expr: t.Optional[nodes.Expr], 523 vars_referenced: bool, 524 num_called_num: bool, 525 ) -> nodes.Output: 526 """Generates a useful node from the data provided.""" 527 newstyle = self.environment.newstyle_gettext # type: ignore 528 node: nodes.Expr 529 530 # no variables referenced? no need to escape for old style 531 # gettext invocations only if there are vars. 532 if not vars_referenced and not newstyle: 533 singular = singular.replace("%%", "%") 534 if plural: 535 plural = plural.replace("%%", "%") 536 537 func_name = "gettext" 538 func_args: t.List[nodes.Expr] = [nodes.Const(singular)] 539 540 if context is not None: 541 func_args.insert(0, nodes.Const(context)) 542 func_name = f"p{func_name}" 543 544 if plural_expr is not None: 545 func_name = f"n{func_name}" 546 func_args.extend((nodes.Const(plural), plural_expr)) 547 548 node = nodes.Call(nodes.Name(func_name, "load"), func_args, [], None, None) 549 550 # in case newstyle gettext is used, the method is powerful 551 # enough to handle the variable expansion and autoescape 552 # handling itself 553 if newstyle: 554 for key, value in variables.items(): 555 # the function adds that later anyways in case num was 556 # called num, so just skip it. 557 if num_called_num and key == "num": 558 continue 559 node.kwargs.append(nodes.Keyword(key, value)) 560 561 # otherwise do that here 562 else: 563 # mark the return value as safe if we are in an 564 # environment with autoescaping turned on 565 node = nodes.MarkSafeIfAutoescape(node) 566 if variables: 567 node = nodes.Mod( 568 node, 569 nodes.Dict( 570 [ 571 nodes.Pair(nodes.Const(key), value) 572 for key, value in variables.items() 573 ] 574 ), 575 ) 576 return nodes.Output([node]) 577 578 579class ExprStmtExtension(Extension): 580 """Adds a `do` tag to Jinja that works like the print statement just 581 that it doesn't print the return value. 582 """ 583 584 tags = {"do"} 585 586 def parse(self, parser: "Parser") -> nodes.ExprStmt: 587 node = nodes.ExprStmt(lineno=next(parser.stream).lineno) 588 node.node = parser.parse_tuple() 589 return node 590 591 592class LoopControlExtension(Extension): 593 """Adds break and continue to the template engine.""" 594 595 tags = {"break", "continue"} 596 597 def parse(self, parser: "Parser") -> t.Union[nodes.Break, nodes.Continue]: 598 token = next(parser.stream) 599 if token.value == "break": 600 return nodes.Break(lineno=token.lineno) 601 return nodes.Continue(lineno=token.lineno) 602 603 604class DebugExtension(Extension): 605 """A ``{% debug %}`` tag that dumps the available variables, 606 filters, and tests. 607 608 .. code-block:: html+jinja 609 610 <pre>{% debug %}</pre> 611 612 .. code-block:: text 613 614 {'context': {'cycler': <class 'jinja2.utils.Cycler'>, 615 ..., 616 'namespace': <class 'jinja2.utils.Namespace'>}, 617 'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd', 618 ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'], 619 'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined', 620 ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']} 621 622 .. versionadded:: 2.11.0 623 """ 624 625 tags = {"debug"} 626 627 def parse(self, parser: "Parser") -> nodes.Output: 628 lineno = parser.stream.expect("name:debug").lineno 629 context = nodes.ContextReference() 630 result = self.call_method("_render", [context], lineno=lineno) 631 return nodes.Output([result], lineno=lineno) 632 633 def _render(self, context: Context) -> str: 634 result = { 635 "context": context.get_all(), 636 "filters": sorted(self.environment.filters.keys()), 637 "tests": sorted(self.environment.tests.keys()), 638 } 639 640 # Set the depth since the intent is to show the top few names. 641 return pprint.pformat(result, depth=3, compact=True) 642 643 644def extract_from_ast( 645 ast: nodes.Template, 646 gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS, 647 babel_style: bool = True, 648) -> t.Iterator[ 649 t.Tuple[int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]]] 650]: 651 """Extract localizable strings from the given template node. Per 652 default this function returns matches in babel style that means non string 653 parameters as well as keyword arguments are returned as `None`. This 654 allows Babel to figure out what you really meant if you are using 655 gettext functions that allow keyword arguments for placeholder expansion. 656 If you don't want that behavior set the `babel_style` parameter to `False` 657 which causes only strings to be returned and parameters are always stored 658 in tuples. As a consequence invalid gettext calls (calls without a single 659 string parameter or string parameters after non-string parameters) are 660 skipped. 661 662 This example explains the behavior: 663 664 >>> from jinja2 import Environment 665 >>> env = Environment() 666 >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}') 667 >>> list(extract_from_ast(node)) 668 [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))] 669 >>> list(extract_from_ast(node, babel_style=False)) 670 [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))] 671 672 For every string found this function yields a ``(lineno, function, 673 message)`` tuple, where: 674 675 * ``lineno`` is the number of the line on which the string was found, 676 * ``function`` is the name of the ``gettext`` function used (if the 677 string was extracted from embedded Python code), and 678 * ``message`` is the string, or a tuple of strings for functions 679 with multiple string arguments. 680 681 This extraction function operates on the AST and is because of that unable 682 to extract any comments. For comment support you have to use the babel 683 extraction interface or extract comments yourself. 684 """ 685 out: t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]] 686 687 for node in ast.find_all(nodes.Call): 688 if ( 689 not isinstance(node.node, nodes.Name) 690 or node.node.name not in gettext_functions 691 ): 692 continue 693 694 strings: t.List[t.Optional[str]] = [] 695 696 for arg in node.args: 697 if isinstance(arg, nodes.Const) and isinstance(arg.value, str): 698 strings.append(arg.value) 699 else: 700 strings.append(None) 701 702 for _ in node.kwargs: 703 strings.append(None) 704 if node.dyn_args is not None: 705 strings.append(None) 706 if node.dyn_kwargs is not None: 707 strings.append(None) 708 709 if not babel_style: 710 out = tuple(x for x in strings if x is not None) 711 712 if not out: 713 continue 714 else: 715 if len(strings) == 1: 716 out = strings[0] 717 else: 718 out = tuple(strings) 719 720 yield node.lineno, node.node.name, out 721 722 723class _CommentFinder: 724 """Helper class to find comments in a token stream. Can only 725 find comments for gettext calls forwards. Once the comment 726 from line 4 is found, a comment for line 1 will not return a 727 usable value. 728 """ 729 730 def __init__( 731 self, tokens: t.Sequence[t.Tuple[int, str, str]], comment_tags: t.Sequence[str] 732 ) -> None: 733 self.tokens = tokens 734 self.comment_tags = comment_tags 735 self.offset = 0 736 self.last_lineno = 0 737 738 def find_backwards(self, offset: int) -> t.List[str]: 739 try: 740 for _, token_type, token_value in reversed( 741 self.tokens[self.offset : offset] 742 ): 743 if token_type in ("comment", "linecomment"): 744 try: 745 prefix, comment = token_value.split(None, 1) 746 except ValueError: 747 continue 748 if prefix in self.comment_tags: 749 return [comment.rstrip()] 750 return [] 751 finally: 752 self.offset = offset 753 754 def find_comments(self, lineno: int) -> t.List[str]: 755 if not self.comment_tags or self.last_lineno > lineno: 756 return [] 757 for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]): 758 if token_lineno > lineno: 759 return self.find_backwards(self.offset + idx) 760 return self.find_backwards(len(self.tokens)) 761 762 763def babel_extract( 764 fileobj: t.BinaryIO, 765 keywords: t.Sequence[str], 766 comment_tags: t.Sequence[str], 767 options: t.Dict[str, t.Any], 768) -> t.Iterator[ 769 t.Tuple[ 770 int, str, t.Union[t.Optional[str], t.Tuple[t.Optional[str], ...]], t.List[str] 771 ] 772]: 773 """Babel extraction method for Jinja templates. 774 775 .. versionchanged:: 2.3 776 Basic support for translation comments was added. If `comment_tags` 777 is now set to a list of keywords for extraction, the extractor will 778 try to find the best preceding comment that begins with one of the 779 keywords. For best results, make sure to not have more than one 780 gettext call in one line of code and the matching comment in the 781 same line or the line before. 782 783 .. versionchanged:: 2.5.1 784 The `newstyle_gettext` flag can be set to `True` to enable newstyle 785 gettext calls. 786 787 .. versionchanged:: 2.7 788 A `silent` option can now be provided. If set to `False` template 789 syntax errors are propagated instead of being ignored. 790 791 :param fileobj: the file-like object the messages should be extracted from 792 :param keywords: a list of keywords (i.e. function names) that should be 793 recognized as translation functions 794 :param comment_tags: a list of translator tags to search for and include 795 in the results. 796 :param options: a dictionary of additional options (optional) 797 :return: an iterator over ``(lineno, funcname, message, comments)`` tuples. 798 (comments will be empty currently) 799 """ 800 extensions: t.Dict[t.Type[Extension], None] = {} 801 802 for extension_name in options.get("extensions", "").split(","): 803 extension_name = extension_name.strip() 804 805 if not extension_name: 806 continue 807 808 extensions[import_string(extension_name)] = None 809 810 if InternationalizationExtension not in extensions: 811 extensions[InternationalizationExtension] = None 812 813 def getbool(options: t.Mapping[str, str], key: str, default: bool = False) -> bool: 814 return options.get(key, str(default)).lower() in {"1", "on", "yes", "true"} 815 816 silent = getbool(options, "silent", True) 817 environment = Environment( 818 options.get("block_start_string", defaults.BLOCK_START_STRING), 819 options.get("block_end_string", defaults.BLOCK_END_STRING), 820 options.get("variable_start_string", defaults.VARIABLE_START_STRING), 821 options.get("variable_end_string", defaults.VARIABLE_END_STRING), 822 options.get("comment_start_string", defaults.COMMENT_START_STRING), 823 options.get("comment_end_string", defaults.COMMENT_END_STRING), 824 options.get("line_statement_prefix") or defaults.LINE_STATEMENT_PREFIX, 825 options.get("line_comment_prefix") or defaults.LINE_COMMENT_PREFIX, 826 getbool(options, "trim_blocks", defaults.TRIM_BLOCKS), 827 getbool(options, "lstrip_blocks", defaults.LSTRIP_BLOCKS), 828 defaults.NEWLINE_SEQUENCE, 829 getbool(options, "keep_trailing_newline", defaults.KEEP_TRAILING_NEWLINE), 830 tuple(extensions), 831 cache_size=0, 832 auto_reload=False, 833 ) 834 835 if getbool(options, "trimmed"): 836 environment.policies["ext.i18n.trimmed"] = True 837 if getbool(options, "newstyle_gettext"): 838 environment.newstyle_gettext = True # type: ignore 839 840 source = fileobj.read().decode(options.get("encoding", "utf-8")) 841 try: 842 node = environment.parse(source) 843 tokens = list(environment.lex(environment.preprocess(source))) 844 except TemplateSyntaxError: 845 if not silent: 846 raise 847 # skip templates with syntax errors 848 return 849 850 finder = _CommentFinder(tokens, comment_tags) 851 for lineno, func, message in extract_from_ast(node, keywords): 852 yield lineno, func, message, finder.find_comments(lineno) 853 854 855#: nicer import names 856i18n = InternationalizationExtension 857do = ExprStmtExtension 858loopcontrols = LoopControlExtension 859debug = DebugExtension 860