1"""Implements a Jinja / Python combination lexer. The ``Lexer`` class 2is used to do some preprocessing. It filters out invalid operators like 3the bitshift operators we don't allow in templates. It separates 4template code and python code in expressions. 5""" 6import re 7import typing as t 8from ast import literal_eval 9from collections import deque 10from sys import intern 11 12from ._identifier import pattern as name_re 13from .exceptions import TemplateSyntaxError 14from .utils import LRUCache 15 16if t.TYPE_CHECKING: 17 import typing_extensions as te 18 from .environment import Environment 19 20# cache for the lexers. Exists in order to be able to have multiple 21# environments with the same lexer 22_lexer_cache: t.MutableMapping[t.Tuple, "Lexer"] = LRUCache(50) # type: ignore 23 24# static regular expressions 25whitespace_re = re.compile(r"\s+") 26newline_re = re.compile(r"(\r\n|\r|\n)") 27string_re = re.compile( 28 r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S 29) 30integer_re = re.compile( 31 r""" 32 ( 33 0b(_?[0-1])+ # binary 34 | 35 0o(_?[0-7])+ # octal 36 | 37 0x(_?[\da-f])+ # hex 38 | 39 [1-9](_?\d)* # decimal 40 | 41 0(_?0)* # decimal zero 42 ) 43 """, 44 re.IGNORECASE | re.VERBOSE, 45) 46float_re = re.compile( 47 r""" 48 (?<!\.) # doesn't start with a . 49 (\d+_)*\d+ # digits, possibly _ separated 50 ( 51 (\.(\d+_)*\d+)? # optional fractional part 52 e[+\-]?(\d+_)*\d+ # exponent part 53 | 54 \.(\d+_)*\d+ # required fractional part 55 ) 56 """, 57 re.IGNORECASE | re.VERBOSE, 58) 59 60# internal the tokens and keep references to them 61TOKEN_ADD = intern("add") 62TOKEN_ASSIGN = intern("assign") 63TOKEN_COLON = intern("colon") 64TOKEN_COMMA = intern("comma") 65TOKEN_DIV = intern("div") 66TOKEN_DOT = intern("dot") 67TOKEN_EQ = intern("eq") 68TOKEN_FLOORDIV = intern("floordiv") 69TOKEN_GT = intern("gt") 70TOKEN_GTEQ = intern("gteq") 71TOKEN_LBRACE = intern("lbrace") 72TOKEN_LBRACKET = intern("lbracket") 73TOKEN_LPAREN = intern("lparen") 74TOKEN_LT = intern("lt") 75TOKEN_LTEQ = intern("lteq") 76TOKEN_MOD = intern("mod") 77TOKEN_MUL = intern("mul") 78TOKEN_NE = intern("ne") 79TOKEN_PIPE = intern("pipe") 80TOKEN_POW = intern("pow") 81TOKEN_RBRACE = intern("rbrace") 82TOKEN_RBRACKET = intern("rbracket") 83TOKEN_RPAREN = intern("rparen") 84TOKEN_SEMICOLON = intern("semicolon") 85TOKEN_SUB = intern("sub") 86TOKEN_TILDE = intern("tilde") 87TOKEN_WHITESPACE = intern("whitespace") 88TOKEN_FLOAT = intern("float") 89TOKEN_INTEGER = intern("integer") 90TOKEN_NAME = intern("name") 91TOKEN_STRING = intern("string") 92TOKEN_OPERATOR = intern("operator") 93TOKEN_BLOCK_BEGIN = intern("block_begin") 94TOKEN_BLOCK_END = intern("block_end") 95TOKEN_VARIABLE_BEGIN = intern("variable_begin") 96TOKEN_VARIABLE_END = intern("variable_end") 97TOKEN_RAW_BEGIN = intern("raw_begin") 98TOKEN_RAW_END = intern("raw_end") 99TOKEN_COMMENT_BEGIN = intern("comment_begin") 100TOKEN_COMMENT_END = intern("comment_end") 101TOKEN_COMMENT = intern("comment") 102TOKEN_LINESTATEMENT_BEGIN = intern("linestatement_begin") 103TOKEN_LINESTATEMENT_END = intern("linestatement_end") 104TOKEN_LINECOMMENT_BEGIN = intern("linecomment_begin") 105TOKEN_LINECOMMENT_END = intern("linecomment_end") 106TOKEN_LINECOMMENT = intern("linecomment") 107TOKEN_DATA = intern("data") 108TOKEN_INITIAL = intern("initial") 109TOKEN_EOF = intern("eof") 110 111# bind operators to token types 112operators = { 113 "+": TOKEN_ADD, 114 "-": TOKEN_SUB, 115 "/": TOKEN_DIV, 116 "//": TOKEN_FLOORDIV, 117 "*": TOKEN_MUL, 118 "%": TOKEN_MOD, 119 "**": TOKEN_POW, 120 "~": TOKEN_TILDE, 121 "[": TOKEN_LBRACKET, 122 "]": TOKEN_RBRACKET, 123 "(": TOKEN_LPAREN, 124 ")": TOKEN_RPAREN, 125 "{": TOKEN_LBRACE, 126 "}": TOKEN_RBRACE, 127 "==": TOKEN_EQ, 128 "!=": TOKEN_NE, 129 ">": TOKEN_GT, 130 ">=": TOKEN_GTEQ, 131 "<": TOKEN_LT, 132 "<=": TOKEN_LTEQ, 133 "=": TOKEN_ASSIGN, 134 ".": TOKEN_DOT, 135 ":": TOKEN_COLON, 136 "|": TOKEN_PIPE, 137 ",": TOKEN_COMMA, 138 ";": TOKEN_SEMICOLON, 139} 140 141reverse_operators = {v: k for k, v in operators.items()} 142assert len(operators) == len(reverse_operators), "operators dropped" 143operator_re = re.compile( 144 f"({'|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))})" 145) 146 147ignored_tokens = frozenset( 148 [ 149 TOKEN_COMMENT_BEGIN, 150 TOKEN_COMMENT, 151 TOKEN_COMMENT_END, 152 TOKEN_WHITESPACE, 153 TOKEN_LINECOMMENT_BEGIN, 154 TOKEN_LINECOMMENT_END, 155 TOKEN_LINECOMMENT, 156 ] 157) 158ignore_if_empty = frozenset( 159 [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT] 160) 161 162 163def _describe_token_type(token_type: str) -> str: 164 if token_type in reverse_operators: 165 return reverse_operators[token_type] 166 167 return { 168 TOKEN_COMMENT_BEGIN: "begin of comment", 169 TOKEN_COMMENT_END: "end of comment", 170 TOKEN_COMMENT: "comment", 171 TOKEN_LINECOMMENT: "comment", 172 TOKEN_BLOCK_BEGIN: "begin of statement block", 173 TOKEN_BLOCK_END: "end of statement block", 174 TOKEN_VARIABLE_BEGIN: "begin of print statement", 175 TOKEN_VARIABLE_END: "end of print statement", 176 TOKEN_LINESTATEMENT_BEGIN: "begin of line statement", 177 TOKEN_LINESTATEMENT_END: "end of line statement", 178 TOKEN_DATA: "template data / text", 179 TOKEN_EOF: "end of template", 180 }.get(token_type, token_type) 181 182 183def describe_token(token: "Token") -> str: 184 """Returns a description of the token.""" 185 if token.type == TOKEN_NAME: 186 return token.value 187 188 return _describe_token_type(token.type) 189 190 191def describe_token_expr(expr: str) -> str: 192 """Like `describe_token` but for token expressions.""" 193 if ":" in expr: 194 type, value = expr.split(":", 1) 195 196 if type == TOKEN_NAME: 197 return value 198 else: 199 type = expr 200 201 return _describe_token_type(type) 202 203 204def count_newlines(value: str) -> int: 205 """Count the number of newline characters in the string. This is 206 useful for extensions that filter a stream. 207 """ 208 return len(newline_re.findall(value)) 209 210 211def compile_rules(environment: "Environment") -> t.List[t.Tuple[str, str]]: 212 """Compiles all the rules from the environment into a list of rules.""" 213 e = re.escape 214 rules = [ 215 ( 216 len(environment.comment_start_string), 217 TOKEN_COMMENT_BEGIN, 218 e(environment.comment_start_string), 219 ), 220 ( 221 len(environment.block_start_string), 222 TOKEN_BLOCK_BEGIN, 223 e(environment.block_start_string), 224 ), 225 ( 226 len(environment.variable_start_string), 227 TOKEN_VARIABLE_BEGIN, 228 e(environment.variable_start_string), 229 ), 230 ] 231 232 if environment.line_statement_prefix is not None: 233 rules.append( 234 ( 235 len(environment.line_statement_prefix), 236 TOKEN_LINESTATEMENT_BEGIN, 237 r"^[ \t\v]*" + e(environment.line_statement_prefix), 238 ) 239 ) 240 if environment.line_comment_prefix is not None: 241 rules.append( 242 ( 243 len(environment.line_comment_prefix), 244 TOKEN_LINECOMMENT_BEGIN, 245 r"(?:^|(?<=\S))[^\S\r\n]*" + e(environment.line_comment_prefix), 246 ) 247 ) 248 249 return [x[1:] for x in sorted(rules, reverse=True)] 250 251 252class Failure: 253 """Class that raises a `TemplateSyntaxError` if called. 254 Used by the `Lexer` to specify known errors. 255 """ 256 257 def __init__( 258 self, message: str, cls: t.Type[TemplateSyntaxError] = TemplateSyntaxError 259 ) -> None: 260 self.message = message 261 self.error_class = cls 262 263 def __call__(self, lineno: int, filename: str) -> "te.NoReturn": 264 raise self.error_class(self.message, lineno, filename) 265 266 267class Token(t.NamedTuple): 268 lineno: int 269 type: str 270 value: str 271 272 def __str__(self) -> str: 273 return describe_token(self) 274 275 def test(self, expr: str) -> bool: 276 """Test a token against a token expression. This can either be a 277 token type or ``'token_type:token_value'``. This can only test 278 against string values and types. 279 """ 280 # here we do a regular string equality check as test_any is usually 281 # passed an iterable of not interned strings. 282 if self.type == expr: 283 return True 284 285 if ":" in expr: 286 return expr.split(":", 1) == [self.type, self.value] 287 288 return False 289 290 def test_any(self, *iterable: str) -> bool: 291 """Test against multiple token expressions.""" 292 return any(self.test(expr) for expr in iterable) 293 294 295class TokenStreamIterator: 296 """The iterator for tokenstreams. Iterate over the stream 297 until the eof token is reached. 298 """ 299 300 def __init__(self, stream: "TokenStream") -> None: 301 self.stream = stream 302 303 def __iter__(self) -> "TokenStreamIterator": 304 return self 305 306 def __next__(self) -> Token: 307 token = self.stream.current 308 309 if token.type is TOKEN_EOF: 310 self.stream.close() 311 raise StopIteration 312 313 next(self.stream) 314 return token 315 316 317class TokenStream: 318 """A token stream is an iterable that yields :class:`Token`\\s. The 319 parser however does not iterate over it but calls :meth:`next` to go 320 one token ahead. The current active token is stored as :attr:`current`. 321 """ 322 323 def __init__( 324 self, 325 generator: t.Iterable[Token], 326 name: t.Optional[str], 327 filename: t.Optional[str], 328 ): 329 self._iter = iter(generator) 330 self._pushed: "te.Deque[Token]" = deque() 331 self.name = name 332 self.filename = filename 333 self.closed = False 334 self.current = Token(1, TOKEN_INITIAL, "") 335 next(self) 336 337 def __iter__(self) -> TokenStreamIterator: 338 return TokenStreamIterator(self) 339 340 def __bool__(self) -> bool: 341 return bool(self._pushed) or self.current.type is not TOKEN_EOF 342 343 @property 344 def eos(self) -> bool: 345 """Are we at the end of the stream?""" 346 return not self 347 348 def push(self, token: Token) -> None: 349 """Push a token back to the stream.""" 350 self._pushed.append(token) 351 352 def look(self) -> Token: 353 """Look at the next token.""" 354 old_token = next(self) 355 result = self.current 356 self.push(result) 357 self.current = old_token 358 return result 359 360 def skip(self, n: int = 1) -> None: 361 """Got n tokens ahead.""" 362 for _ in range(n): 363 next(self) 364 365 def next_if(self, expr: str) -> t.Optional[Token]: 366 """Perform the token test and return the token if it matched. 367 Otherwise the return value is `None`. 368 """ 369 if self.current.test(expr): 370 return next(self) 371 372 return None 373 374 def skip_if(self, expr: str) -> bool: 375 """Like :meth:`next_if` but only returns `True` or `False`.""" 376 return self.next_if(expr) is not None 377 378 def __next__(self) -> Token: 379 """Go one token ahead and return the old one. 380 381 Use the built-in :func:`next` instead of calling this directly. 382 """ 383 rv = self.current 384 385 if self._pushed: 386 self.current = self._pushed.popleft() 387 elif self.current.type is not TOKEN_EOF: 388 try: 389 self.current = next(self._iter) 390 except StopIteration: 391 self.close() 392 393 return rv 394 395 def close(self) -> None: 396 """Close the stream.""" 397 self.current = Token(self.current.lineno, TOKEN_EOF, "") 398 self._iter = iter(()) 399 self.closed = True 400 401 def expect(self, expr: str) -> Token: 402 """Expect a given token type and return it. This accepts the same 403 argument as :meth:`jinja2.lexer.Token.test`. 404 """ 405 if not self.current.test(expr): 406 expr = describe_token_expr(expr) 407 408 if self.current.type is TOKEN_EOF: 409 raise TemplateSyntaxError( 410 f"unexpected end of template, expected {expr!r}.", 411 self.current.lineno, 412 self.name, 413 self.filename, 414 ) 415 416 raise TemplateSyntaxError( 417 f"expected token {expr!r}, got {describe_token(self.current)!r}", 418 self.current.lineno, 419 self.name, 420 self.filename, 421 ) 422 423 return next(self) 424 425 426def get_lexer(environment: "Environment") -> "Lexer": 427 """Return a lexer which is probably cached.""" 428 key = ( 429 environment.block_start_string, 430 environment.block_end_string, 431 environment.variable_start_string, 432 environment.variable_end_string, 433 environment.comment_start_string, 434 environment.comment_end_string, 435 environment.line_statement_prefix, 436 environment.line_comment_prefix, 437 environment.trim_blocks, 438 environment.lstrip_blocks, 439 environment.newline_sequence, 440 environment.keep_trailing_newline, 441 ) 442 lexer = _lexer_cache.get(key) 443 444 if lexer is None: 445 _lexer_cache[key] = lexer = Lexer(environment) 446 447 return lexer 448 449 450class OptionalLStrip(tuple): 451 """A special tuple for marking a point in the state that can have 452 lstrip applied. 453 """ 454 455 __slots__ = () 456 457 # Even though it looks like a no-op, creating instances fails 458 # without this. 459 def __new__(cls, *members, **kwargs): # type: ignore 460 return super().__new__(cls, members) 461 462 463class _Rule(t.NamedTuple): 464 pattern: t.Pattern[str] 465 tokens: t.Union[str, t.Tuple[str, ...], t.Tuple[Failure]] 466 command: t.Optional[str] 467 468 469class Lexer: 470 """Class that implements a lexer for a given environment. Automatically 471 created by the environment class, usually you don't have to do that. 472 473 Note that the lexer is not automatically bound to an environment. 474 Multiple environments can share the same lexer. 475 """ 476 477 def __init__(self, environment: "Environment") -> None: 478 # shortcuts 479 e = re.escape 480 481 def c(x: str) -> t.Pattern[str]: 482 return re.compile(x, re.M | re.S) 483 484 # lexing rules for tags 485 tag_rules: t.List[_Rule] = [ 486 _Rule(whitespace_re, TOKEN_WHITESPACE, None), 487 _Rule(float_re, TOKEN_FLOAT, None), 488 _Rule(integer_re, TOKEN_INTEGER, None), 489 _Rule(name_re, TOKEN_NAME, None), 490 _Rule(string_re, TOKEN_STRING, None), 491 _Rule(operator_re, TOKEN_OPERATOR, None), 492 ] 493 494 # assemble the root lexing rule. because "|" is ungreedy 495 # we have to sort by length so that the lexer continues working 496 # as expected when we have parsing rules like <% for block and 497 # <%= for variables. (if someone wants asp like syntax) 498 # variables are just part of the rules if variable processing 499 # is required. 500 root_tag_rules = compile_rules(environment) 501 502 block_start_re = e(environment.block_start_string) 503 block_end_re = e(environment.block_end_string) 504 comment_end_re = e(environment.comment_end_string) 505 variable_end_re = e(environment.variable_end_string) 506 507 # block suffix if trimming is enabled 508 block_suffix_re = "\\n?" if environment.trim_blocks else "" 509 510 self.lstrip_blocks = environment.lstrip_blocks 511 512 self.newline_sequence = environment.newline_sequence 513 self.keep_trailing_newline = environment.keep_trailing_newline 514 515 root_raw_re = ( 516 rf"(?P<raw_begin>{block_start_re}(\-|\+|)\s*raw\s*" 517 rf"(?:\-{block_end_re}\s*|{block_end_re}))" 518 ) 519 root_parts_re = "|".join( 520 [root_raw_re] + [rf"(?P<{n}>{r}(\-|\+|))" for n, r in root_tag_rules] 521 ) 522 523 # global lexing rules 524 self.rules: t.Dict[str, t.List[_Rule]] = { 525 "root": [ 526 # directives 527 _Rule( 528 c(rf"(.*?)(?:{root_parts_re})"), 529 OptionalLStrip(TOKEN_DATA, "#bygroup"), # type: ignore 530 "#bygroup", 531 ), 532 # data 533 _Rule(c(".+"), TOKEN_DATA, None), 534 ], 535 # comments 536 TOKEN_COMMENT_BEGIN: [ 537 _Rule( 538 c( 539 rf"(.*?)((?:\+{comment_end_re}|\-{comment_end_re}\s*" 540 rf"|{comment_end_re}{block_suffix_re}))" 541 ), 542 (TOKEN_COMMENT, TOKEN_COMMENT_END), 543 "#pop", 544 ), 545 _Rule(c(r"(.)"), (Failure("Missing end of comment tag"),), None), 546 ], 547 # blocks 548 TOKEN_BLOCK_BEGIN: [ 549 _Rule( 550 c( 551 rf"(?:\+{block_end_re}|\-{block_end_re}\s*" 552 rf"|{block_end_re}{block_suffix_re})" 553 ), 554 TOKEN_BLOCK_END, 555 "#pop", 556 ), 557 ] 558 + tag_rules, 559 # variables 560 TOKEN_VARIABLE_BEGIN: [ 561 _Rule( 562 c(rf"\-{variable_end_re}\s*|{variable_end_re}"), 563 TOKEN_VARIABLE_END, 564 "#pop", 565 ) 566 ] 567 + tag_rules, 568 # raw block 569 TOKEN_RAW_BEGIN: [ 570 _Rule( 571 c( 572 rf"(.*?)((?:{block_start_re}(\-|\+|))\s*endraw\s*" 573 rf"(?:\+{block_end_re}|\-{block_end_re}\s*" 574 rf"|{block_end_re}{block_suffix_re}))" 575 ), 576 OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END), # type: ignore 577 "#pop", 578 ), 579 _Rule(c(r"(.)"), (Failure("Missing end of raw directive"),), None), 580 ], 581 # line statements 582 TOKEN_LINESTATEMENT_BEGIN: [ 583 _Rule(c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop") 584 ] 585 + tag_rules, 586 # line comments 587 TOKEN_LINECOMMENT_BEGIN: [ 588 _Rule( 589 c(r"(.*?)()(?=\n|$)"), 590 (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END), 591 "#pop", 592 ) 593 ], 594 } 595 596 def _normalize_newlines(self, value: str) -> str: 597 """Replace all newlines with the configured sequence in strings 598 and template data. 599 """ 600 return newline_re.sub(self.newline_sequence, value) 601 602 def tokenize( 603 self, 604 source: str, 605 name: t.Optional[str] = None, 606 filename: t.Optional[str] = None, 607 state: t.Optional[str] = None, 608 ) -> TokenStream: 609 """Calls tokeniter + tokenize and wraps it in a token stream.""" 610 stream = self.tokeniter(source, name, filename, state) 611 return TokenStream(self.wrap(stream, name, filename), name, filename) 612 613 def wrap( 614 self, 615 stream: t.Iterable[t.Tuple[int, str, str]], 616 name: t.Optional[str] = None, 617 filename: t.Optional[str] = None, 618 ) -> t.Iterator[Token]: 619 """This is called with the stream as returned by `tokenize` and wraps 620 every token in a :class:`Token` and converts the value. 621 """ 622 for lineno, token, value_str in stream: 623 if token in ignored_tokens: 624 continue 625 626 value: t.Any = value_str 627 628 if token == TOKEN_LINESTATEMENT_BEGIN: 629 token = TOKEN_BLOCK_BEGIN 630 elif token == TOKEN_LINESTATEMENT_END: 631 token = TOKEN_BLOCK_END 632 # we are not interested in those tokens in the parser 633 elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END): 634 continue 635 elif token == TOKEN_DATA: 636 value = self._normalize_newlines(value_str) 637 elif token == "keyword": 638 token = value_str 639 elif token == TOKEN_NAME: 640 value = value_str 641 642 if not value.isidentifier(): 643 raise TemplateSyntaxError( 644 "Invalid character in identifier", lineno, name, filename 645 ) 646 elif token == TOKEN_STRING: 647 # try to unescape string 648 try: 649 value = ( 650 self._normalize_newlines(value_str[1:-1]) 651 .encode("ascii", "backslashreplace") 652 .decode("unicode-escape") 653 ) 654 except Exception as e: 655 msg = str(e).split(":")[-1].strip() 656 raise TemplateSyntaxError(msg, lineno, name, filename) from e 657 elif token == TOKEN_INTEGER: 658 value = int(value_str.replace("_", ""), 0) 659 elif token == TOKEN_FLOAT: 660 # remove all "_" first to support more Python versions 661 value = literal_eval(value_str.replace("_", "")) 662 elif token == TOKEN_OPERATOR: 663 token = operators[value_str] 664 665 yield Token(lineno, token, value) 666 667 def tokeniter( 668 self, 669 source: str, 670 name: t.Optional[str], 671 filename: t.Optional[str] = None, 672 state: t.Optional[str] = None, 673 ) -> t.Iterator[t.Tuple[int, str, str]]: 674 """This method tokenizes the text and returns the tokens in a 675 generator. Use this method if you just want to tokenize a template. 676 677 .. versionchanged:: 3.0 678 Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line 679 breaks. 680 """ 681 lines = newline_re.split(source)[::2] 682 683 if not self.keep_trailing_newline and lines[-1] == "": 684 del lines[-1] 685 686 source = "\n".join(lines) 687 pos = 0 688 lineno = 1 689 stack = ["root"] 690 691 if state is not None and state != "root": 692 assert state in ("variable", "block"), "invalid state" 693 stack.append(state + "_begin") 694 695 statetokens = self.rules[stack[-1]] 696 source_length = len(source) 697 balancing_stack: t.List[str] = [] 698 newlines_stripped = 0 699 line_starting = True 700 701 while True: 702 # tokenizer loop 703 for regex, tokens, new_state in statetokens: 704 m = regex.match(source, pos) 705 706 # if no match we try again with the next rule 707 if m is None: 708 continue 709 710 # we only match blocks and variables if braces / parentheses 711 # are balanced. continue parsing with the lower rule which 712 # is the operator rule. do this only if the end tags look 713 # like operators 714 if balancing_stack and tokens in ( 715 TOKEN_VARIABLE_END, 716 TOKEN_BLOCK_END, 717 TOKEN_LINESTATEMENT_END, 718 ): 719 continue 720 721 # tuples support more options 722 if isinstance(tokens, tuple): 723 groups: t.Sequence[str] = m.groups() 724 725 if isinstance(tokens, OptionalLStrip): 726 # Rule supports lstrip. Match will look like 727 # text, block type, whitespace control, type, control, ... 728 text = groups[0] 729 # Skipping the text and first type, every other group is the 730 # whitespace control for each type. One of the groups will be 731 # -, +, or empty string instead of None. 732 strip_sign = next(g for g in groups[2::2] if g is not None) 733 734 if strip_sign == "-": 735 # Strip all whitespace between the text and the tag. 736 stripped = text.rstrip() 737 newlines_stripped = text[len(stripped) :].count("\n") 738 groups = [stripped, *groups[1:]] 739 elif ( 740 # Not marked for preserving whitespace. 741 strip_sign != "+" 742 # lstrip is enabled. 743 and self.lstrip_blocks 744 # Not a variable expression. 745 and not m.groupdict().get(TOKEN_VARIABLE_BEGIN) 746 ): 747 # The start of text between the last newline and the tag. 748 l_pos = text.rfind("\n") + 1 749 750 if l_pos > 0 or line_starting: 751 # If there's only whitespace between the newline and the 752 # tag, strip it. 753 if whitespace_re.fullmatch(text, l_pos): 754 groups = [text[:l_pos], *groups[1:]] 755 756 for idx, token in enumerate(tokens): 757 # failure group 758 if token.__class__ is Failure: 759 raise token(lineno, filename) 760 # bygroup is a bit more complex, in that case we 761 # yield for the current token the first named 762 # group that matched 763 elif token == "#bygroup": 764 for key, value in m.groupdict().items(): 765 if value is not None: 766 yield lineno, key, value 767 lineno += value.count("\n") 768 break 769 else: 770 raise RuntimeError( 771 f"{regex!r} wanted to resolve the token dynamically" 772 " but no group matched" 773 ) 774 # normal group 775 else: 776 data = groups[idx] 777 778 if data or token not in ignore_if_empty: 779 yield lineno, token, data 780 781 lineno += data.count("\n") + newlines_stripped 782 newlines_stripped = 0 783 784 # strings as token just are yielded as it. 785 else: 786 data = m.group() 787 788 # update brace/parentheses balance 789 if tokens == TOKEN_OPERATOR: 790 if data == "{": 791 balancing_stack.append("}") 792 elif data == "(": 793 balancing_stack.append(")") 794 elif data == "[": 795 balancing_stack.append("]") 796 elif data in ("}", ")", "]"): 797 if not balancing_stack: 798 raise TemplateSyntaxError( 799 f"unexpected '{data}'", lineno, name, filename 800 ) 801 802 expected_op = balancing_stack.pop() 803 804 if expected_op != data: 805 raise TemplateSyntaxError( 806 f"unexpected '{data}', expected '{expected_op}'", 807 lineno, 808 name, 809 filename, 810 ) 811 812 # yield items 813 if data or tokens not in ignore_if_empty: 814 yield lineno, tokens, data 815 816 lineno += data.count("\n") 817 818 line_starting = m.group()[-1:] == "\n" 819 # fetch new position into new variable so that we can check 820 # if there is a internal parsing error which would result 821 # in an infinite loop 822 pos2 = m.end() 823 824 # handle state changes 825 if new_state is not None: 826 # remove the uppermost state 827 if new_state == "#pop": 828 stack.pop() 829 # resolve the new state by group checking 830 elif new_state == "#bygroup": 831 for key, value in m.groupdict().items(): 832 if value is not None: 833 stack.append(key) 834 break 835 else: 836 raise RuntimeError( 837 f"{regex!r} wanted to resolve the new state dynamically" 838 f" but no group matched" 839 ) 840 # direct state name given 841 else: 842 stack.append(new_state) 843 844 statetokens = self.rules[stack[-1]] 845 # we are still at the same position and no stack change. 846 # this means a loop without break condition, avoid that and 847 # raise error 848 elif pos2 == pos: 849 raise RuntimeError( 850 f"{regex!r} yielded empty string without stack change" 851 ) 852 853 # publish new function and start again 854 pos = pos2 855 break 856 # if loop terminated without break we haven't found a single match 857 # either we are at the end of the file or we have a problem 858 else: 859 # end of text 860 if pos >= source_length: 861 return 862 863 # something went wrong 864 raise TemplateSyntaxError( 865 f"unexpected char {source[pos]!r} at {pos}", lineno, name, filename 866 ) 867