1"""Parse tokens from the lexer into nodes for the compiler.""" 2import typing 3import typing as t 4 5from . import nodes 6from .exceptions import TemplateAssertionError 7from .exceptions import TemplateSyntaxError 8from .lexer import describe_token 9from .lexer import describe_token_expr 10 11if t.TYPE_CHECKING: 12 import typing_extensions as te 13 from .environment import Environment 14 15_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include) 16_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock) 17 18_statement_keywords = frozenset( 19 [ 20 "for", 21 "if", 22 "block", 23 "extends", 24 "print", 25 "macro", 26 "include", 27 "from", 28 "import", 29 "set", 30 "with", 31 "autoescape", 32 ] 33) 34_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"]) 35 36_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = { 37 "add": nodes.Add, 38 "sub": nodes.Sub, 39 "mul": nodes.Mul, 40 "div": nodes.Div, 41 "floordiv": nodes.FloorDiv, 42 "mod": nodes.Mod, 43} 44 45 46class Parser: 47 """This is the central parsing class Jinja uses. It's passed to 48 extensions and can be used to parse expressions or statements. 49 """ 50 51 def __init__( 52 self, 53 environment: "Environment", 54 source: str, 55 name: t.Optional[str] = None, 56 filename: t.Optional[str] = None, 57 state: t.Optional[str] = None, 58 ) -> None: 59 self.environment = environment 60 self.stream = environment._tokenize(source, name, filename, state) 61 self.name = name 62 self.filename = filename 63 self.closed = False 64 self.extensions: t.Dict[ 65 str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]] 66 ] = {} 67 for extension in environment.iter_extensions(): 68 for tag in extension.tags: 69 self.extensions[tag] = extension.parse 70 self._last_identifier = 0 71 self._tag_stack: t.List[str] = [] 72 self._end_token_stack: t.List[t.Tuple[str, ...]] = [] 73 74 def fail( 75 self, 76 msg: str, 77 lineno: t.Optional[int] = None, 78 exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError, 79 ) -> "te.NoReturn": 80 """Convenience method that raises `exc` with the message, passed 81 line number or last line number as well as the current name and 82 filename. 83 """ 84 if lineno is None: 85 lineno = self.stream.current.lineno 86 raise exc(msg, lineno, self.name, self.filename) 87 88 def _fail_ut_eof( 89 self, 90 name: t.Optional[str], 91 end_token_stack: t.List[t.Tuple[str, ...]], 92 lineno: t.Optional[int], 93 ) -> "te.NoReturn": 94 expected: t.Set[str] = set() 95 for exprs in end_token_stack: 96 expected.update(map(describe_token_expr, exprs)) 97 if end_token_stack: 98 currently_looking: t.Optional[str] = " or ".join( 99 map(repr, map(describe_token_expr, end_token_stack[-1])) 100 ) 101 else: 102 currently_looking = None 103 104 if name is None: 105 message = ["Unexpected end of template."] 106 else: 107 message = [f"Encountered unknown tag {name!r}."] 108 109 if currently_looking: 110 if name is not None and name in expected: 111 message.append( 112 "You probably made a nesting mistake. Jinja is expecting this tag," 113 f" but currently looking for {currently_looking}." 114 ) 115 else: 116 message.append( 117 f"Jinja was looking for the following tags: {currently_looking}." 118 ) 119 120 if self._tag_stack: 121 message.append( 122 "The innermost block that needs to be closed is" 123 f" {self._tag_stack[-1]!r}." 124 ) 125 126 self.fail(" ".join(message), lineno) 127 128 def fail_unknown_tag( 129 self, name: str, lineno: t.Optional[int] = None 130 ) -> "te.NoReturn": 131 """Called if the parser encounters an unknown tag. Tries to fail 132 with a human readable error message that could help to identify 133 the problem. 134 """ 135 self._fail_ut_eof(name, self._end_token_stack, lineno) 136 137 def fail_eof( 138 self, 139 end_tokens: t.Optional[t.Tuple[str, ...]] = None, 140 lineno: t.Optional[int] = None, 141 ) -> "te.NoReturn": 142 """Like fail_unknown_tag but for end of template situations.""" 143 stack = list(self._end_token_stack) 144 if end_tokens is not None: 145 stack.append(end_tokens) 146 self._fail_ut_eof(None, stack, lineno) 147 148 def is_tuple_end( 149 self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None 150 ) -> bool: 151 """Are we at the end of a tuple?""" 152 if self.stream.current.type in ("variable_end", "block_end", "rparen"): 153 return True 154 elif extra_end_rules is not None: 155 return self.stream.current.test_any(extra_end_rules) # type: ignore 156 return False 157 158 def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName: 159 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" 160 self._last_identifier += 1 161 rv = object.__new__(nodes.InternalName) 162 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno) 163 return rv 164 165 def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]: 166 """Parse a single statement.""" 167 token = self.stream.current 168 if token.type != "name": 169 self.fail("tag name expected", token.lineno) 170 self._tag_stack.append(token.value) 171 pop_tag = True 172 try: 173 if token.value in _statement_keywords: 174 f = getattr(self, f"parse_{self.stream.current.value}") 175 return f() # type: ignore 176 if token.value == "call": 177 return self.parse_call_block() 178 if token.value == "filter": 179 return self.parse_filter_block() 180 ext = self.extensions.get(token.value) 181 if ext is not None: 182 return ext(self) 183 184 # did not work out, remove the token we pushed by accident 185 # from the stack so that the unknown tag fail function can 186 # produce a proper error message. 187 self._tag_stack.pop() 188 pop_tag = False 189 self.fail_unknown_tag(token.value, token.lineno) 190 finally: 191 if pop_tag: 192 self._tag_stack.pop() 193 194 def parse_statements( 195 self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False 196 ) -> t.List[nodes.Node]: 197 """Parse multiple statements into a list until one of the end tokens 198 is reached. This is used to parse the body of statements as it also 199 parses template data if appropriate. The parser checks first if the 200 current token is a colon and skips it if there is one. Then it checks 201 for the block end and parses until if one of the `end_tokens` is 202 reached. Per default the active token in the stream at the end of 203 the call is the matched end token. If this is not wanted `drop_needle` 204 can be set to `True` and the end token is removed. 205 """ 206 # the first token may be a colon for python compatibility 207 self.stream.skip_if("colon") 208 209 # in the future it would be possible to add whole code sections 210 # by adding some sort of end of statement token and parsing those here. 211 self.stream.expect("block_end") 212 result = self.subparse(end_tokens) 213 214 # we reached the end of the template too early, the subparser 215 # does not check for this, so we do that now 216 if self.stream.current.type == "eof": 217 self.fail_eof(end_tokens) 218 219 if drop_needle: 220 next(self.stream) 221 return result 222 223 def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]: 224 """Parse an assign statement.""" 225 lineno = next(self.stream).lineno 226 target = self.parse_assign_target(with_namespace=True) 227 if self.stream.skip_if("assign"): 228 expr = self.parse_tuple() 229 return nodes.Assign(target, expr, lineno=lineno) 230 filter_node = self.parse_filter(None) 231 body = self.parse_statements(("name:endset",), drop_needle=True) 232 return nodes.AssignBlock(target, filter_node, body, lineno=lineno) 233 234 def parse_for(self) -> nodes.For: 235 """Parse a for loop.""" 236 lineno = self.stream.expect("name:for").lineno 237 target = self.parse_assign_target(extra_end_rules=("name:in",)) 238 self.stream.expect("name:in") 239 iter = self.parse_tuple( 240 with_condexpr=False, extra_end_rules=("name:recursive",) 241 ) 242 test = None 243 if self.stream.skip_if("name:if"): 244 test = self.parse_expression() 245 recursive = self.stream.skip_if("name:recursive") 246 body = self.parse_statements(("name:endfor", "name:else")) 247 if next(self.stream).value == "endfor": 248 else_ = [] 249 else: 250 else_ = self.parse_statements(("name:endfor",), drop_needle=True) 251 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) 252 253 def parse_if(self) -> nodes.If: 254 """Parse an if construct.""" 255 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) 256 while True: 257 node.test = self.parse_tuple(with_condexpr=False) 258 node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) 259 node.elif_ = [] 260 node.else_ = [] 261 token = next(self.stream) 262 if token.test("name:elif"): 263 node = nodes.If(lineno=self.stream.current.lineno) 264 result.elif_.append(node) 265 continue 266 elif token.test("name:else"): 267 result.else_ = self.parse_statements(("name:endif",), drop_needle=True) 268 break 269 return result 270 271 def parse_with(self) -> nodes.With: 272 node = nodes.With(lineno=next(self.stream).lineno) 273 targets: t.List[nodes.Expr] = [] 274 values: t.List[nodes.Expr] = [] 275 while self.stream.current.type != "block_end": 276 if targets: 277 self.stream.expect("comma") 278 target = self.parse_assign_target() 279 target.set_ctx("param") 280 targets.append(target) 281 self.stream.expect("assign") 282 values.append(self.parse_expression()) 283 node.targets = targets 284 node.values = values 285 node.body = self.parse_statements(("name:endwith",), drop_needle=True) 286 return node 287 288 def parse_autoescape(self) -> nodes.Scope: 289 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) 290 node.options = [nodes.Keyword("autoescape", self.parse_expression())] 291 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) 292 return nodes.Scope([node]) 293 294 def parse_block(self) -> nodes.Block: 295 node = nodes.Block(lineno=next(self.stream).lineno) 296 node.name = self.stream.expect("name").value 297 node.scoped = self.stream.skip_if("name:scoped") 298 node.required = self.stream.skip_if("name:required") 299 300 # common problem people encounter when switching from django 301 # to jinja. we do not support hyphens in block names, so let's 302 # raise a nicer error message in that case. 303 if self.stream.current.type == "sub": 304 self.fail( 305 "Block names in Jinja have to be valid Python identifiers and may not" 306 " contain hyphens, use an underscore instead." 307 ) 308 309 node.body = self.parse_statements(("name:endblock",), drop_needle=True) 310 311 # enforce that required blocks only contain whitespace or comments 312 # by asserting that the body, if not empty, is just TemplateData nodes 313 # with whitespace data 314 if node.required and not all( 315 isinstance(child, nodes.TemplateData) and child.data.isspace() 316 for body in node.body 317 for child in body.nodes # type: ignore 318 ): 319 self.fail("Required blocks can only contain comments or whitespace") 320 321 self.stream.skip_if("name:" + node.name) 322 return node 323 324 def parse_extends(self) -> nodes.Extends: 325 node = nodes.Extends(lineno=next(self.stream).lineno) 326 node.template = self.parse_expression() 327 return node 328 329 def parse_import_context( 330 self, node: _ImportInclude, default: bool 331 ) -> _ImportInclude: 332 if self.stream.current.test_any( 333 "name:with", "name:without" 334 ) and self.stream.look().test("name:context"): 335 node.with_context = next(self.stream).value == "with" 336 self.stream.skip() 337 else: 338 node.with_context = default 339 return node 340 341 def parse_include(self) -> nodes.Include: 342 node = nodes.Include(lineno=next(self.stream).lineno) 343 node.template = self.parse_expression() 344 if self.stream.current.test("name:ignore") and self.stream.look().test( 345 "name:missing" 346 ): 347 node.ignore_missing = True 348 self.stream.skip(2) 349 else: 350 node.ignore_missing = False 351 return self.parse_import_context(node, True) 352 353 def parse_import(self) -> nodes.Import: 354 node = nodes.Import(lineno=next(self.stream).lineno) 355 node.template = self.parse_expression() 356 self.stream.expect("name:as") 357 node.target = self.parse_assign_target(name_only=True).name 358 return self.parse_import_context(node, False) 359 360 def parse_from(self) -> nodes.FromImport: 361 node = nodes.FromImport(lineno=next(self.stream).lineno) 362 node.template = self.parse_expression() 363 self.stream.expect("name:import") 364 node.names = [] 365 366 def parse_context() -> bool: 367 if self.stream.current.value in { 368 "with", 369 "without", 370 } and self.stream.look().test("name:context"): 371 node.with_context = next(self.stream).value == "with" 372 self.stream.skip() 373 return True 374 return False 375 376 while True: 377 if node.names: 378 self.stream.expect("comma") 379 if self.stream.current.type == "name": 380 if parse_context(): 381 break 382 target = self.parse_assign_target(name_only=True) 383 if target.name.startswith("_"): 384 self.fail( 385 "names starting with an underline can not be imported", 386 target.lineno, 387 exc=TemplateAssertionError, 388 ) 389 if self.stream.skip_if("name:as"): 390 alias = self.parse_assign_target(name_only=True) 391 node.names.append((target.name, alias.name)) 392 else: 393 node.names.append(target.name) 394 if parse_context() or self.stream.current.type != "comma": 395 break 396 else: 397 self.stream.expect("name") 398 if not hasattr(node, "with_context"): 399 node.with_context = False 400 return node 401 402 def parse_signature(self, node: _MacroCall) -> None: 403 args = node.args = [] 404 defaults = node.defaults = [] 405 self.stream.expect("lparen") 406 while self.stream.current.type != "rparen": 407 if args: 408 self.stream.expect("comma") 409 arg = self.parse_assign_target(name_only=True) 410 arg.set_ctx("param") 411 if self.stream.skip_if("assign"): 412 defaults.append(self.parse_expression()) 413 elif defaults: 414 self.fail("non-default argument follows default argument") 415 args.append(arg) 416 self.stream.expect("rparen") 417 418 def parse_call_block(self) -> nodes.CallBlock: 419 node = nodes.CallBlock(lineno=next(self.stream).lineno) 420 if self.stream.current.type == "lparen": 421 self.parse_signature(node) 422 else: 423 node.args = [] 424 node.defaults = [] 425 426 call_node = self.parse_expression() 427 if not isinstance(call_node, nodes.Call): 428 self.fail("expected call", node.lineno) 429 node.call = call_node 430 node.body = self.parse_statements(("name:endcall",), drop_needle=True) 431 return node 432 433 def parse_filter_block(self) -> nodes.FilterBlock: 434 node = nodes.FilterBlock(lineno=next(self.stream).lineno) 435 node.filter = self.parse_filter(None, start_inline=True) # type: ignore 436 node.body = self.parse_statements(("name:endfilter",), drop_needle=True) 437 return node 438 439 def parse_macro(self) -> nodes.Macro: 440 node = nodes.Macro(lineno=next(self.stream).lineno) 441 node.name = self.parse_assign_target(name_only=True).name 442 self.parse_signature(node) 443 node.body = self.parse_statements(("name:endmacro",), drop_needle=True) 444 return node 445 446 def parse_print(self) -> nodes.Output: 447 node = nodes.Output(lineno=next(self.stream).lineno) 448 node.nodes = [] 449 while self.stream.current.type != "block_end": 450 if node.nodes: 451 self.stream.expect("comma") 452 node.nodes.append(self.parse_expression()) 453 return node 454 455 @typing.overload 456 def parse_assign_target( 457 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ... 458 ) -> nodes.Name: 459 ... 460 461 @typing.overload 462 def parse_assign_target( 463 self, 464 with_tuple: bool = True, 465 name_only: bool = False, 466 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 467 with_namespace: bool = False, 468 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: 469 ... 470 471 def parse_assign_target( 472 self, 473 with_tuple: bool = True, 474 name_only: bool = False, 475 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 476 with_namespace: bool = False, 477 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: 478 """Parse an assignment target. As Jinja allows assignments to 479 tuples, this function can parse all allowed assignment targets. Per 480 default assignments to tuples are parsed, that can be disable however 481 by setting `with_tuple` to `False`. If only assignments to names are 482 wanted `name_only` can be set to `True`. The `extra_end_rules` 483 parameter is forwarded to the tuple parsing function. If 484 `with_namespace` is enabled, a namespace assignment may be parsed. 485 """ 486 target: nodes.Expr 487 488 if with_namespace and self.stream.look().type == "dot": 489 token = self.stream.expect("name") 490 next(self.stream) # dot 491 attr = self.stream.expect("name") 492 target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) 493 elif name_only: 494 token = self.stream.expect("name") 495 target = nodes.Name(token.value, "store", lineno=token.lineno) 496 else: 497 if with_tuple: 498 target = self.parse_tuple( 499 simplified=True, extra_end_rules=extra_end_rules 500 ) 501 else: 502 target = self.parse_primary() 503 504 target.set_ctx("store") 505 506 if not target.can_assign(): 507 self.fail( 508 f"can't assign to {type(target).__name__.lower()!r}", target.lineno 509 ) 510 511 return target # type: ignore 512 513 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr: 514 """Parse an expression. Per default all expressions are parsed, if 515 the optional `with_condexpr` parameter is set to `False` conditional 516 expressions are not parsed. 517 """ 518 if with_condexpr: 519 return self.parse_condexpr() 520 return self.parse_or() 521 522 def parse_condexpr(self) -> nodes.Expr: 523 lineno = self.stream.current.lineno 524 expr1 = self.parse_or() 525 expr3: t.Optional[nodes.Expr] 526 527 while self.stream.skip_if("name:if"): 528 expr2 = self.parse_or() 529 if self.stream.skip_if("name:else"): 530 expr3 = self.parse_condexpr() 531 else: 532 expr3 = None 533 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) 534 lineno = self.stream.current.lineno 535 return expr1 536 537 def parse_or(self) -> nodes.Expr: 538 lineno = self.stream.current.lineno 539 left = self.parse_and() 540 while self.stream.skip_if("name:or"): 541 right = self.parse_and() 542 left = nodes.Or(left, right, lineno=lineno) 543 lineno = self.stream.current.lineno 544 return left 545 546 def parse_and(self) -> nodes.Expr: 547 lineno = self.stream.current.lineno 548 left = self.parse_not() 549 while self.stream.skip_if("name:and"): 550 right = self.parse_not() 551 left = nodes.And(left, right, lineno=lineno) 552 lineno = self.stream.current.lineno 553 return left 554 555 def parse_not(self) -> nodes.Expr: 556 if self.stream.current.test("name:not"): 557 lineno = next(self.stream).lineno 558 return nodes.Not(self.parse_not(), lineno=lineno) 559 return self.parse_compare() 560 561 def parse_compare(self) -> nodes.Expr: 562 lineno = self.stream.current.lineno 563 expr = self.parse_math1() 564 ops = [] 565 while True: 566 token_type = self.stream.current.type 567 if token_type in _compare_operators: 568 next(self.stream) 569 ops.append(nodes.Operand(token_type, self.parse_math1())) 570 elif self.stream.skip_if("name:in"): 571 ops.append(nodes.Operand("in", self.parse_math1())) 572 elif self.stream.current.test("name:not") and self.stream.look().test( 573 "name:in" 574 ): 575 self.stream.skip(2) 576 ops.append(nodes.Operand("notin", self.parse_math1())) 577 else: 578 break 579 lineno = self.stream.current.lineno 580 if not ops: 581 return expr 582 return nodes.Compare(expr, ops, lineno=lineno) 583 584 def parse_math1(self) -> nodes.Expr: 585 lineno = self.stream.current.lineno 586 left = self.parse_concat() 587 while self.stream.current.type in ("add", "sub"): 588 cls = _math_nodes[self.stream.current.type] 589 next(self.stream) 590 right = self.parse_concat() 591 left = cls(left, right, lineno=lineno) 592 lineno = self.stream.current.lineno 593 return left 594 595 def parse_concat(self) -> nodes.Expr: 596 lineno = self.stream.current.lineno 597 args = [self.parse_math2()] 598 while self.stream.current.type == "tilde": 599 next(self.stream) 600 args.append(self.parse_math2()) 601 if len(args) == 1: 602 return args[0] 603 return nodes.Concat(args, lineno=lineno) 604 605 def parse_math2(self) -> nodes.Expr: 606 lineno = self.stream.current.lineno 607 left = self.parse_pow() 608 while self.stream.current.type in ("mul", "div", "floordiv", "mod"): 609 cls = _math_nodes[self.stream.current.type] 610 next(self.stream) 611 right = self.parse_pow() 612 left = cls(left, right, lineno=lineno) 613 lineno = self.stream.current.lineno 614 return left 615 616 def parse_pow(self) -> nodes.Expr: 617 lineno = self.stream.current.lineno 618 left = self.parse_unary() 619 while self.stream.current.type == "pow": 620 next(self.stream) 621 right = self.parse_unary() 622 left = nodes.Pow(left, right, lineno=lineno) 623 lineno = self.stream.current.lineno 624 return left 625 626 def parse_unary(self, with_filter: bool = True) -> nodes.Expr: 627 token_type = self.stream.current.type 628 lineno = self.stream.current.lineno 629 node: nodes.Expr 630 631 if token_type == "sub": 632 next(self.stream) 633 node = nodes.Neg(self.parse_unary(False), lineno=lineno) 634 elif token_type == "add": 635 next(self.stream) 636 node = nodes.Pos(self.parse_unary(False), lineno=lineno) 637 else: 638 node = self.parse_primary() 639 node = self.parse_postfix(node) 640 if with_filter: 641 node = self.parse_filter_expr(node) 642 return node 643 644 def parse_primary(self) -> nodes.Expr: 645 token = self.stream.current 646 node: nodes.Expr 647 if token.type == "name": 648 if token.value in ("true", "false", "True", "False"): 649 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) 650 elif token.value in ("none", "None"): 651 node = nodes.Const(None, lineno=token.lineno) 652 else: 653 node = nodes.Name(token.value, "load", lineno=token.lineno) 654 next(self.stream) 655 elif token.type == "string": 656 next(self.stream) 657 buf = [token.value] 658 lineno = token.lineno 659 while self.stream.current.type == "string": 660 buf.append(self.stream.current.value) 661 next(self.stream) 662 node = nodes.Const("".join(buf), lineno=lineno) 663 elif token.type in ("integer", "float"): 664 next(self.stream) 665 node = nodes.Const(token.value, lineno=token.lineno) 666 elif token.type == "lparen": 667 next(self.stream) 668 node = self.parse_tuple(explicit_parentheses=True) 669 self.stream.expect("rparen") 670 elif token.type == "lbracket": 671 node = self.parse_list() 672 elif token.type == "lbrace": 673 node = self.parse_dict() 674 else: 675 self.fail(f"unexpected {describe_token(token)!r}", token.lineno) 676 return node 677 678 def parse_tuple( 679 self, 680 simplified: bool = False, 681 with_condexpr: bool = True, 682 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 683 explicit_parentheses: bool = False, 684 ) -> t.Union[nodes.Tuple, nodes.Expr]: 685 """Works like `parse_expression` but if multiple expressions are 686 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. 687 This method could also return a regular expression instead of a tuple 688 if no commas where found. 689 690 The default parsing mode is a full tuple. If `simplified` is `True` 691 only names and literals are parsed. The `no_condexpr` parameter is 692 forwarded to :meth:`parse_expression`. 693 694 Because tuples do not require delimiters and may end in a bogus comma 695 an extra hint is needed that marks the end of a tuple. For example 696 for loops support tuples between `for` and `in`. In that case the 697 `extra_end_rules` is set to ``['name:in']``. 698 699 `explicit_parentheses` is true if the parsing was triggered by an 700 expression in parentheses. This is used to figure out if an empty 701 tuple is a valid expression or not. 702 """ 703 lineno = self.stream.current.lineno 704 if simplified: 705 parse = self.parse_primary 706 elif with_condexpr: 707 parse = self.parse_expression 708 else: 709 710 def parse() -> nodes.Expr: 711 return self.parse_expression(with_condexpr=False) 712 713 args: t.List[nodes.Expr] = [] 714 is_tuple = False 715 716 while True: 717 if args: 718 self.stream.expect("comma") 719 if self.is_tuple_end(extra_end_rules): 720 break 721 args.append(parse()) 722 if self.stream.current.type == "comma": 723 is_tuple = True 724 else: 725 break 726 lineno = self.stream.current.lineno 727 728 if not is_tuple: 729 if args: 730 return args[0] 731 732 # if we don't have explicit parentheses, an empty tuple is 733 # not a valid expression. This would mean nothing (literally 734 # nothing) in the spot of an expression would be an empty 735 # tuple. 736 if not explicit_parentheses: 737 self.fail( 738 "Expected an expression," 739 f" got {describe_token(self.stream.current)!r}" 740 ) 741 742 return nodes.Tuple(args, "load", lineno=lineno) 743 744 def parse_list(self) -> nodes.List: 745 token = self.stream.expect("lbracket") 746 items: t.List[nodes.Expr] = [] 747 while self.stream.current.type != "rbracket": 748 if items: 749 self.stream.expect("comma") 750 if self.stream.current.type == "rbracket": 751 break 752 items.append(self.parse_expression()) 753 self.stream.expect("rbracket") 754 return nodes.List(items, lineno=token.lineno) 755 756 def parse_dict(self) -> nodes.Dict: 757 token = self.stream.expect("lbrace") 758 items: t.List[nodes.Pair] = [] 759 while self.stream.current.type != "rbrace": 760 if items: 761 self.stream.expect("comma") 762 if self.stream.current.type == "rbrace": 763 break 764 key = self.parse_expression() 765 self.stream.expect("colon") 766 value = self.parse_expression() 767 items.append(nodes.Pair(key, value, lineno=key.lineno)) 768 self.stream.expect("rbrace") 769 return nodes.Dict(items, lineno=token.lineno) 770 771 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr: 772 while True: 773 token_type = self.stream.current.type 774 if token_type == "dot" or token_type == "lbracket": 775 node = self.parse_subscript(node) 776 # calls are valid both after postfix expressions (getattr 777 # and getitem) as well as filters and tests 778 elif token_type == "lparen": 779 node = self.parse_call(node) 780 else: 781 break 782 return node 783 784 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr: 785 while True: 786 token_type = self.stream.current.type 787 if token_type == "pipe": 788 node = self.parse_filter(node) # type: ignore 789 elif token_type == "name" and self.stream.current.value == "is": 790 node = self.parse_test(node) 791 # calls are valid both after postfix expressions (getattr 792 # and getitem) as well as filters and tests 793 elif token_type == "lparen": 794 node = self.parse_call(node) 795 else: 796 break 797 return node 798 799 def parse_subscript( 800 self, node: nodes.Expr 801 ) -> t.Union[nodes.Getattr, nodes.Getitem]: 802 token = next(self.stream) 803 arg: nodes.Expr 804 805 if token.type == "dot": 806 attr_token = self.stream.current 807 next(self.stream) 808 if attr_token.type == "name": 809 return nodes.Getattr( 810 node, attr_token.value, "load", lineno=token.lineno 811 ) 812 elif attr_token.type != "integer": 813 self.fail("expected name or number", attr_token.lineno) 814 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) 815 return nodes.Getitem(node, arg, "load", lineno=token.lineno) 816 if token.type == "lbracket": 817 args: t.List[nodes.Expr] = [] 818 while self.stream.current.type != "rbracket": 819 if args: 820 self.stream.expect("comma") 821 args.append(self.parse_subscribed()) 822 self.stream.expect("rbracket") 823 if len(args) == 1: 824 arg = args[0] 825 else: 826 arg = nodes.Tuple(args, "load", lineno=token.lineno) 827 return nodes.Getitem(node, arg, "load", lineno=token.lineno) 828 self.fail("expected subscript expression", token.lineno) 829 830 def parse_subscribed(self) -> nodes.Expr: 831 lineno = self.stream.current.lineno 832 args: t.List[t.Optional[nodes.Expr]] 833 834 if self.stream.current.type == "colon": 835 next(self.stream) 836 args = [None] 837 else: 838 node = self.parse_expression() 839 if self.stream.current.type != "colon": 840 return node 841 next(self.stream) 842 args = [node] 843 844 if self.stream.current.type == "colon": 845 args.append(None) 846 elif self.stream.current.type not in ("rbracket", "comma"): 847 args.append(self.parse_expression()) 848 else: 849 args.append(None) 850 851 if self.stream.current.type == "colon": 852 next(self.stream) 853 if self.stream.current.type not in ("rbracket", "comma"): 854 args.append(self.parse_expression()) 855 else: 856 args.append(None) 857 else: 858 args.append(None) 859 860 return nodes.Slice(lineno=lineno, *args) 861 862 def parse_call_args(self) -> t.Tuple: 863 token = self.stream.expect("lparen") 864 args = [] 865 kwargs = [] 866 dyn_args = None 867 dyn_kwargs = None 868 require_comma = False 869 870 def ensure(expr: bool) -> None: 871 if not expr: 872 self.fail("invalid syntax for function call expression", token.lineno) 873 874 while self.stream.current.type != "rparen": 875 if require_comma: 876 self.stream.expect("comma") 877 878 # support for trailing comma 879 if self.stream.current.type == "rparen": 880 break 881 882 if self.stream.current.type == "mul": 883 ensure(dyn_args is None and dyn_kwargs is None) 884 next(self.stream) 885 dyn_args = self.parse_expression() 886 elif self.stream.current.type == "pow": 887 ensure(dyn_kwargs is None) 888 next(self.stream) 889 dyn_kwargs = self.parse_expression() 890 else: 891 if ( 892 self.stream.current.type == "name" 893 and self.stream.look().type == "assign" 894 ): 895 # Parsing a kwarg 896 ensure(dyn_kwargs is None) 897 key = self.stream.current.value 898 self.stream.skip(2) 899 value = self.parse_expression() 900 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) 901 else: 902 # Parsing an arg 903 ensure(dyn_args is None and dyn_kwargs is None and not kwargs) 904 args.append(self.parse_expression()) 905 906 require_comma = True 907 908 self.stream.expect("rparen") 909 return args, kwargs, dyn_args, dyn_kwargs 910 911 def parse_call(self, node: nodes.Expr) -> nodes.Call: 912 # The lparen will be expected in parse_call_args, but the lineno 913 # needs to be recorded before the stream is advanced. 914 token = self.stream.current 915 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 916 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) 917 918 def parse_filter( 919 self, node: t.Optional[nodes.Expr], start_inline: bool = False 920 ) -> t.Optional[nodes.Expr]: 921 while self.stream.current.type == "pipe" or start_inline: 922 if not start_inline: 923 next(self.stream) 924 token = self.stream.expect("name") 925 name = token.value 926 while self.stream.current.type == "dot": 927 next(self.stream) 928 name += "." + self.stream.expect("name").value 929 if self.stream.current.type == "lparen": 930 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 931 else: 932 args = [] 933 kwargs = [] 934 dyn_args = dyn_kwargs = None 935 node = nodes.Filter( 936 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno 937 ) 938 start_inline = False 939 return node 940 941 def parse_test(self, node: nodes.Expr) -> nodes.Expr: 942 token = next(self.stream) 943 if self.stream.current.test("name:not"): 944 next(self.stream) 945 negated = True 946 else: 947 negated = False 948 name = self.stream.expect("name").value 949 while self.stream.current.type == "dot": 950 next(self.stream) 951 name += "." + self.stream.expect("name").value 952 dyn_args = dyn_kwargs = None 953 kwargs = [] 954 if self.stream.current.type == "lparen": 955 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 956 elif self.stream.current.type in { 957 "name", 958 "string", 959 "integer", 960 "float", 961 "lparen", 962 "lbracket", 963 "lbrace", 964 } and not self.stream.current.test_any("name:else", "name:or", "name:and"): 965 if self.stream.current.test("name:is"): 966 self.fail("You cannot chain multiple tests with is") 967 arg_node = self.parse_primary() 968 arg_node = self.parse_postfix(arg_node) 969 args = [arg_node] 970 else: 971 args = [] 972 node = nodes.Test( 973 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno 974 ) 975 if negated: 976 node = nodes.Not(node, lineno=token.lineno) 977 return node 978 979 def subparse( 980 self, end_tokens: t.Optional[t.Tuple[str, ...]] = None 981 ) -> t.List[nodes.Node]: 982 body: t.List[nodes.Node] = [] 983 data_buffer: t.List[nodes.Node] = [] 984 add_data = data_buffer.append 985 986 if end_tokens is not None: 987 self._end_token_stack.append(end_tokens) 988 989 def flush_data() -> None: 990 if data_buffer: 991 lineno = data_buffer[0].lineno 992 body.append(nodes.Output(data_buffer[:], lineno=lineno)) 993 del data_buffer[:] 994 995 try: 996 while self.stream: 997 token = self.stream.current 998 if token.type == "data": 999 if token.value: 1000 add_data(nodes.TemplateData(token.value, lineno=token.lineno)) 1001 next(self.stream) 1002 elif token.type == "variable_begin": 1003 next(self.stream) 1004 add_data(self.parse_tuple(with_condexpr=True)) 1005 self.stream.expect("variable_end") 1006 elif token.type == "block_begin": 1007 flush_data() 1008 next(self.stream) 1009 if end_tokens is not None and self.stream.current.test_any( 1010 *end_tokens 1011 ): 1012 return body 1013 rv = self.parse_statement() 1014 if isinstance(rv, list): 1015 body.extend(rv) 1016 else: 1017 body.append(rv) 1018 self.stream.expect("block_end") 1019 else: 1020 raise AssertionError("internal parsing error") 1021 1022 flush_data() 1023 finally: 1024 if end_tokens is not None: 1025 self._end_token_stack.pop() 1026 return body 1027 1028 def parse(self) -> nodes.Template: 1029 """Parse the whole template into a `Template` node.""" 1030 result = nodes.Template(self.subparse(), lineno=1) 1031 result.set_environment(self.environment) 1032 return result 1033