• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2"""
3    jinja2.parser
4    ~~~~~~~~~~~~~
5
6    Implements the template parser.
7
8    :copyright: (c) 2010 by the Jinja Team.
9    :license: BSD, see LICENSE for more details.
10"""
11from jinja2 import nodes
12from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError
13from jinja2.lexer import describe_token, describe_token_expr
14from jinja2._compat import next, imap
15
16
17#: statements that callinto
18_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print',
19                                 'macro', 'include', 'from', 'import',
20                                 'set'])
21_compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq'])
22
23
24class Parser(object):
25    """This is the central parsing class Jinja2 uses.  It's passed to
26    extensions and can be used to parse expressions or statements.
27    """
28
29    def __init__(self, environment, source, name=None, filename=None,
30                 state=None):
31        self.environment = environment
32        self.stream = environment._tokenize(source, name, filename, state)
33        self.name = name
34        self.filename = filename
35        self.closed = False
36        self.extensions = {}
37        for extension in environment.iter_extensions():
38            for tag in extension.tags:
39                self.extensions[tag] = extension.parse
40        self._last_identifier = 0
41        self._tag_stack = []
42        self._end_token_stack = []
43
44    def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
45        """Convenience method that raises `exc` with the message, passed
46        line number or last line number as well as the current name and
47        filename.
48        """
49        if lineno is None:
50            lineno = self.stream.current.lineno
51        raise exc(msg, lineno, self.name, self.filename)
52
53    def _fail_ut_eof(self, name, end_token_stack, lineno):
54        expected = []
55        for exprs in end_token_stack:
56            expected.extend(imap(describe_token_expr, exprs))
57        if end_token_stack:
58            currently_looking = ' or '.join(
59                "'%s'" % describe_token_expr(expr)
60                for expr in end_token_stack[-1])
61        else:
62            currently_looking = None
63
64        if name is None:
65            message = ['Unexpected end of template.']
66        else:
67            message = ['Encountered unknown tag \'%s\'.' % name]
68
69        if currently_looking:
70            if name is not None and name in expected:
71                message.append('You probably made a nesting mistake. Jinja '
72                               'is expecting this tag, but currently looking '
73                               'for %s.' % currently_looking)
74            else:
75                message.append('Jinja was looking for the following tags: '
76                               '%s.' % currently_looking)
77
78        if self._tag_stack:
79            message.append('The innermost block that needs to be '
80                           'closed is \'%s\'.' % self._tag_stack[-1])
81
82        self.fail(' '.join(message), lineno)
83
84    def fail_unknown_tag(self, name, lineno=None):
85        """Called if the parser encounters an unknown tag.  Tries to fail
86        with a human readable error message that could help to identify
87        the problem.
88        """
89        return self._fail_ut_eof(name, self._end_token_stack, lineno)
90
91    def fail_eof(self, end_tokens=None, lineno=None):
92        """Like fail_unknown_tag but for end of template situations."""
93        stack = list(self._end_token_stack)
94        if end_tokens is not None:
95            stack.append(end_tokens)
96        return self._fail_ut_eof(None, stack, lineno)
97
98    def is_tuple_end(self, extra_end_rules=None):
99        """Are we at the end of a tuple?"""
100        if self.stream.current.type in ('variable_end', 'block_end', 'rparen'):
101            return True
102        elif extra_end_rules is not None:
103            return self.stream.current.test_any(extra_end_rules)
104        return False
105
106    def free_identifier(self, lineno=None):
107        """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
108        self._last_identifier += 1
109        rv = object.__new__(nodes.InternalName)
110        nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno)
111        return rv
112
113    def parse_statement(self):
114        """Parse a single statement."""
115        token = self.stream.current
116        if token.type != 'name':
117            self.fail('tag name expected', token.lineno)
118        self._tag_stack.append(token.value)
119        pop_tag = True
120        try:
121            if token.value in _statement_keywords:
122                return getattr(self, 'parse_' + self.stream.current.value)()
123            if token.value == 'call':
124                return self.parse_call_block()
125            if token.value == 'filter':
126                return self.parse_filter_block()
127            ext = self.extensions.get(token.value)
128            if ext is not None:
129                return ext(self)
130
131            # did not work out, remove the token we pushed by accident
132            # from the stack so that the unknown tag fail function can
133            # produce a proper error message.
134            self._tag_stack.pop()
135            pop_tag = False
136            self.fail_unknown_tag(token.value, token.lineno)
137        finally:
138            if pop_tag:
139                self._tag_stack.pop()
140
141    def parse_statements(self, end_tokens, drop_needle=False):
142        """Parse multiple statements into a list until one of the end tokens
143        is reached.  This is used to parse the body of statements as it also
144        parses template data if appropriate.  The parser checks first if the
145        current token is a colon and skips it if there is one.  Then it checks
146        for the block end and parses until if one of the `end_tokens` is
147        reached.  Per default the active token in the stream at the end of
148        the call is the matched end token.  If this is not wanted `drop_needle`
149        can be set to `True` and the end token is removed.
150        """
151        # the first token may be a colon for python compatibility
152        self.stream.skip_if('colon')
153
154        # in the future it would be possible to add whole code sections
155        # by adding some sort of end of statement token and parsing those here.
156        self.stream.expect('block_end')
157        result = self.subparse(end_tokens)
158
159        # we reached the end of the template too early, the subparser
160        # does not check for this, so we do that now
161        if self.stream.current.type == 'eof':
162            self.fail_eof(end_tokens)
163
164        if drop_needle:
165            next(self.stream)
166        return result
167
168    def parse_set(self):
169        """Parse an assign statement."""
170        lineno = next(self.stream).lineno
171        target = self.parse_assign_target()
172        self.stream.expect('assign')
173        expr = self.parse_tuple()
174        return nodes.Assign(target, expr, lineno=lineno)
175
176    def parse_for(self):
177        """Parse a for loop."""
178        lineno = self.stream.expect('name:for').lineno
179        target = self.parse_assign_target(extra_end_rules=('name:in',))
180        self.stream.expect('name:in')
181        iter = self.parse_tuple(with_condexpr=False,
182                                extra_end_rules=('name:recursive',))
183        test = None
184        if self.stream.skip_if('name:if'):
185            test = self.parse_expression()
186        recursive = self.stream.skip_if('name:recursive')
187        body = self.parse_statements(('name:endfor', 'name:else'))
188        if next(self.stream).value == 'endfor':
189            else_ = []
190        else:
191            else_ = self.parse_statements(('name:endfor',), drop_needle=True)
192        return nodes.For(target, iter, body, else_, test,
193                         recursive, lineno=lineno)
194
195    def parse_if(self):
196        """Parse an if construct."""
197        node = result = nodes.If(lineno=self.stream.expect('name:if').lineno)
198        while 1:
199            node.test = self.parse_tuple(with_condexpr=False)
200            node.body = self.parse_statements(('name:elif', 'name:else',
201                                               'name:endif'))
202            token = next(self.stream)
203            if token.test('name:elif'):
204                new_node = nodes.If(lineno=self.stream.current.lineno)
205                node.else_ = [new_node]
206                node = new_node
207                continue
208            elif token.test('name:else'):
209                node.else_ = self.parse_statements(('name:endif',),
210                                                   drop_needle=True)
211            else:
212                node.else_ = []
213            break
214        return result
215
216    def parse_block(self):
217        node = nodes.Block(lineno=next(self.stream).lineno)
218        node.name = self.stream.expect('name').value
219        node.scoped = self.stream.skip_if('name:scoped')
220
221        # common problem people encounter when switching from django
222        # to jinja.  we do not support hyphens in block names, so let's
223        # raise a nicer error message in that case.
224        if self.stream.current.type == 'sub':
225            self.fail('Block names in Jinja have to be valid Python '
226                      'identifiers and may not contain hyphens, use an '
227                      'underscore instead.')
228
229        node.body = self.parse_statements(('name:endblock',), drop_needle=True)
230        self.stream.skip_if('name:' + node.name)
231        return node
232
233    def parse_extends(self):
234        node = nodes.Extends(lineno=next(self.stream).lineno)
235        node.template = self.parse_expression()
236        return node
237
238    def parse_import_context(self, node, default):
239        if self.stream.current.test_any('name:with', 'name:without') and \
240           self.stream.look().test('name:context'):
241            node.with_context = next(self.stream).value == 'with'
242            self.stream.skip()
243        else:
244            node.with_context = default
245        return node
246
247    def parse_include(self):
248        node = nodes.Include(lineno=next(self.stream).lineno)
249        node.template = self.parse_expression()
250        if self.stream.current.test('name:ignore') and \
251           self.stream.look().test('name:missing'):
252            node.ignore_missing = True
253            self.stream.skip(2)
254        else:
255            node.ignore_missing = False
256        return self.parse_import_context(node, True)
257
258    def parse_import(self):
259        node = nodes.Import(lineno=next(self.stream).lineno)
260        node.template = self.parse_expression()
261        self.stream.expect('name:as')
262        node.target = self.parse_assign_target(name_only=True).name
263        return self.parse_import_context(node, False)
264
265    def parse_from(self):
266        node = nodes.FromImport(lineno=next(self.stream).lineno)
267        node.template = self.parse_expression()
268        self.stream.expect('name:import')
269        node.names = []
270
271        def parse_context():
272            if self.stream.current.value in ('with', 'without') and \
273               self.stream.look().test('name:context'):
274                node.with_context = next(self.stream).value == 'with'
275                self.stream.skip()
276                return True
277            return False
278
279        while 1:
280            if node.names:
281                self.stream.expect('comma')
282            if self.stream.current.type == 'name':
283                if parse_context():
284                    break
285                target = self.parse_assign_target(name_only=True)
286                if target.name.startswith('_'):
287                    self.fail('names starting with an underline can not '
288                              'be imported', target.lineno,
289                              exc=TemplateAssertionError)
290                if self.stream.skip_if('name:as'):
291                    alias = self.parse_assign_target(name_only=True)
292                    node.names.append((target.name, alias.name))
293                else:
294                    node.names.append(target.name)
295                if parse_context() or self.stream.current.type != 'comma':
296                    break
297            else:
298                break
299        if not hasattr(node, 'with_context'):
300            node.with_context = False
301            self.stream.skip_if('comma')
302        return node
303
304    def parse_signature(self, node):
305        node.args = args = []
306        node.defaults = defaults = []
307        self.stream.expect('lparen')
308        while self.stream.current.type != 'rparen':
309            if args:
310                self.stream.expect('comma')
311            arg = self.parse_assign_target(name_only=True)
312            arg.set_ctx('param')
313            if self.stream.skip_if('assign'):
314                defaults.append(self.parse_expression())
315            args.append(arg)
316        self.stream.expect('rparen')
317
318    def parse_call_block(self):
319        node = nodes.CallBlock(lineno=next(self.stream).lineno)
320        if self.stream.current.type == 'lparen':
321            self.parse_signature(node)
322        else:
323            node.args = []
324            node.defaults = []
325
326        node.call = self.parse_expression()
327        if not isinstance(node.call, nodes.Call):
328            self.fail('expected call', node.lineno)
329        node.body = self.parse_statements(('name:endcall',), drop_needle=True)
330        return node
331
332    def parse_filter_block(self):
333        node = nodes.FilterBlock(lineno=next(self.stream).lineno)
334        node.filter = self.parse_filter(None, start_inline=True)
335        node.body = self.parse_statements(('name:endfilter',),
336                                          drop_needle=True)
337        return node
338
339    def parse_macro(self):
340        node = nodes.Macro(lineno=next(self.stream).lineno)
341        node.name = self.parse_assign_target(name_only=True).name
342        self.parse_signature(node)
343        node.body = self.parse_statements(('name:endmacro',),
344                                          drop_needle=True)
345        return node
346
347    def parse_print(self):
348        node = nodes.Output(lineno=next(self.stream).lineno)
349        node.nodes = []
350        while self.stream.current.type != 'block_end':
351            if node.nodes:
352                self.stream.expect('comma')
353            node.nodes.append(self.parse_expression())
354        return node
355
356    def parse_assign_target(self, with_tuple=True, name_only=False,
357                            extra_end_rules=None):
358        """Parse an assignment target.  As Jinja2 allows assignments to
359        tuples, this function can parse all allowed assignment targets.  Per
360        default assignments to tuples are parsed, that can be disable however
361        by setting `with_tuple` to `False`.  If only assignments to names are
362        wanted `name_only` can be set to `True`.  The `extra_end_rules`
363        parameter is forwarded to the tuple parsing function.
364        """
365        if name_only:
366            token = self.stream.expect('name')
367            target = nodes.Name(token.value, 'store', lineno=token.lineno)
368        else:
369            if with_tuple:
370                target = self.parse_tuple(simplified=True,
371                                          extra_end_rules=extra_end_rules)
372            else:
373                target = self.parse_primary()
374            target.set_ctx('store')
375        if not target.can_assign():
376            self.fail('can\'t assign to %r' % target.__class__.
377                      __name__.lower(), target.lineno)
378        return target
379
380    def parse_expression(self, with_condexpr=True):
381        """Parse an expression.  Per default all expressions are parsed, if
382        the optional `with_condexpr` parameter is set to `False` conditional
383        expressions are not parsed.
384        """
385        if with_condexpr:
386            return self.parse_condexpr()
387        return self.parse_or()
388
389    def parse_condexpr(self):
390        lineno = self.stream.current.lineno
391        expr1 = self.parse_or()
392        while self.stream.skip_if('name:if'):
393            expr2 = self.parse_or()
394            if self.stream.skip_if('name:else'):
395                expr3 = self.parse_condexpr()
396            else:
397                expr3 = None
398            expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
399            lineno = self.stream.current.lineno
400        return expr1
401
402    def parse_or(self):
403        lineno = self.stream.current.lineno
404        left = self.parse_and()
405        while self.stream.skip_if('name:or'):
406            right = self.parse_and()
407            left = nodes.Or(left, right, lineno=lineno)
408            lineno = self.stream.current.lineno
409        return left
410
411    def parse_and(self):
412        lineno = self.stream.current.lineno
413        left = self.parse_not()
414        while self.stream.skip_if('name:and'):
415            right = self.parse_not()
416            left = nodes.And(left, right, lineno=lineno)
417            lineno = self.stream.current.lineno
418        return left
419
420    def parse_not(self):
421        if self.stream.current.test('name:not'):
422            lineno = next(self.stream).lineno
423            return nodes.Not(self.parse_not(), lineno=lineno)
424        return self.parse_compare()
425
426    def parse_compare(self):
427        lineno = self.stream.current.lineno
428        expr = self.parse_add()
429        ops = []
430        while 1:
431            token_type = self.stream.current.type
432            if token_type in _compare_operators:
433                next(self.stream)
434                ops.append(nodes.Operand(token_type, self.parse_add()))
435            elif self.stream.skip_if('name:in'):
436                ops.append(nodes.Operand('in', self.parse_add()))
437            elif self.stream.current.test('name:not') and \
438                 self.stream.look().test('name:in'):
439                self.stream.skip(2)
440                ops.append(nodes.Operand('notin', self.parse_add()))
441            else:
442                break
443            lineno = self.stream.current.lineno
444        if not ops:
445            return expr
446        return nodes.Compare(expr, ops, lineno=lineno)
447
448    def parse_add(self):
449        lineno = self.stream.current.lineno
450        left = self.parse_sub()
451        while self.stream.current.type == 'add':
452            next(self.stream)
453            right = self.parse_sub()
454            left = nodes.Add(left, right, lineno=lineno)
455            lineno = self.stream.current.lineno
456        return left
457
458    def parse_sub(self):
459        lineno = self.stream.current.lineno
460        left = self.parse_concat()
461        while self.stream.current.type == 'sub':
462            next(self.stream)
463            right = self.parse_concat()
464            left = nodes.Sub(left, right, lineno=lineno)
465            lineno = self.stream.current.lineno
466        return left
467
468    def parse_concat(self):
469        lineno = self.stream.current.lineno
470        args = [self.parse_mul()]
471        while self.stream.current.type == 'tilde':
472            next(self.stream)
473            args.append(self.parse_mul())
474        if len(args) == 1:
475            return args[0]
476        return nodes.Concat(args, lineno=lineno)
477
478    def parse_mul(self):
479        lineno = self.stream.current.lineno
480        left = self.parse_div()
481        while self.stream.current.type == 'mul':
482            next(self.stream)
483            right = self.parse_div()
484            left = nodes.Mul(left, right, lineno=lineno)
485            lineno = self.stream.current.lineno
486        return left
487
488    def parse_div(self):
489        lineno = self.stream.current.lineno
490        left = self.parse_floordiv()
491        while self.stream.current.type == 'div':
492            next(self.stream)
493            right = self.parse_floordiv()
494            left = nodes.Div(left, right, lineno=lineno)
495            lineno = self.stream.current.lineno
496        return left
497
498    def parse_floordiv(self):
499        lineno = self.stream.current.lineno
500        left = self.parse_mod()
501        while self.stream.current.type == 'floordiv':
502            next(self.stream)
503            right = self.parse_mod()
504            left = nodes.FloorDiv(left, right, lineno=lineno)
505            lineno = self.stream.current.lineno
506        return left
507
508    def parse_mod(self):
509        lineno = self.stream.current.lineno
510        left = self.parse_pow()
511        while self.stream.current.type == 'mod':
512            next(self.stream)
513            right = self.parse_pow()
514            left = nodes.Mod(left, right, lineno=lineno)
515            lineno = self.stream.current.lineno
516        return left
517
518    def parse_pow(self):
519        lineno = self.stream.current.lineno
520        left = self.parse_unary()
521        while self.stream.current.type == 'pow':
522            next(self.stream)
523            right = self.parse_unary()
524            left = nodes.Pow(left, right, lineno=lineno)
525            lineno = self.stream.current.lineno
526        return left
527
528    def parse_unary(self, with_filter=True):
529        token_type = self.stream.current.type
530        lineno = self.stream.current.lineno
531        if token_type == 'sub':
532            next(self.stream)
533            node = nodes.Neg(self.parse_unary(False), lineno=lineno)
534        elif token_type == 'add':
535            next(self.stream)
536            node = nodes.Pos(self.parse_unary(False), lineno=lineno)
537        else:
538            node = self.parse_primary()
539        node = self.parse_postfix(node)
540        if with_filter:
541            node = self.parse_filter_expr(node)
542        return node
543
544    def parse_primary(self):
545        token = self.stream.current
546        if token.type == 'name':
547            if token.value in ('true', 'false', 'True', 'False'):
548                node = nodes.Const(token.value in ('true', 'True'),
549                                   lineno=token.lineno)
550            elif token.value in ('none', 'None'):
551                node = nodes.Const(None, lineno=token.lineno)
552            else:
553                node = nodes.Name(token.value, 'load', lineno=token.lineno)
554            next(self.stream)
555        elif token.type == 'string':
556            next(self.stream)
557            buf = [token.value]
558            lineno = token.lineno
559            while self.stream.current.type == 'string':
560                buf.append(self.stream.current.value)
561                next(self.stream)
562            node = nodes.Const(''.join(buf), lineno=lineno)
563        elif token.type in ('integer', 'float'):
564            next(self.stream)
565            node = nodes.Const(token.value, lineno=token.lineno)
566        elif token.type == 'lparen':
567            next(self.stream)
568            node = self.parse_tuple(explicit_parentheses=True)
569            self.stream.expect('rparen')
570        elif token.type == 'lbracket':
571            node = self.parse_list()
572        elif token.type == 'lbrace':
573            node = self.parse_dict()
574        else:
575            self.fail("unexpected '%s'" % describe_token(token), token.lineno)
576        return node
577
578    def parse_tuple(self, simplified=False, with_condexpr=True,
579                    extra_end_rules=None, explicit_parentheses=False):
580        """Works like `parse_expression` but if multiple expressions are
581        delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
582        This method could also return a regular expression instead of a tuple
583        if no commas where found.
584
585        The default parsing mode is a full tuple.  If `simplified` is `True`
586        only names and literals are parsed.  The `no_condexpr` parameter is
587        forwarded to :meth:`parse_expression`.
588
589        Because tuples do not require delimiters and may end in a bogus comma
590        an extra hint is needed that marks the end of a tuple.  For example
591        for loops support tuples between `for` and `in`.  In that case the
592        `extra_end_rules` is set to ``['name:in']``.
593
594        `explicit_parentheses` is true if the parsing was triggered by an
595        expression in parentheses.  This is used to figure out if an empty
596        tuple is a valid expression or not.
597        """
598        lineno = self.stream.current.lineno
599        if simplified:
600            parse = self.parse_primary
601        elif with_condexpr:
602            parse = self.parse_expression
603        else:
604            parse = lambda: self.parse_expression(with_condexpr=False)
605        args = []
606        is_tuple = False
607        while 1:
608            if args:
609                self.stream.expect('comma')
610            if self.is_tuple_end(extra_end_rules):
611                break
612            args.append(parse())
613            if self.stream.current.type == 'comma':
614                is_tuple = True
615            else:
616                break
617            lineno = self.stream.current.lineno
618
619        if not is_tuple:
620            if args:
621                return args[0]
622
623            # if we don't have explicit parentheses, an empty tuple is
624            # not a valid expression.  This would mean nothing (literally
625            # nothing) in the spot of an expression would be an empty
626            # tuple.
627            if not explicit_parentheses:
628                self.fail('Expected an expression, got \'%s\'' %
629                          describe_token(self.stream.current))
630
631        return nodes.Tuple(args, 'load', lineno=lineno)
632
633    def parse_list(self):
634        token = self.stream.expect('lbracket')
635        items = []
636        while self.stream.current.type != 'rbracket':
637            if items:
638                self.stream.expect('comma')
639            if self.stream.current.type == 'rbracket':
640                break
641            items.append(self.parse_expression())
642        self.stream.expect('rbracket')
643        return nodes.List(items, lineno=token.lineno)
644
645    def parse_dict(self):
646        token = self.stream.expect('lbrace')
647        items = []
648        while self.stream.current.type != 'rbrace':
649            if items:
650                self.stream.expect('comma')
651            if self.stream.current.type == 'rbrace':
652                break
653            key = self.parse_expression()
654            self.stream.expect('colon')
655            value = self.parse_expression()
656            items.append(nodes.Pair(key, value, lineno=key.lineno))
657        self.stream.expect('rbrace')
658        return nodes.Dict(items, lineno=token.lineno)
659
660    def parse_postfix(self, node):
661        while 1:
662            token_type = self.stream.current.type
663            if token_type == 'dot' or token_type == 'lbracket':
664                node = self.parse_subscript(node)
665            # calls are valid both after postfix expressions (getattr
666            # and getitem) as well as filters and tests
667            elif token_type == 'lparen':
668                node = self.parse_call(node)
669            else:
670                break
671        return node
672
673    def parse_filter_expr(self, node):
674        while 1:
675            token_type = self.stream.current.type
676            if token_type == 'pipe':
677                node = self.parse_filter(node)
678            elif token_type == 'name' and self.stream.current.value == 'is':
679                node = self.parse_test(node)
680            # calls are valid both after postfix expressions (getattr
681            # and getitem) as well as filters and tests
682            elif token_type == 'lparen':
683                node = self.parse_call(node)
684            else:
685                break
686        return node
687
688    def parse_subscript(self, node):
689        token = next(self.stream)
690        if token.type == 'dot':
691            attr_token = self.stream.current
692            next(self.stream)
693            if attr_token.type == 'name':
694                return nodes.Getattr(node, attr_token.value, 'load',
695                                     lineno=token.lineno)
696            elif attr_token.type != 'integer':
697                self.fail('expected name or number', attr_token.lineno)
698            arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
699            return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
700        if token.type == 'lbracket':
701            args = []
702            while self.stream.current.type != 'rbracket':
703                if args:
704                    self.stream.expect('comma')
705                args.append(self.parse_subscribed())
706            self.stream.expect('rbracket')
707            if len(args) == 1:
708                arg = args[0]
709            else:
710                arg = nodes.Tuple(args, 'load', lineno=token.lineno)
711            return nodes.Getitem(node, arg, 'load', lineno=token.lineno)
712        self.fail('expected subscript expression', self.lineno)
713
714    def parse_subscribed(self):
715        lineno = self.stream.current.lineno
716
717        if self.stream.current.type == 'colon':
718            next(self.stream)
719            args = [None]
720        else:
721            node = self.parse_expression()
722            if self.stream.current.type != 'colon':
723                return node
724            next(self.stream)
725            args = [node]
726
727        if self.stream.current.type == 'colon':
728            args.append(None)
729        elif self.stream.current.type not in ('rbracket', 'comma'):
730            args.append(self.parse_expression())
731        else:
732            args.append(None)
733
734        if self.stream.current.type == 'colon':
735            next(self.stream)
736            if self.stream.current.type not in ('rbracket', 'comma'):
737                args.append(self.parse_expression())
738            else:
739                args.append(None)
740        else:
741            args.append(None)
742
743        return nodes.Slice(lineno=lineno, *args)
744
745    def parse_call(self, node):
746        token = self.stream.expect('lparen')
747        args = []
748        kwargs = []
749        dyn_args = dyn_kwargs = None
750        require_comma = False
751
752        def ensure(expr):
753            if not expr:
754                self.fail('invalid syntax for function call expression',
755                          token.lineno)
756
757        while self.stream.current.type != 'rparen':
758            if require_comma:
759                self.stream.expect('comma')
760                # support for trailing comma
761                if self.stream.current.type == 'rparen':
762                    break
763            if self.stream.current.type == 'mul':
764                ensure(dyn_args is None and dyn_kwargs is None)
765                next(self.stream)
766                dyn_args = self.parse_expression()
767            elif self.stream.current.type == 'pow':
768                ensure(dyn_kwargs is None)
769                next(self.stream)
770                dyn_kwargs = self.parse_expression()
771            else:
772                ensure(dyn_args is None and dyn_kwargs is None)
773                if self.stream.current.type == 'name' and \
774                    self.stream.look().type == 'assign':
775                    key = self.stream.current.value
776                    self.stream.skip(2)
777                    value = self.parse_expression()
778                    kwargs.append(nodes.Keyword(key, value,
779                                                lineno=value.lineno))
780                else:
781                    ensure(not kwargs)
782                    args.append(self.parse_expression())
783
784            require_comma = True
785        self.stream.expect('rparen')
786
787        if node is None:
788            return args, kwargs, dyn_args, dyn_kwargs
789        return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs,
790                          lineno=token.lineno)
791
792    def parse_filter(self, node, start_inline=False):
793        while self.stream.current.type == 'pipe' or start_inline:
794            if not start_inline:
795                next(self.stream)
796            token = self.stream.expect('name')
797            name = token.value
798            while self.stream.current.type == 'dot':
799                next(self.stream)
800                name += '.' + self.stream.expect('name').value
801            if self.stream.current.type == 'lparen':
802                args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
803            else:
804                args = []
805                kwargs = []
806                dyn_args = dyn_kwargs = None
807            node = nodes.Filter(node, name, args, kwargs, dyn_args,
808                                dyn_kwargs, lineno=token.lineno)
809            start_inline = False
810        return node
811
812    def parse_test(self, node):
813        token = next(self.stream)
814        if self.stream.current.test('name:not'):
815            next(self.stream)
816            negated = True
817        else:
818            negated = False
819        name = self.stream.expect('name').value
820        while self.stream.current.type == 'dot':
821            next(self.stream)
822            name += '.' + self.stream.expect('name').value
823        dyn_args = dyn_kwargs = None
824        kwargs = []
825        if self.stream.current.type == 'lparen':
826            args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
827        elif self.stream.current.type in ('name', 'string', 'integer',
828                                          'float', 'lparen', 'lbracket',
829                                          'lbrace') and not \
830             self.stream.current.test_any('name:else', 'name:or',
831                                          'name:and'):
832            if self.stream.current.test('name:is'):
833                self.fail('You cannot chain multiple tests with is')
834            args = [self.parse_expression()]
835        else:
836            args = []
837        node = nodes.Test(node, name, args, kwargs, dyn_args,
838                          dyn_kwargs, lineno=token.lineno)
839        if negated:
840            node = nodes.Not(node, lineno=token.lineno)
841        return node
842
843    def subparse(self, end_tokens=None):
844        body = []
845        data_buffer = []
846        add_data = data_buffer.append
847
848        if end_tokens is not None:
849            self._end_token_stack.append(end_tokens)
850
851        def flush_data():
852            if data_buffer:
853                lineno = data_buffer[0].lineno
854                body.append(nodes.Output(data_buffer[:], lineno=lineno))
855                del data_buffer[:]
856
857        try:
858            while self.stream:
859                token = self.stream.current
860                if token.type == 'data':
861                    if token.value:
862                        add_data(nodes.TemplateData(token.value,
863                                                    lineno=token.lineno))
864                    next(self.stream)
865                elif token.type == 'variable_begin':
866                    next(self.stream)
867                    add_data(self.parse_tuple(with_condexpr=True))
868                    self.stream.expect('variable_end')
869                elif token.type == 'block_begin':
870                    flush_data()
871                    next(self.stream)
872                    if end_tokens is not None and \
873                       self.stream.current.test_any(*end_tokens):
874                        return body
875                    rv = self.parse_statement()
876                    if isinstance(rv, list):
877                        body.extend(rv)
878                    else:
879                        body.append(rv)
880                    self.stream.expect('block_end')
881                else:
882                    raise AssertionError('internal parsing error')
883
884            flush_data()
885        finally:
886            if end_tokens is not None:
887                self._end_token_stack.pop()
888
889        return body
890
891    def parse(self):
892        """Parse the whole template into a `Template` node."""
893        result = nodes.Template(self.subparse(), lineno=1)
894        result.set_environment(self.environment)
895        return result
896