• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2"""
3    jinja2.compiler
4    ~~~~~~~~~~~~~~~
5
6    Compiles nodes into python code.
7
8    :copyright: (c) 2017 by the Jinja Team.
9    :license: BSD, see LICENSE for more details.
10"""
11from itertools import chain
12from copy import deepcopy
13from keyword import iskeyword as is_python_keyword
14from functools import update_wrapper
15from jinja2 import nodes
16from jinja2.nodes import EvalContext
17from jinja2.visitor import NodeVisitor
18from jinja2.optimizer import Optimizer
19from jinja2.exceptions import TemplateAssertionError
20from jinja2.utils import Markup, concat, escape
21from jinja2._compat import range_type, text_type, string_types, \
22     iteritems, NativeStringIO, imap, izip
23from jinja2.idtracking import Symbols, VAR_LOAD_PARAMETER, \
24     VAR_LOAD_RESOLVE, VAR_LOAD_ALIAS, VAR_LOAD_UNDEFINED
25
26
27operators = {
28    'eq':       '==',
29    'ne':       '!=',
30    'gt':       '>',
31    'gteq':     '>=',
32    'lt':       '<',
33    'lteq':     '<=',
34    'in':       'in',
35    'notin':    'not in'
36}
37
38# what method to iterate over items do we want to use for dict iteration
39# in generated code?  on 2.x let's go with iteritems, on 3.x with items
40if hasattr(dict, 'iteritems'):
41    dict_item_iter = 'iteritems'
42else:
43    dict_item_iter = 'items'
44
45code_features = ['division']
46
47# does this python version support generator stops? (PEP 0479)
48try:
49    exec('from __future__ import generator_stop')
50    code_features.append('generator_stop')
51except SyntaxError:
52    pass
53
54# does this python version support yield from?
55try:
56    exec('def f(): yield from x()')
57except SyntaxError:
58    supports_yield_from = False
59else:
60    supports_yield_from = True
61
62
63def optimizeconst(f):
64    def new_func(self, node, frame, **kwargs):
65        # Only optimize if the frame is not volatile
66        if self.optimized and not frame.eval_ctx.volatile:
67            new_node = self.optimizer.visit(node, frame.eval_ctx)
68            if new_node != node:
69                return self.visit(new_node, frame)
70        return f(self, node, frame, **kwargs)
71    return update_wrapper(new_func, f)
72
73
74def generate(node, environment, name, filename, stream=None,
75             defer_init=False, optimized=True):
76    """Generate the python source for a node tree."""
77    if not isinstance(node, nodes.Template):
78        raise TypeError('Can\'t compile non template nodes')
79    generator = environment.code_generator_class(environment, name, filename,
80                                                 stream, defer_init,
81                                                 optimized)
82    generator.visit(node)
83    if stream is None:
84        return generator.stream.getvalue()
85
86
87def has_safe_repr(value):
88    """Does the node have a safe representation?"""
89    if value is None or value is NotImplemented or value is Ellipsis:
90        return True
91    if type(value) in (bool, int, float, complex, range_type, Markup) + string_types:
92        return True
93    if type(value) in (tuple, list, set, frozenset):
94        for item in value:
95            if not has_safe_repr(item):
96                return False
97        return True
98    elif type(value) is dict:
99        for key, value in iteritems(value):
100            if not has_safe_repr(key):
101                return False
102            if not has_safe_repr(value):
103                return False
104        return True
105    return False
106
107
108def find_undeclared(nodes, names):
109    """Check if the names passed are accessed undeclared.  The return value
110    is a set of all the undeclared names from the sequence of names found.
111    """
112    visitor = UndeclaredNameVisitor(names)
113    try:
114        for node in nodes:
115            visitor.visit(node)
116    except VisitorExit:
117        pass
118    return visitor.undeclared
119
120
121class MacroRef(object):
122
123    def __init__(self, node):
124        self.node = node
125        self.accesses_caller = False
126        self.accesses_kwargs = False
127        self.accesses_varargs = False
128
129
130class Frame(object):
131    """Holds compile time information for us."""
132
133    def __init__(self, eval_ctx, parent=None, level=None):
134        self.eval_ctx = eval_ctx
135        self.symbols = Symbols(parent and parent.symbols or None,
136                               level=level)
137
138        # a toplevel frame is the root + soft frames such as if conditions.
139        self.toplevel = False
140
141        # the root frame is basically just the outermost frame, so no if
142        # conditions.  This information is used to optimize inheritance
143        # situations.
144        self.rootlevel = False
145
146        # in some dynamic inheritance situations the compiler needs to add
147        # write tests around output statements.
148        self.require_output_check = parent and parent.require_output_check
149
150        # inside some tags we are using a buffer rather than yield statements.
151        # this for example affects {% filter %} or {% macro %}.  If a frame
152        # is buffered this variable points to the name of the list used as
153        # buffer.
154        self.buffer = None
155
156        # the name of the block we're in, otherwise None.
157        self.block = parent and parent.block or None
158
159        # the parent of this frame
160        self.parent = parent
161
162        if parent is not None:
163            self.buffer = parent.buffer
164
165    def copy(self):
166        """Create a copy of the current one."""
167        rv = object.__new__(self.__class__)
168        rv.__dict__.update(self.__dict__)
169        rv.symbols = self.symbols.copy()
170        return rv
171
172    def inner(self, isolated=False):
173        """Return an inner frame."""
174        if isolated:
175            return Frame(self.eval_ctx, level=self.symbols.level + 1)
176        return Frame(self.eval_ctx, self)
177
178    def soft(self):
179        """Return a soft frame.  A soft frame may not be modified as
180        standalone thing as it shares the resources with the frame it
181        was created of, but it's not a rootlevel frame any longer.
182
183        This is only used to implement if-statements.
184        """
185        rv = self.copy()
186        rv.rootlevel = False
187        return rv
188
189    __copy__ = copy
190
191
192class VisitorExit(RuntimeError):
193    """Exception used by the `UndeclaredNameVisitor` to signal a stop."""
194
195
196class DependencyFinderVisitor(NodeVisitor):
197    """A visitor that collects filter and test calls."""
198
199    def __init__(self):
200        self.filters = set()
201        self.tests = set()
202
203    def visit_Filter(self, node):
204        self.generic_visit(node)
205        self.filters.add(node.name)
206
207    def visit_Test(self, node):
208        self.generic_visit(node)
209        self.tests.add(node.name)
210
211    def visit_Block(self, node):
212        """Stop visiting at blocks."""
213
214
215class UndeclaredNameVisitor(NodeVisitor):
216    """A visitor that checks if a name is accessed without being
217    declared.  This is different from the frame visitor as it will
218    not stop at closure frames.
219    """
220
221    def __init__(self, names):
222        self.names = set(names)
223        self.undeclared = set()
224
225    def visit_Name(self, node):
226        if node.ctx == 'load' and node.name in self.names:
227            self.undeclared.add(node.name)
228            if self.undeclared == self.names:
229                raise VisitorExit()
230        else:
231            self.names.discard(node.name)
232
233    def visit_Block(self, node):
234        """Stop visiting a blocks."""
235
236
237class CompilerExit(Exception):
238    """Raised if the compiler encountered a situation where it just
239    doesn't make sense to further process the code.  Any block that
240    raises such an exception is not further processed.
241    """
242
243
244class CodeGenerator(NodeVisitor):
245
246    def __init__(self, environment, name, filename, stream=None,
247                 defer_init=False, optimized=True):
248        if stream is None:
249            stream = NativeStringIO()
250        self.environment = environment
251        self.name = name
252        self.filename = filename
253        self.stream = stream
254        self.created_block_context = False
255        self.defer_init = defer_init
256        self.optimized = optimized
257        if optimized:
258            self.optimizer = Optimizer(environment)
259
260        # aliases for imports
261        self.import_aliases = {}
262
263        # a registry for all blocks.  Because blocks are moved out
264        # into the global python scope they are registered here
265        self.blocks = {}
266
267        # the number of extends statements so far
268        self.extends_so_far = 0
269
270        # some templates have a rootlevel extends.  In this case we
271        # can safely assume that we're a child template and do some
272        # more optimizations.
273        self.has_known_extends = False
274
275        # the current line number
276        self.code_lineno = 1
277
278        # registry of all filters and tests (global, not block local)
279        self.tests = {}
280        self.filters = {}
281
282        # the debug information
283        self.debug_info = []
284        self._write_debug_info = None
285
286        # the number of new lines before the next write()
287        self._new_lines = 0
288
289        # the line number of the last written statement
290        self._last_line = 0
291
292        # true if nothing was written so far.
293        self._first_write = True
294
295        # used by the `temporary_identifier` method to get new
296        # unique, temporary identifier
297        self._last_identifier = 0
298
299        # the current indentation
300        self._indentation = 0
301
302        # Tracks toplevel assignments
303        self._assign_stack = []
304
305        # Tracks parameter definition blocks
306        self._param_def_block = []
307
308        # Tracks the current context.
309        self._context_reference_stack = ['context']
310
311    # -- Various compilation helpers
312
313    def fail(self, msg, lineno):
314        """Fail with a :exc:`TemplateAssertionError`."""
315        raise TemplateAssertionError(msg, lineno, self.name, self.filename)
316
317    def temporary_identifier(self):
318        """Get a new unique identifier."""
319        self._last_identifier += 1
320        return 't_%d' % self._last_identifier
321
322    def buffer(self, frame):
323        """Enable buffering for the frame from that point onwards."""
324        frame.buffer = self.temporary_identifier()
325        self.writeline('%s = []' % frame.buffer)
326
327    def return_buffer_contents(self, frame, force_unescaped=False):
328        """Return the buffer contents of the frame."""
329        if not force_unescaped:
330            if frame.eval_ctx.volatile:
331                self.writeline('if context.eval_ctx.autoescape:')
332                self.indent()
333                self.writeline('return Markup(concat(%s))' % frame.buffer)
334                self.outdent()
335                self.writeline('else:')
336                self.indent()
337                self.writeline('return concat(%s)' % frame.buffer)
338                self.outdent()
339                return
340            elif frame.eval_ctx.autoescape:
341                self.writeline('return Markup(concat(%s))' % frame.buffer)
342                return
343        self.writeline('return concat(%s)' % frame.buffer)
344
345    def indent(self):
346        """Indent by one."""
347        self._indentation += 1
348
349    def outdent(self, step=1):
350        """Outdent by step."""
351        self._indentation -= step
352
353    def start_write(self, frame, node=None):
354        """Yield or write into the frame buffer."""
355        if frame.buffer is None:
356            self.writeline('yield ', node)
357        else:
358            self.writeline('%s.append(' % frame.buffer, node)
359
360    def end_write(self, frame):
361        """End the writing process started by `start_write`."""
362        if frame.buffer is not None:
363            self.write(')')
364
365    def simple_write(self, s, frame, node=None):
366        """Simple shortcut for start_write + write + end_write."""
367        self.start_write(frame, node)
368        self.write(s)
369        self.end_write(frame)
370
371    def blockvisit(self, nodes, frame):
372        """Visit a list of nodes as block in a frame.  If the current frame
373        is no buffer a dummy ``if 0: yield None`` is written automatically.
374        """
375        try:
376            self.writeline('pass')
377            for node in nodes:
378                self.visit(node, frame)
379        except CompilerExit:
380            pass
381
382    def write(self, x):
383        """Write a string into the output stream."""
384        if self._new_lines:
385            if not self._first_write:
386                self.stream.write('\n' * self._new_lines)
387                self.code_lineno += self._new_lines
388                if self._write_debug_info is not None:
389                    self.debug_info.append((self._write_debug_info,
390                                            self.code_lineno))
391                    self._write_debug_info = None
392            self._first_write = False
393            self.stream.write('    ' * self._indentation)
394            self._new_lines = 0
395        self.stream.write(x)
396
397    def writeline(self, x, node=None, extra=0):
398        """Combination of newline and write."""
399        self.newline(node, extra)
400        self.write(x)
401
402    def newline(self, node=None, extra=0):
403        """Add one or more newlines before the next write."""
404        self._new_lines = max(self._new_lines, 1 + extra)
405        if node is not None and node.lineno != self._last_line:
406            self._write_debug_info = node.lineno
407            self._last_line = node.lineno
408
409    def signature(self, node, frame, extra_kwargs=None):
410        """Writes a function call to the stream for the current node.
411        A leading comma is added automatically.  The extra keyword
412        arguments may not include python keywords otherwise a syntax
413        error could occour.  The extra keyword arguments should be given
414        as python dict.
415        """
416        # if any of the given keyword arguments is a python keyword
417        # we have to make sure that no invalid call is created.
418        kwarg_workaround = False
419        for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()):
420            if is_python_keyword(kwarg):
421                kwarg_workaround = True
422                break
423
424        for arg in node.args:
425            self.write(', ')
426            self.visit(arg, frame)
427
428        if not kwarg_workaround:
429            for kwarg in node.kwargs:
430                self.write(', ')
431                self.visit(kwarg, frame)
432            if extra_kwargs is not None:
433                for key, value in iteritems(extra_kwargs):
434                    self.write(', %s=%s' % (key, value))
435        if node.dyn_args:
436            self.write(', *')
437            self.visit(node.dyn_args, frame)
438
439        if kwarg_workaround:
440            if node.dyn_kwargs is not None:
441                self.write(', **dict({')
442            else:
443                self.write(', **{')
444            for kwarg in node.kwargs:
445                self.write('%r: ' % kwarg.key)
446                self.visit(kwarg.value, frame)
447                self.write(', ')
448            if extra_kwargs is not None:
449                for key, value in iteritems(extra_kwargs):
450                    self.write('%r: %s, ' % (key, value))
451            if node.dyn_kwargs is not None:
452                self.write('}, **')
453                self.visit(node.dyn_kwargs, frame)
454                self.write(')')
455            else:
456                self.write('}')
457
458        elif node.dyn_kwargs is not None:
459            self.write(', **')
460            self.visit(node.dyn_kwargs, frame)
461
462    def pull_dependencies(self, nodes):
463        """Pull all the dependencies."""
464        visitor = DependencyFinderVisitor()
465        for node in nodes:
466            visitor.visit(node)
467        for dependency in 'filters', 'tests':
468            mapping = getattr(self, dependency)
469            for name in getattr(visitor, dependency):
470                if name not in mapping:
471                    mapping[name] = self.temporary_identifier()
472                self.writeline('%s = environment.%s[%r]' %
473                               (mapping[name], dependency, name))
474
475    def enter_frame(self, frame):
476        undefs = []
477        for target, (action, param) in iteritems(frame.symbols.loads):
478            if action == VAR_LOAD_PARAMETER:
479                pass
480            elif action == VAR_LOAD_RESOLVE:
481                self.writeline('%s = %s(%r)' %
482                               (target, self.get_resolve_func(), param))
483            elif action == VAR_LOAD_ALIAS:
484                self.writeline('%s = %s' % (target, param))
485            elif action == VAR_LOAD_UNDEFINED:
486                undefs.append(target)
487            else:
488                raise NotImplementedError('unknown load instruction')
489        if undefs:
490            self.writeline('%s = missing' % ' = '.join(undefs))
491
492    def leave_frame(self, frame, with_python_scope=False):
493        if not with_python_scope:
494            undefs = []
495            for target, _ in iteritems(frame.symbols.loads):
496                undefs.append(target)
497            if undefs:
498                self.writeline('%s = missing' % ' = '.join(undefs))
499
500    def func(self, name):
501        if self.environment.is_async:
502            return 'async def %s' % name
503        return 'def %s' % name
504
505    def macro_body(self, node, frame):
506        """Dump the function def of a macro or call block."""
507        frame = frame.inner()
508        frame.symbols.analyze_node(node)
509        macro_ref = MacroRef(node)
510
511        explicit_caller = None
512        skip_special_params = set()
513        args = []
514        for idx, arg in enumerate(node.args):
515            if arg.name == 'caller':
516                explicit_caller = idx
517            if arg.name in ('kwargs', 'varargs'):
518                skip_special_params.add(arg.name)
519            args.append(frame.symbols.ref(arg.name))
520
521        undeclared = find_undeclared(node.body, ('caller', 'kwargs', 'varargs'))
522
523        if 'caller' in undeclared:
524            # In older Jinja2 versions there was a bug that allowed caller
525            # to retain the special behavior even if it was mentioned in
526            # the argument list.  However thankfully this was only really
527            # working if it was the last argument.  So we are explicitly
528            # checking this now and error out if it is anywhere else in
529            # the argument list.
530            if explicit_caller is not None:
531                try:
532                    node.defaults[explicit_caller - len(node.args)]
533                except IndexError:
534                    self.fail('When defining macros or call blocks the '
535                              'special "caller" argument must be omitted '
536                              'or be given a default.', node.lineno)
537            else:
538                args.append(frame.symbols.declare_parameter('caller'))
539            macro_ref.accesses_caller = True
540        if 'kwargs' in undeclared and not 'kwargs' in skip_special_params:
541            args.append(frame.symbols.declare_parameter('kwargs'))
542            macro_ref.accesses_kwargs = True
543        if 'varargs' in undeclared and not 'varargs' in skip_special_params:
544            args.append(frame.symbols.declare_parameter('varargs'))
545            macro_ref.accesses_varargs = True
546
547        # macros are delayed, they never require output checks
548        frame.require_output_check = False
549        frame.symbols.analyze_node(node)
550        self.writeline('%s(%s):' % (self.func('macro'), ', '.join(args)), node)
551        self.indent()
552
553        self.buffer(frame)
554        self.enter_frame(frame)
555
556        self.push_parameter_definitions(frame)
557        for idx, arg in enumerate(node.args):
558            ref = frame.symbols.ref(arg.name)
559            self.writeline('if %s is missing:' % ref)
560            self.indent()
561            try:
562                default = node.defaults[idx - len(node.args)]
563            except IndexError:
564                self.writeline('%s = undefined(%r, name=%r)' % (
565                    ref,
566                    'parameter %r was not provided' % arg.name,
567                    arg.name))
568            else:
569                self.writeline('%s = ' % ref)
570                self.visit(default, frame)
571            self.mark_parameter_stored(ref)
572            self.outdent()
573        self.pop_parameter_definitions()
574
575        self.blockvisit(node.body, frame)
576        self.return_buffer_contents(frame, force_unescaped=True)
577        self.leave_frame(frame, with_python_scope=True)
578        self.outdent()
579
580        return frame, macro_ref
581
582    def macro_def(self, macro_ref, frame):
583        """Dump the macro definition for the def created by macro_body."""
584        arg_tuple = ', '.join(repr(x.name) for x in macro_ref.node.args)
585        name = getattr(macro_ref.node, 'name', None)
586        if len(macro_ref.node.args) == 1:
587            arg_tuple += ','
588        self.write('Macro(environment, macro, %r, (%s), %r, %r, %r, '
589                   'context.eval_ctx.autoescape)' %
590                   (name, arg_tuple, macro_ref.accesses_kwargs,
591                    macro_ref.accesses_varargs, macro_ref.accesses_caller))
592
593    def position(self, node):
594        """Return a human readable position for the node."""
595        rv = 'line %d' % node.lineno
596        if self.name is not None:
597            rv += ' in ' + repr(self.name)
598        return rv
599
600    def dump_local_context(self, frame):
601        return '{%s}' % ', '.join(
602            '%r: %s' % (name, target) for name, target
603            in iteritems(frame.symbols.dump_stores()))
604
605    def write_commons(self):
606        """Writes a common preamble that is used by root and block functions.
607        Primarily this sets up common local helpers and enforces a generator
608        through a dead branch.
609        """
610        self.writeline('resolve = context.resolve_or_missing')
611        self.writeline('undefined = environment.undefined')
612        self.writeline('if 0: yield None')
613
614    def push_parameter_definitions(self, frame):
615        """Pushes all parameter targets from the given frame into a local
616        stack that permits tracking of yet to be assigned parameters.  In
617        particular this enables the optimization from `visit_Name` to skip
618        undefined expressions for parameters in macros as macros can reference
619        otherwise unbound parameters.
620        """
621        self._param_def_block.append(frame.symbols.dump_param_targets())
622
623    def pop_parameter_definitions(self):
624        """Pops the current parameter definitions set."""
625        self._param_def_block.pop()
626
627    def mark_parameter_stored(self, target):
628        """Marks a parameter in the current parameter definitions as stored.
629        This will skip the enforced undefined checks.
630        """
631        if self._param_def_block:
632            self._param_def_block[-1].discard(target)
633
634    def push_context_reference(self, target):
635        self._context_reference_stack.append(target)
636
637    def pop_context_reference(self):
638        self._context_reference_stack.pop()
639
640    def get_context_ref(self):
641        return self._context_reference_stack[-1]
642
643    def get_resolve_func(self):
644        target = self._context_reference_stack[-1]
645        if target == 'context':
646            return 'resolve'
647        return '%s.resolve' % target
648
649    def derive_context(self, frame):
650        return '%s.derived(%s)' % (
651            self.get_context_ref(),
652            self.dump_local_context(frame),
653        )
654
655    def parameter_is_undeclared(self, target):
656        """Checks if a given target is an undeclared parameter."""
657        if not self._param_def_block:
658            return False
659        return target in self._param_def_block[-1]
660
661    def push_assign_tracking(self):
662        """Pushes a new layer for assignment tracking."""
663        self._assign_stack.append(set())
664
665    def pop_assign_tracking(self, frame):
666        """Pops the topmost level for assignment tracking and updates the
667        context variables if necessary.
668        """
669        vars = self._assign_stack.pop()
670        if not frame.toplevel or not vars:
671            return
672        public_names = [x for x in vars if x[:1] != '_']
673        if len(vars) == 1:
674            name = next(iter(vars))
675            ref = frame.symbols.ref(name)
676            self.writeline('context.vars[%r] = %s' % (name, ref))
677        else:
678            self.writeline('context.vars.update({')
679            for idx, name in enumerate(vars):
680                if idx:
681                    self.write(', ')
682                ref = frame.symbols.ref(name)
683                self.write('%r: %s' % (name, ref))
684            self.write('})')
685        if public_names:
686            if len(public_names) == 1:
687                self.writeline('context.exported_vars.add(%r)' %
688                               public_names[0])
689            else:
690                self.writeline('context.exported_vars.update((%s))' %
691                               ', '.join(imap(repr, public_names)))
692
693    # -- Statement Visitors
694
695    def visit_Template(self, node, frame=None):
696        assert frame is None, 'no root frame allowed'
697        eval_ctx = EvalContext(self.environment, self.name)
698
699        from jinja2.runtime import __all__ as exported
700        self.writeline('from __future__ import %s' % ', '.join(code_features))
701        self.writeline('from jinja2.runtime import ' + ', '.join(exported))
702
703        if self.environment.is_async:
704            self.writeline('from jinja2.asyncsupport import auto_await, '
705                           'auto_aiter, make_async_loop_context')
706
707        # if we want a deferred initialization we cannot move the
708        # environment into a local name
709        envenv = not self.defer_init and ', environment=environment' or ''
710
711        # do we have an extends tag at all?  If not, we can save some
712        # overhead by just not processing any inheritance code.
713        have_extends = node.find(nodes.Extends) is not None
714
715        # find all blocks
716        for block in node.find_all(nodes.Block):
717            if block.name in self.blocks:
718                self.fail('block %r defined twice' % block.name, block.lineno)
719            self.blocks[block.name] = block
720
721        # find all imports and import them
722        for import_ in node.find_all(nodes.ImportedName):
723            if import_.importname not in self.import_aliases:
724                imp = import_.importname
725                self.import_aliases[imp] = alias = self.temporary_identifier()
726                if '.' in imp:
727                    module, obj = imp.rsplit('.', 1)
728                    self.writeline('from %s import %s as %s' %
729                                   (module, obj, alias))
730                else:
731                    self.writeline('import %s as %s' % (imp, alias))
732
733        # add the load name
734        self.writeline('name = %r' % self.name)
735
736        # generate the root render function.
737        self.writeline('%s(context, missing=missing%s):' %
738                       (self.func('root'), envenv), extra=1)
739        self.indent()
740        self.write_commons()
741
742        # process the root
743        frame = Frame(eval_ctx)
744        if 'self' in find_undeclared(node.body, ('self',)):
745            ref = frame.symbols.declare_parameter('self')
746            self.writeline('%s = TemplateReference(context)' % ref)
747        frame.symbols.analyze_node(node)
748        frame.toplevel = frame.rootlevel = True
749        frame.require_output_check = have_extends and not self.has_known_extends
750        if have_extends:
751            self.writeline('parent_template = None')
752        self.enter_frame(frame)
753        self.pull_dependencies(node.body)
754        self.blockvisit(node.body, frame)
755        self.leave_frame(frame, with_python_scope=True)
756        self.outdent()
757
758        # make sure that the parent root is called.
759        if have_extends:
760            if not self.has_known_extends:
761                self.indent()
762                self.writeline('if parent_template is not None:')
763            self.indent()
764            if supports_yield_from and not self.environment.is_async:
765                self.writeline('yield from parent_template.'
766                               'root_render_func(context)')
767            else:
768                self.writeline('%sfor event in parent_template.'
769                               'root_render_func(context):' %
770                               (self.environment.is_async and 'async ' or ''))
771                self.indent()
772                self.writeline('yield event')
773                self.outdent()
774            self.outdent(1 + (not self.has_known_extends))
775
776        # at this point we now have the blocks collected and can visit them too.
777        for name, block in iteritems(self.blocks):
778            self.writeline('%s(context, missing=missing%s):' %
779                           (self.func('block_' + name), envenv),
780                           block, 1)
781            self.indent()
782            self.write_commons()
783            # It's important that we do not make this frame a child of the
784            # toplevel template.  This would cause a variety of
785            # interesting issues with identifier tracking.
786            block_frame = Frame(eval_ctx)
787            undeclared = find_undeclared(block.body, ('self', 'super'))
788            if 'self' in undeclared:
789                ref = block_frame.symbols.declare_parameter('self')
790                self.writeline('%s = TemplateReference(context)' % ref)
791            if 'super' in undeclared:
792                ref = block_frame.symbols.declare_parameter('super')
793                self.writeline('%s = context.super(%r, '
794                               'block_%s)' % (ref, name, name))
795            block_frame.symbols.analyze_node(block)
796            block_frame.block = name
797            self.enter_frame(block_frame)
798            self.pull_dependencies(block.body)
799            self.blockvisit(block.body, block_frame)
800            self.leave_frame(block_frame, with_python_scope=True)
801            self.outdent()
802
803        self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x)
804                                                   for x in self.blocks),
805                       extra=1)
806
807        # add a function that returns the debug info
808        self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x
809                                                    in self.debug_info))
810
811    def visit_Block(self, node, frame):
812        """Call a block and register it for the template."""
813        level = 0
814        if frame.toplevel:
815            # if we know that we are a child template, there is no need to
816            # check if we are one
817            if self.has_known_extends:
818                return
819            if self.extends_so_far > 0:
820                self.writeline('if parent_template is None:')
821                self.indent()
822                level += 1
823
824        if node.scoped:
825            context = self.derive_context(frame)
826        else:
827            context = self.get_context_ref()
828
829        if supports_yield_from and not self.environment.is_async and \
830           frame.buffer is None:
831            self.writeline('yield from context.blocks[%r][0](%s)' % (
832                           node.name, context), node)
833        else:
834            loop = self.environment.is_async and 'async for' or 'for'
835            self.writeline('%s event in context.blocks[%r][0](%s):' % (
836                           loop, node.name, context), node)
837            self.indent()
838            self.simple_write('event', frame)
839            self.outdent()
840
841        self.outdent(level)
842
843    def visit_Extends(self, node, frame):
844        """Calls the extender."""
845        if not frame.toplevel:
846            self.fail('cannot use extend from a non top-level scope',
847                      node.lineno)
848
849        # if the number of extends statements in general is zero so
850        # far, we don't have to add a check if something extended
851        # the template before this one.
852        if self.extends_so_far > 0:
853
854            # if we have a known extends we just add a template runtime
855            # error into the generated code.  We could catch that at compile
856            # time too, but i welcome it not to confuse users by throwing the
857            # same error at different times just "because we can".
858            if not self.has_known_extends:
859                self.writeline('if parent_template is not None:')
860                self.indent()
861            self.writeline('raise TemplateRuntimeError(%r)' %
862                           'extended multiple times')
863
864            # if we have a known extends already we don't need that code here
865            # as we know that the template execution will end here.
866            if self.has_known_extends:
867                raise CompilerExit()
868            else:
869                self.outdent()
870
871        self.writeline('parent_template = environment.get_template(', node)
872        self.visit(node.template, frame)
873        self.write(', %r)' % self.name)
874        self.writeline('for name, parent_block in parent_template.'
875                       'blocks.%s():' % dict_item_iter)
876        self.indent()
877        self.writeline('context.blocks.setdefault(name, []).'
878                       'append(parent_block)')
879        self.outdent()
880
881        # if this extends statement was in the root level we can take
882        # advantage of that information and simplify the generated code
883        # in the top level from this point onwards
884        if frame.rootlevel:
885            self.has_known_extends = True
886
887        # and now we have one more
888        self.extends_so_far += 1
889
890    def visit_Include(self, node, frame):
891        """Handles includes."""
892        if node.ignore_missing:
893            self.writeline('try:')
894            self.indent()
895
896        func_name = 'get_or_select_template'
897        if isinstance(node.template, nodes.Const):
898            if isinstance(node.template.value, string_types):
899                func_name = 'get_template'
900            elif isinstance(node.template.value, (tuple, list)):
901                func_name = 'select_template'
902        elif isinstance(node.template, (nodes.Tuple, nodes.List)):
903            func_name = 'select_template'
904
905        self.writeline('template = environment.%s(' % func_name, node)
906        self.visit(node.template, frame)
907        self.write(', %r)' % self.name)
908        if node.ignore_missing:
909            self.outdent()
910            self.writeline('except TemplateNotFound:')
911            self.indent()
912            self.writeline('pass')
913            self.outdent()
914            self.writeline('else:')
915            self.indent()
916
917        skip_event_yield = False
918        if node.with_context:
919            loop = self.environment.is_async and 'async for' or 'for'
920            self.writeline('%s event in template.root_render_func('
921                           'template.new_context(context.get_all(), True, '
922                           '%s)):' % (loop, self.dump_local_context(frame)))
923        elif self.environment.is_async:
924            self.writeline('for event in (await '
925                           'template._get_default_module_async())'
926                           '._body_stream:')
927        else:
928            if supports_yield_from:
929                self.writeline('yield from template._get_default_module()'
930                               '._body_stream')
931                skip_event_yield = True
932            else:
933                self.writeline('for event in template._get_default_module()'
934                               '._body_stream:')
935
936        if not skip_event_yield:
937            self.indent()
938            self.simple_write('event', frame)
939            self.outdent()
940
941        if node.ignore_missing:
942            self.outdent()
943
944    def visit_Import(self, node, frame):
945        """Visit regular imports."""
946        self.writeline('%s = ' % frame.symbols.ref(node.target), node)
947        if frame.toplevel:
948            self.write('context.vars[%r] = ' % node.target)
949        if self.environment.is_async:
950            self.write('await ')
951        self.write('environment.get_template(')
952        self.visit(node.template, frame)
953        self.write(', %r).' % self.name)
954        if node.with_context:
955            self.write('make_module%s(context.get_all(), True, %s)'
956                       % (self.environment.is_async and '_async' or '',
957                          self.dump_local_context(frame)))
958        elif self.environment.is_async:
959            self.write('_get_default_module_async()')
960        else:
961            self.write('_get_default_module()')
962        if frame.toplevel and not node.target.startswith('_'):
963            self.writeline('context.exported_vars.discard(%r)' % node.target)
964
965    def visit_FromImport(self, node, frame):
966        """Visit named imports."""
967        self.newline(node)
968        self.write('included_template = %senvironment.get_template('
969                   % (self.environment.is_async and 'await ' or ''))
970        self.visit(node.template, frame)
971        self.write(', %r).' % self.name)
972        if node.with_context:
973            self.write('make_module%s(context.get_all(), True, %s)'
974                       % (self.environment.is_async and '_async' or '',
975                          self.dump_local_context(frame)))
976        elif self.environment.is_async:
977            self.write('_get_default_module_async()')
978        else:
979            self.write('_get_default_module()')
980
981        var_names = []
982        discarded_names = []
983        for name in node.names:
984            if isinstance(name, tuple):
985                name, alias = name
986            else:
987                alias = name
988            self.writeline('%s = getattr(included_template, '
989                           '%r, missing)' % (frame.symbols.ref(alias), name))
990            self.writeline('if %s is missing:' % frame.symbols.ref(alias))
991            self.indent()
992            self.writeline('%s = undefined(%r %% '
993                           'included_template.__name__, '
994                           'name=%r)' %
995                           (frame.symbols.ref(alias),
996                            'the template %%r (imported on %s) does '
997                            'not export the requested name %s' % (
998                                self.position(node),
999                                repr(name)
1000                           ), name))
1001            self.outdent()
1002            if frame.toplevel:
1003                var_names.append(alias)
1004                if not alias.startswith('_'):
1005                    discarded_names.append(alias)
1006
1007        if var_names:
1008            if len(var_names) == 1:
1009                name = var_names[0]
1010                self.writeline('context.vars[%r] = %s' %
1011                               (name, frame.symbols.ref(name)))
1012            else:
1013                self.writeline('context.vars.update({%s})' % ', '.join(
1014                    '%r: %s' % (name, frame.symbols.ref(name)) for name in var_names
1015                ))
1016        if discarded_names:
1017            if len(discarded_names) == 1:
1018                self.writeline('context.exported_vars.discard(%r)' %
1019                               discarded_names[0])
1020            else:
1021                self.writeline('context.exported_vars.difference_'
1022                               'update((%s))' % ', '.join(imap(repr, discarded_names)))
1023
1024    def visit_For(self, node, frame):
1025        loop_frame = frame.inner()
1026        test_frame = frame.inner()
1027        else_frame = frame.inner()
1028
1029        # try to figure out if we have an extended loop.  An extended loop
1030        # is necessary if the loop is in recursive mode if the special loop
1031        # variable is accessed in the body.
1032        extended_loop = node.recursive or 'loop' in \
1033                        find_undeclared(node.iter_child_nodes(
1034                            only=('body',)), ('loop',))
1035
1036        loop_ref = None
1037        if extended_loop:
1038            loop_ref = loop_frame.symbols.declare_parameter('loop')
1039
1040        loop_frame.symbols.analyze_node(node, for_branch='body')
1041        if node.else_:
1042            else_frame.symbols.analyze_node(node, for_branch='else')
1043
1044        if node.test:
1045            loop_filter_func = self.temporary_identifier()
1046            test_frame.symbols.analyze_node(node, for_branch='test')
1047            self.writeline('%s(fiter):' % self.func(loop_filter_func), node.test)
1048            self.indent()
1049            self.enter_frame(test_frame)
1050            self.writeline(self.environment.is_async and 'async for ' or 'for ')
1051            self.visit(node.target, loop_frame)
1052            self.write(' in ')
1053            self.write(self.environment.is_async and 'auto_aiter(fiter)' or 'fiter')
1054            self.write(':')
1055            self.indent()
1056            self.writeline('if ', node.test)
1057            self.visit(node.test, test_frame)
1058            self.write(':')
1059            self.indent()
1060            self.writeline('yield ')
1061            self.visit(node.target, loop_frame)
1062            self.outdent(3)
1063            self.leave_frame(test_frame, with_python_scope=True)
1064
1065        # if we don't have an recursive loop we have to find the shadowed
1066        # variables at that point.  Because loops can be nested but the loop
1067        # variable is a special one we have to enforce aliasing for it.
1068        if node.recursive:
1069            self.writeline('%s(reciter, loop_render_func, depth=0):' %
1070                           self.func('loop'), node)
1071            self.indent()
1072            self.buffer(loop_frame)
1073
1074            # Use the same buffer for the else frame
1075            else_frame.buffer = loop_frame.buffer
1076
1077        # make sure the loop variable is a special one and raise a template
1078        # assertion error if a loop tries to write to loop
1079        if extended_loop:
1080            self.writeline('%s = missing' % loop_ref)
1081
1082        for name in node.find_all(nodes.Name):
1083            if name.ctx == 'store' and name.name == 'loop':
1084                self.fail('Can\'t assign to special loop variable '
1085                          'in for-loop target', name.lineno)
1086
1087        if node.else_:
1088            iteration_indicator = self.temporary_identifier()
1089            self.writeline('%s = 1' % iteration_indicator)
1090
1091        self.writeline(self.environment.is_async and 'async for ' or 'for ', node)
1092        self.visit(node.target, loop_frame)
1093        if extended_loop:
1094            if self.environment.is_async:
1095                self.write(', %s in await make_async_loop_context(' % loop_ref)
1096            else:
1097                self.write(', %s in LoopContext(' % loop_ref)
1098        else:
1099            self.write(' in ')
1100
1101        if node.test:
1102            self.write('%s(' % loop_filter_func)
1103        if node.recursive:
1104            self.write('reciter')
1105        else:
1106            if self.environment.is_async and not extended_loop:
1107                self.write('auto_aiter(')
1108            self.visit(node.iter, frame)
1109            if self.environment.is_async and not extended_loop:
1110                self.write(')')
1111        if node.test:
1112            self.write(')')
1113
1114        if node.recursive:
1115            self.write(', undefined, loop_render_func, depth):')
1116        else:
1117            self.write(extended_loop and ', undefined):' or ':')
1118
1119        self.indent()
1120        self.enter_frame(loop_frame)
1121
1122        self.blockvisit(node.body, loop_frame)
1123        if node.else_:
1124            self.writeline('%s = 0' % iteration_indicator)
1125        self.outdent()
1126        self.leave_frame(loop_frame, with_python_scope=node.recursive
1127                         and not node.else_)
1128
1129        if node.else_:
1130            self.writeline('if %s:' % iteration_indicator)
1131            self.indent()
1132            self.enter_frame(else_frame)
1133            self.blockvisit(node.else_, else_frame)
1134            self.leave_frame(else_frame)
1135            self.outdent()
1136
1137        # if the node was recursive we have to return the buffer contents
1138        # and start the iteration code
1139        if node.recursive:
1140            self.return_buffer_contents(loop_frame)
1141            self.outdent()
1142            self.start_write(frame, node)
1143            if self.environment.is_async:
1144                self.write('await ')
1145            self.write('loop(')
1146            if self.environment.is_async:
1147                self.write('auto_aiter(')
1148            self.visit(node.iter, frame)
1149            if self.environment.is_async:
1150                self.write(')')
1151            self.write(', loop)')
1152            self.end_write(frame)
1153
1154    def visit_If(self, node, frame):
1155        if_frame = frame.soft()
1156        self.writeline('if ', node)
1157        self.visit(node.test, if_frame)
1158        self.write(':')
1159        self.indent()
1160        self.blockvisit(node.body, if_frame)
1161        self.outdent()
1162        for elif_ in node.elif_:
1163            self.writeline('elif ', elif_)
1164            self.visit(elif_.test, if_frame)
1165            self.write(':')
1166            self.indent()
1167            self.blockvisit(elif_.body, if_frame)
1168            self.outdent()
1169        if node.else_:
1170            self.writeline('else:')
1171            self.indent()
1172            self.blockvisit(node.else_, if_frame)
1173            self.outdent()
1174
1175    def visit_Macro(self, node, frame):
1176        macro_frame, macro_ref = self.macro_body(node, frame)
1177        self.newline()
1178        if frame.toplevel:
1179            if not node.name.startswith('_'):
1180                self.write('context.exported_vars.add(%r)' % node.name)
1181            ref = frame.symbols.ref(node.name)
1182            self.writeline('context.vars[%r] = ' % node.name)
1183        self.write('%s = ' % frame.symbols.ref(node.name))
1184        self.macro_def(macro_ref, macro_frame)
1185
1186    def visit_CallBlock(self, node, frame):
1187        call_frame, macro_ref = self.macro_body(node, frame)
1188        self.writeline('caller = ')
1189        self.macro_def(macro_ref, call_frame)
1190        self.start_write(frame, node)
1191        self.visit_Call(node.call, frame, forward_caller=True)
1192        self.end_write(frame)
1193
1194    def visit_FilterBlock(self, node, frame):
1195        filter_frame = frame.inner()
1196        filter_frame.symbols.analyze_node(node)
1197        self.enter_frame(filter_frame)
1198        self.buffer(filter_frame)
1199        self.blockvisit(node.body, filter_frame)
1200        self.start_write(frame, node)
1201        self.visit_Filter(node.filter, filter_frame)
1202        self.end_write(frame)
1203        self.leave_frame(filter_frame)
1204
1205    def visit_With(self, node, frame):
1206        with_frame = frame.inner()
1207        with_frame.symbols.analyze_node(node)
1208        self.enter_frame(with_frame)
1209        for idx, (target, expr) in enumerate(izip(node.targets, node.values)):
1210            self.newline()
1211            self.visit(target, with_frame)
1212            self.write(' = ')
1213            self.visit(expr, frame)
1214        self.blockvisit(node.body, with_frame)
1215        self.leave_frame(with_frame)
1216
1217    def visit_ExprStmt(self, node, frame):
1218        self.newline(node)
1219        self.visit(node.node, frame)
1220
1221    def visit_Output(self, node, frame):
1222        # if we have a known extends statement, we don't output anything
1223        # if we are in a require_output_check section
1224        if self.has_known_extends and frame.require_output_check:
1225            return
1226
1227        allow_constant_finalize = True
1228        if self.environment.finalize:
1229            func = self.environment.finalize
1230            if getattr(func, 'contextfunction', False) or \
1231               getattr(func, 'evalcontextfunction', False):
1232                allow_constant_finalize = False
1233            elif getattr(func, 'environmentfunction', False):
1234                finalize = lambda x: text_type(
1235                    self.environment.finalize(self.environment, x))
1236            else:
1237                finalize = lambda x: text_type(self.environment.finalize(x))
1238        else:
1239            finalize = text_type
1240
1241        # if we are inside a frame that requires output checking, we do so
1242        outdent_later = False
1243        if frame.require_output_check:
1244            self.writeline('if parent_template is None:')
1245            self.indent()
1246            outdent_later = True
1247
1248        # try to evaluate as many chunks as possible into a static
1249        # string at compile time.
1250        body = []
1251        for child in node.nodes:
1252            try:
1253                if not allow_constant_finalize:
1254                    raise nodes.Impossible()
1255                const = child.as_const(frame.eval_ctx)
1256            except nodes.Impossible:
1257                body.append(child)
1258                continue
1259            # the frame can't be volatile here, becaus otherwise the
1260            # as_const() function would raise an Impossible exception
1261            # at that point.
1262            try:
1263                if frame.eval_ctx.autoescape:
1264                    if hasattr(const, '__html__'):
1265                        const = const.__html__()
1266                    else:
1267                        const = escape(const)
1268                const = finalize(const)
1269            except Exception:
1270                # if something goes wrong here we evaluate the node
1271                # at runtime for easier debugging
1272                body.append(child)
1273                continue
1274            if body and isinstance(body[-1], list):
1275                body[-1].append(const)
1276            else:
1277                body.append([const])
1278
1279        # if we have less than 3 nodes or a buffer we yield or extend/append
1280        if len(body) < 3 or frame.buffer is not None:
1281            if frame.buffer is not None:
1282                # for one item we append, for more we extend
1283                if len(body) == 1:
1284                    self.writeline('%s.append(' % frame.buffer)
1285                else:
1286                    self.writeline('%s.extend((' % frame.buffer)
1287                self.indent()
1288            for item in body:
1289                if isinstance(item, list):
1290                    val = repr(concat(item))
1291                    if frame.buffer is None:
1292                        self.writeline('yield ' + val)
1293                    else:
1294                        self.writeline(val + ',')
1295                else:
1296                    if frame.buffer is None:
1297                        self.writeline('yield ', item)
1298                    else:
1299                        self.newline(item)
1300                    close = 1
1301                    if frame.eval_ctx.volatile:
1302                        self.write('(escape if context.eval_ctx.autoescape'
1303                                   ' else to_string)(')
1304                    elif frame.eval_ctx.autoescape:
1305                        self.write('escape(')
1306                    else:
1307                        self.write('to_string(')
1308                    if self.environment.finalize is not None:
1309                        self.write('environment.finalize(')
1310                        if getattr(self.environment.finalize,
1311                                   "contextfunction", False):
1312                            self.write('context, ')
1313                        close += 1
1314                    self.visit(item, frame)
1315                    self.write(')' * close)
1316                    if frame.buffer is not None:
1317                        self.write(',')
1318            if frame.buffer is not None:
1319                # close the open parentheses
1320                self.outdent()
1321                self.writeline(len(body) == 1 and ')' or '))')
1322
1323        # otherwise we create a format string as this is faster in that case
1324        else:
1325            format = []
1326            arguments = []
1327            for item in body:
1328                if isinstance(item, list):
1329                    format.append(concat(item).replace('%', '%%'))
1330                else:
1331                    format.append('%s')
1332                    arguments.append(item)
1333            self.writeline('yield ')
1334            self.write(repr(concat(format)) + ' % (')
1335            self.indent()
1336            for argument in arguments:
1337                self.newline(argument)
1338                close = 0
1339                if frame.eval_ctx.volatile:
1340                    self.write('(escape if context.eval_ctx.autoescape else'
1341                               ' to_string)(')
1342                    close += 1
1343                elif frame.eval_ctx.autoescape:
1344                    self.write('escape(')
1345                    close += 1
1346                if self.environment.finalize is not None:
1347                    self.write('environment.finalize(')
1348                    if getattr(self.environment.finalize,
1349                               'contextfunction', False):
1350                        self.write('context, ')
1351                    elif getattr(self.environment.finalize,
1352                               'evalcontextfunction', False):
1353                        self.write('context.eval_ctx, ')
1354                    elif getattr(self.environment.finalize,
1355                               'environmentfunction', False):
1356                        self.write('environment, ')
1357                    close += 1
1358                self.visit(argument, frame)
1359                self.write(')' * close + ', ')
1360            self.outdent()
1361            self.writeline(')')
1362
1363        if outdent_later:
1364            self.outdent()
1365
1366    def visit_Assign(self, node, frame):
1367        self.push_assign_tracking()
1368        self.newline(node)
1369        self.visit(node.target, frame)
1370        self.write(' = ')
1371        self.visit(node.node, frame)
1372        self.pop_assign_tracking(frame)
1373
1374    def visit_AssignBlock(self, node, frame):
1375        self.push_assign_tracking()
1376        block_frame = frame.inner()
1377        # This is a special case.  Since a set block always captures we
1378        # will disable output checks.  This way one can use set blocks
1379        # toplevel even in extended templates.
1380        block_frame.require_output_check = False
1381        block_frame.symbols.analyze_node(node)
1382        self.enter_frame(block_frame)
1383        self.buffer(block_frame)
1384        self.blockvisit(node.body, block_frame)
1385        self.newline(node)
1386        self.visit(node.target, frame)
1387        self.write(' = (Markup if context.eval_ctx.autoescape '
1388                   'else identity)(')
1389        if node.filter is not None:
1390            self.visit_Filter(node.filter, block_frame)
1391        else:
1392            self.write('concat(%s)' % block_frame.buffer)
1393        self.write(')')
1394        self.pop_assign_tracking(frame)
1395        self.leave_frame(block_frame)
1396
1397    # -- Expression Visitors
1398
1399    def visit_Name(self, node, frame):
1400        if node.ctx == 'store' and frame.toplevel:
1401            if self._assign_stack:
1402                self._assign_stack[-1].add(node.name)
1403        ref = frame.symbols.ref(node.name)
1404
1405        # If we are looking up a variable we might have to deal with the
1406        # case where it's undefined.  We can skip that case if the load
1407        # instruction indicates a parameter which are always defined.
1408        if node.ctx == 'load':
1409            load = frame.symbols.find_load(ref)
1410            if not (load is not None and load[0] == VAR_LOAD_PARAMETER and \
1411                    not self.parameter_is_undeclared(ref)):
1412                self.write('(undefined(name=%r) if %s is missing else %s)' %
1413                           (node.name, ref, ref))
1414                return
1415
1416        self.write(ref)
1417
1418    def visit_NSRef(self, node, frame):
1419        # NSRefs can only be used to store values; since they use the normal
1420        # `foo.bar` notation they will be parsed as a normal attribute access
1421        # when used anywhere but in a `set` context
1422        ref = frame.symbols.ref(node.name)
1423        self.writeline('if not isinstance(%s, Namespace):' % ref)
1424        self.indent()
1425        self.writeline('raise TemplateRuntimeError(%r)' %
1426                       'cannot assign attribute on non-namespace object')
1427        self.outdent()
1428        self.writeline('%s[%r]' % (ref, node.attr))
1429
1430    def visit_Const(self, node, frame):
1431        val = node.as_const(frame.eval_ctx)
1432        if isinstance(val, float):
1433            self.write(str(val))
1434        else:
1435            self.write(repr(val))
1436
1437    def visit_TemplateData(self, node, frame):
1438        try:
1439            self.write(repr(node.as_const(frame.eval_ctx)))
1440        except nodes.Impossible:
1441            self.write('(Markup if context.eval_ctx.autoescape else identity)(%r)'
1442                       % node.data)
1443
1444    def visit_Tuple(self, node, frame):
1445        self.write('(')
1446        idx = -1
1447        for idx, item in enumerate(node.items):
1448            if idx:
1449                self.write(', ')
1450            self.visit(item, frame)
1451        self.write(idx == 0 and ',)' or ')')
1452
1453    def visit_List(self, node, frame):
1454        self.write('[')
1455        for idx, item in enumerate(node.items):
1456            if idx:
1457                self.write(', ')
1458            self.visit(item, frame)
1459        self.write(']')
1460
1461    def visit_Dict(self, node, frame):
1462        self.write('{')
1463        for idx, item in enumerate(node.items):
1464            if idx:
1465                self.write(', ')
1466            self.visit(item.key, frame)
1467            self.write(': ')
1468            self.visit(item.value, frame)
1469        self.write('}')
1470
1471    def binop(operator, interceptable=True):
1472        @optimizeconst
1473        def visitor(self, node, frame):
1474            if self.environment.sandboxed and \
1475               operator in self.environment.intercepted_binops:
1476                self.write('environment.call_binop(context, %r, ' % operator)
1477                self.visit(node.left, frame)
1478                self.write(', ')
1479                self.visit(node.right, frame)
1480            else:
1481                self.write('(')
1482                self.visit(node.left, frame)
1483                self.write(' %s ' % operator)
1484                self.visit(node.right, frame)
1485            self.write(')')
1486        return visitor
1487
1488    def uaop(operator, interceptable=True):
1489        @optimizeconst
1490        def visitor(self, node, frame):
1491            if self.environment.sandboxed and \
1492               operator in self.environment.intercepted_unops:
1493                self.write('environment.call_unop(context, %r, ' % operator)
1494                self.visit(node.node, frame)
1495            else:
1496                self.write('(' + operator)
1497                self.visit(node.node, frame)
1498            self.write(')')
1499        return visitor
1500
1501    visit_Add = binop('+')
1502    visit_Sub = binop('-')
1503    visit_Mul = binop('*')
1504    visit_Div = binop('/')
1505    visit_FloorDiv = binop('//')
1506    visit_Pow = binop('**')
1507    visit_Mod = binop('%')
1508    visit_And = binop('and', interceptable=False)
1509    visit_Or = binop('or', interceptable=False)
1510    visit_Pos = uaop('+')
1511    visit_Neg = uaop('-')
1512    visit_Not = uaop('not ', interceptable=False)
1513    del binop, uaop
1514
1515    @optimizeconst
1516    def visit_Concat(self, node, frame):
1517        if frame.eval_ctx.volatile:
1518            func_name = '(context.eval_ctx.volatile and' \
1519                        ' markup_join or unicode_join)'
1520        elif frame.eval_ctx.autoescape:
1521            func_name = 'markup_join'
1522        else:
1523            func_name = 'unicode_join'
1524        self.write('%s((' % func_name)
1525        for arg in node.nodes:
1526            self.visit(arg, frame)
1527            self.write(', ')
1528        self.write('))')
1529
1530    @optimizeconst
1531    def visit_Compare(self, node, frame):
1532        self.visit(node.expr, frame)
1533        for op in node.ops:
1534            self.visit(op, frame)
1535
1536    def visit_Operand(self, node, frame):
1537        self.write(' %s ' % operators[node.op])
1538        self.visit(node.expr, frame)
1539
1540    @optimizeconst
1541    def visit_Getattr(self, node, frame):
1542        self.write('environment.getattr(')
1543        self.visit(node.node, frame)
1544        self.write(', %r)' % node.attr)
1545
1546    @optimizeconst
1547    def visit_Getitem(self, node, frame):
1548        # slices bypass the environment getitem method.
1549        if isinstance(node.arg, nodes.Slice):
1550            self.visit(node.node, frame)
1551            self.write('[')
1552            self.visit(node.arg, frame)
1553            self.write(']')
1554        else:
1555            self.write('environment.getitem(')
1556            self.visit(node.node, frame)
1557            self.write(', ')
1558            self.visit(node.arg, frame)
1559            self.write(')')
1560
1561    def visit_Slice(self, node, frame):
1562        if node.start is not None:
1563            self.visit(node.start, frame)
1564        self.write(':')
1565        if node.stop is not None:
1566            self.visit(node.stop, frame)
1567        if node.step is not None:
1568            self.write(':')
1569            self.visit(node.step, frame)
1570
1571    @optimizeconst
1572    def visit_Filter(self, node, frame):
1573        if self.environment.is_async:
1574            self.write('await auto_await(')
1575        self.write(self.filters[node.name] + '(')
1576        func = self.environment.filters.get(node.name)
1577        if func is None:
1578            self.fail('no filter named %r' % node.name, node.lineno)
1579        if getattr(func, 'contextfilter', False):
1580            self.write('context, ')
1581        elif getattr(func, 'evalcontextfilter', False):
1582            self.write('context.eval_ctx, ')
1583        elif getattr(func, 'environmentfilter', False):
1584            self.write('environment, ')
1585
1586        # if the filter node is None we are inside a filter block
1587        # and want to write to the current buffer
1588        if node.node is not None:
1589            self.visit(node.node, frame)
1590        elif frame.eval_ctx.volatile:
1591            self.write('(context.eval_ctx.autoescape and'
1592                       ' Markup(concat(%s)) or concat(%s))' %
1593                       (frame.buffer, frame.buffer))
1594        elif frame.eval_ctx.autoescape:
1595            self.write('Markup(concat(%s))' % frame.buffer)
1596        else:
1597            self.write('concat(%s)' % frame.buffer)
1598        self.signature(node, frame)
1599        self.write(')')
1600        if self.environment.is_async:
1601            self.write(')')
1602
1603    @optimizeconst
1604    def visit_Test(self, node, frame):
1605        self.write(self.tests[node.name] + '(')
1606        if node.name not in self.environment.tests:
1607            self.fail('no test named %r' % node.name, node.lineno)
1608        self.visit(node.node, frame)
1609        self.signature(node, frame)
1610        self.write(')')
1611
1612    @optimizeconst
1613    def visit_CondExpr(self, node, frame):
1614        def write_expr2():
1615            if node.expr2 is not None:
1616                return self.visit(node.expr2, frame)
1617            self.write('undefined(%r)' % ('the inline if-'
1618                       'expression on %s evaluated to false and '
1619                       'no else section was defined.' % self.position(node)))
1620
1621        self.write('(')
1622        self.visit(node.expr1, frame)
1623        self.write(' if ')
1624        self.visit(node.test, frame)
1625        self.write(' else ')
1626        write_expr2()
1627        self.write(')')
1628
1629    @optimizeconst
1630    def visit_Call(self, node, frame, forward_caller=False):
1631        if self.environment.is_async:
1632            self.write('await auto_await(')
1633        if self.environment.sandboxed:
1634            self.write('environment.call(context, ')
1635        else:
1636            self.write('context.call(')
1637        self.visit(node.node, frame)
1638        extra_kwargs = forward_caller and {'caller': 'caller'} or None
1639        self.signature(node, frame, extra_kwargs)
1640        self.write(')')
1641        if self.environment.is_async:
1642            self.write(')')
1643
1644    def visit_Keyword(self, node, frame):
1645        self.write(node.key + '=')
1646        self.visit(node.value, frame)
1647
1648    # -- Unused nodes for extensions
1649
1650    def visit_MarkSafe(self, node, frame):
1651        self.write('Markup(')
1652        self.visit(node.expr, frame)
1653        self.write(')')
1654
1655    def visit_MarkSafeIfAutoescape(self, node, frame):
1656        self.write('(context.eval_ctx.autoescape and Markup or identity)(')
1657        self.visit(node.expr, frame)
1658        self.write(')')
1659
1660    def visit_EnvironmentAttribute(self, node, frame):
1661        self.write('environment.' + node.name)
1662
1663    def visit_ExtensionAttribute(self, node, frame):
1664        self.write('environment.extensions[%r].%s' % (node.identifier, node.name))
1665
1666    def visit_ImportedName(self, node, frame):
1667        self.write(self.import_aliases[node.importname])
1668
1669    def visit_InternalName(self, node, frame):
1670        self.write(node.name)
1671
1672    def visit_ContextReference(self, node, frame):
1673        self.write('context')
1674
1675    def visit_Continue(self, node, frame):
1676        self.writeline('continue', node)
1677
1678    def visit_Break(self, node, frame):
1679        self.writeline('break', node)
1680
1681    def visit_Scope(self, node, frame):
1682        scope_frame = frame.inner()
1683        scope_frame.symbols.analyze_node(node)
1684        self.enter_frame(scope_frame)
1685        self.blockvisit(node.body, scope_frame)
1686        self.leave_frame(scope_frame)
1687
1688    def visit_OverlayScope(self, node, frame):
1689        ctx = self.temporary_identifier()
1690        self.writeline('%s = %s' % (ctx, self.derive_context(frame)))
1691        self.writeline('%s.vars = ' % ctx)
1692        self.visit(node.context, frame)
1693        self.push_context_reference(ctx)
1694
1695        scope_frame = frame.inner(isolated=True)
1696        scope_frame.symbols.analyze_node(node)
1697        self.enter_frame(scope_frame)
1698        self.blockvisit(node.body, scope_frame)
1699        self.leave_frame(scope_frame)
1700        self.pop_context_reference()
1701
1702    def visit_EvalContextModifier(self, node, frame):
1703        for keyword in node.options:
1704            self.writeline('context.eval_ctx.%s = ' % keyword.key)
1705            self.visit(keyword.value, frame)
1706            try:
1707                val = keyword.value.as_const(frame.eval_ctx)
1708            except nodes.Impossible:
1709                frame.eval_ctx.volatile = True
1710            else:
1711                setattr(frame.eval_ctx, keyword.key, val)
1712
1713    def visit_ScopedEvalContextModifier(self, node, frame):
1714        old_ctx_name = self.temporary_identifier()
1715        saved_ctx = frame.eval_ctx.save()
1716        self.writeline('%s = context.eval_ctx.save()' % old_ctx_name)
1717        self.visit_EvalContextModifier(node, frame)
1718        for child in node.body:
1719            self.visit(child, frame)
1720        frame.eval_ctx.revert(saved_ctx)
1721        self.writeline('context.eval_ctx.revert(%s)' % old_ctx_name)
1722