• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2"""
3    jinja2.environment
4    ~~~~~~~~~~~~~~~~~~
5
6    Provides a class that holds runtime and parsing time options.
7
8    :copyright: (c) 2017 by the Jinja Team.
9    :license: BSD, see LICENSE for more details.
10"""
11import os
12import sys
13import weakref
14from functools import reduce, partial
15from jinja2 import nodes
16from jinja2.defaults import BLOCK_START_STRING, \
17     BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
18     COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \
19     LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \
20     DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \
21     DEFAULT_POLICIES, KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS
22from jinja2.lexer import get_lexer, TokenStream
23from jinja2.parser import Parser
24from jinja2.nodes import EvalContext
25from jinja2.compiler import generate, CodeGenerator
26from jinja2.runtime import Undefined, new_context, Context
27from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \
28     TemplatesNotFound, TemplateRuntimeError
29from jinja2.utils import import_string, LRUCache, Markup, missing, \
30     concat, consume, internalcode, have_async_gen
31from jinja2._compat import imap, ifilter, string_types, iteritems, \
32     text_type, reraise, implements_iterator, implements_to_string, \
33     encode_filename, PY2, PYPY
34
35
36# for direct template usage we have up to ten living environments
37_spontaneous_environments = LRUCache(10)
38
39# the function to create jinja traceback objects.  This is dynamically
40# imported on the first exception in the exception handler.
41_make_traceback = None
42
43
44def get_spontaneous_environment(*args):
45    """Return a new spontaneous environment.  A spontaneous environment is an
46    unnamed and unaccessible (in theory) environment that is used for
47    templates generated from a string and not from the file system.
48    """
49    try:
50        env = _spontaneous_environments.get(args)
51    except TypeError:
52        return Environment(*args)
53    if env is not None:
54        return env
55    _spontaneous_environments[args] = env = Environment(*args)
56    env.shared = True
57    return env
58
59
60def create_cache(size):
61    """Return the cache class for the given size."""
62    if size == 0:
63        return None
64    if size < 0:
65        return {}
66    return LRUCache(size)
67
68
69def copy_cache(cache):
70    """Create an empty copy of the given cache."""
71    if cache is None:
72        return None
73    elif type(cache) is dict:
74        return {}
75    return LRUCache(cache.capacity)
76
77
78def load_extensions(environment, extensions):
79    """Load the extensions from the list and bind it to the environment.
80    Returns a dict of instantiated environments.
81    """
82    result = {}
83    for extension in extensions:
84        if isinstance(extension, string_types):
85            extension = import_string(extension)
86        result[extension.identifier] = extension(environment)
87    return result
88
89
90def fail_for_missing_callable(string, name):
91    msg = string % name
92    if isinstance(name, Undefined):
93        try:
94            name._fail_with_undefined_error()
95        except Exception as e:
96            msg = '%s (%s; did you forget to quote the callable name?)' % (msg, e)
97    raise TemplateRuntimeError(msg)
98
99
100def _environment_sanity_check(environment):
101    """Perform a sanity check on the environment."""
102    assert issubclass(environment.undefined, Undefined), 'undefined must ' \
103        'be a subclass of undefined because filters depend on it.'
104    assert environment.block_start_string != \
105        environment.variable_start_string != \
106        environment.comment_start_string, 'block, variable and comment ' \
107        'start strings must be different'
108    assert environment.newline_sequence in ('\r', '\r\n', '\n'), \
109        'newline_sequence set to unknown line ending string.'
110    return environment
111
112
113class Environment(object):
114    r"""The core component of Jinja is the `Environment`.  It contains
115    important shared variables like configuration, filters, tests,
116    globals and others.  Instances of this class may be modified if
117    they are not shared and if no template was loaded so far.
118    Modifications on environments after the first template was loaded
119    will lead to surprising effects and undefined behavior.
120
121    Here are the possible initialization parameters:
122
123        `block_start_string`
124            The string marking the beginning of a block.  Defaults to ``'{%'``.
125
126        `block_end_string`
127            The string marking the end of a block.  Defaults to ``'%}'``.
128
129        `variable_start_string`
130            The string marking the beginning of a print statement.
131            Defaults to ``'{{'``.
132
133        `variable_end_string`
134            The string marking the end of a print statement.  Defaults to
135            ``'}}'``.
136
137        `comment_start_string`
138            The string marking the beginning of a comment.  Defaults to ``'{#'``.
139
140        `comment_end_string`
141            The string marking the end of a comment.  Defaults to ``'#}'``.
142
143        `line_statement_prefix`
144            If given and a string, this will be used as prefix for line based
145            statements.  See also :ref:`line-statements`.
146
147        `line_comment_prefix`
148            If given and a string, this will be used as prefix for line based
149            comments.  See also :ref:`line-statements`.
150
151            .. versionadded:: 2.2
152
153        `trim_blocks`
154            If this is set to ``True`` the first newline after a block is
155            removed (block, not variable tag!).  Defaults to `False`.
156
157        `lstrip_blocks`
158            If this is set to ``True`` leading spaces and tabs are stripped
159            from the start of a line to a block.  Defaults to `False`.
160
161        `newline_sequence`
162            The sequence that starts a newline.  Must be one of ``'\r'``,
163            ``'\n'`` or ``'\r\n'``.  The default is ``'\n'`` which is a
164            useful default for Linux and OS X systems as well as web
165            applications.
166
167        `keep_trailing_newline`
168            Preserve the trailing newline when rendering templates.
169            The default is ``False``, which causes a single newline,
170            if present, to be stripped from the end of the template.
171
172            .. versionadded:: 2.7
173
174        `extensions`
175            List of Jinja extensions to use.  This can either be import paths
176            as strings or extension classes.  For more information have a
177            look at :ref:`the extensions documentation <jinja-extensions>`.
178
179        `optimized`
180            should the optimizer be enabled?  Default is ``True``.
181
182        `undefined`
183            :class:`Undefined` or a subclass of it that is used to represent
184            undefined values in the template.
185
186        `finalize`
187            A callable that can be used to process the result of a variable
188            expression before it is output.  For example one can convert
189            ``None`` implicitly into an empty string here.
190
191        `autoescape`
192            If set to ``True`` the XML/HTML autoescaping feature is enabled by
193            default.  For more details about autoescaping see
194            :class:`~jinja2.utils.Markup`.  As of Jinja 2.4 this can also
195            be a callable that is passed the template name and has to
196            return ``True`` or ``False`` depending on autoescape should be
197            enabled by default.
198
199            .. versionchanged:: 2.4
200               `autoescape` can now be a function
201
202        `loader`
203            The template loader for this environment.
204
205        `cache_size`
206            The size of the cache.  Per default this is ``400`` which means
207            that if more than 400 templates are loaded the loader will clean
208            out the least recently used template.  If the cache size is set to
209            ``0`` templates are recompiled all the time, if the cache size is
210            ``-1`` the cache will not be cleaned.
211
212            .. versionchanged:: 2.8
213               The cache size was increased to 400 from a low 50.
214
215        `auto_reload`
216            Some loaders load templates from locations where the template
217            sources may change (ie: file system or database).  If
218            ``auto_reload`` is set to ``True`` (default) every time a template is
219            requested the loader checks if the source changed and if yes, it
220            will reload the template.  For higher performance it's possible to
221            disable that.
222
223        `bytecode_cache`
224            If set to a bytecode cache object, this object will provide a
225            cache for the internal Jinja bytecode so that templates don't
226            have to be parsed if they were not changed.
227
228            See :ref:`bytecode-cache` for more information.
229
230        `enable_async`
231            If set to true this enables async template execution which allows
232            you to take advantage of newer Python features.  This requires
233            Python 3.6 or later.
234    """
235
236    #: if this environment is sandboxed.  Modifying this variable won't make
237    #: the environment sandboxed though.  For a real sandboxed environment
238    #: have a look at jinja2.sandbox.  This flag alone controls the code
239    #: generation by the compiler.
240    sandboxed = False
241
242    #: True if the environment is just an overlay
243    overlayed = False
244
245    #: the environment this environment is linked to if it is an overlay
246    linked_to = None
247
248    #: shared environments have this set to `True`.  A shared environment
249    #: must not be modified
250    shared = False
251
252    #: these are currently EXPERIMENTAL undocumented features.
253    exception_handler = None
254    exception_formatter = None
255
256    #: the class that is used for code generation.  See
257    #: :class:`~jinja2.compiler.CodeGenerator` for more information.
258    code_generator_class = CodeGenerator
259
260    #: the context class thatis used for templates.  See
261    #: :class:`~jinja2.runtime.Context` for more information.
262    context_class = Context
263
264    def __init__(self,
265                 block_start_string=BLOCK_START_STRING,
266                 block_end_string=BLOCK_END_STRING,
267                 variable_start_string=VARIABLE_START_STRING,
268                 variable_end_string=VARIABLE_END_STRING,
269                 comment_start_string=COMMENT_START_STRING,
270                 comment_end_string=COMMENT_END_STRING,
271                 line_statement_prefix=LINE_STATEMENT_PREFIX,
272                 line_comment_prefix=LINE_COMMENT_PREFIX,
273                 trim_blocks=TRIM_BLOCKS,
274                 lstrip_blocks=LSTRIP_BLOCKS,
275                 newline_sequence=NEWLINE_SEQUENCE,
276                 keep_trailing_newline=KEEP_TRAILING_NEWLINE,
277                 extensions=(),
278                 optimized=True,
279                 undefined=Undefined,
280                 finalize=None,
281                 autoescape=False,
282                 loader=None,
283                 cache_size=400,
284                 auto_reload=True,
285                 bytecode_cache=None,
286                 enable_async=False):
287        # !!Important notice!!
288        #   The constructor accepts quite a few arguments that should be
289        #   passed by keyword rather than position.  However it's important to
290        #   not change the order of arguments because it's used at least
291        #   internally in those cases:
292        #       -   spontaneous environments (i18n extension and Template)
293        #       -   unittests
294        #   If parameter changes are required only add parameters at the end
295        #   and don't change the arguments (or the defaults!) of the arguments
296        #   existing already.
297
298        # lexer / parser information
299        self.block_start_string = block_start_string
300        self.block_end_string = block_end_string
301        self.variable_start_string = variable_start_string
302        self.variable_end_string = variable_end_string
303        self.comment_start_string = comment_start_string
304        self.comment_end_string = comment_end_string
305        self.line_statement_prefix = line_statement_prefix
306        self.line_comment_prefix = line_comment_prefix
307        self.trim_blocks = trim_blocks
308        self.lstrip_blocks = lstrip_blocks
309        self.newline_sequence = newline_sequence
310        self.keep_trailing_newline = keep_trailing_newline
311
312        # runtime information
313        self.undefined = undefined
314        self.optimized = optimized
315        self.finalize = finalize
316        self.autoescape = autoescape
317
318        # defaults
319        self.filters = DEFAULT_FILTERS.copy()
320        self.tests = DEFAULT_TESTS.copy()
321        self.globals = DEFAULT_NAMESPACE.copy()
322
323        # set the loader provided
324        self.loader = loader
325        self.cache = create_cache(cache_size)
326        self.bytecode_cache = bytecode_cache
327        self.auto_reload = auto_reload
328
329        # configurable policies
330        self.policies = DEFAULT_POLICIES.copy()
331
332        # load extensions
333        self.extensions = load_extensions(self, extensions)
334
335        self.enable_async = enable_async
336        self.is_async = self.enable_async and have_async_gen
337
338        _environment_sanity_check(self)
339
340    def add_extension(self, extension):
341        """Adds an extension after the environment was created.
342
343        .. versionadded:: 2.5
344        """
345        self.extensions.update(load_extensions(self, [extension]))
346
347    def extend(self, **attributes):
348        """Add the items to the instance of the environment if they do not exist
349        yet.  This is used by :ref:`extensions <writing-extensions>` to register
350        callbacks and configuration values without breaking inheritance.
351        """
352        for key, value in iteritems(attributes):
353            if not hasattr(self, key):
354                setattr(self, key, value)
355
356    def overlay(self, block_start_string=missing, block_end_string=missing,
357                variable_start_string=missing, variable_end_string=missing,
358                comment_start_string=missing, comment_end_string=missing,
359                line_statement_prefix=missing, line_comment_prefix=missing,
360                trim_blocks=missing, lstrip_blocks=missing,
361                extensions=missing, optimized=missing,
362                undefined=missing, finalize=missing, autoescape=missing,
363                loader=missing, cache_size=missing, auto_reload=missing,
364                bytecode_cache=missing):
365        """Create a new overlay environment that shares all the data with the
366        current environment except for cache and the overridden attributes.
367        Extensions cannot be removed for an overlayed environment.  An overlayed
368        environment automatically gets all the extensions of the environment it
369        is linked to plus optional extra extensions.
370
371        Creating overlays should happen after the initial environment was set
372        up completely.  Not all attributes are truly linked, some are just
373        copied over so modifications on the original environment may not shine
374        through.
375        """
376        args = dict(locals())
377        del args['self'], args['cache_size'], args['extensions']
378
379        rv = object.__new__(self.__class__)
380        rv.__dict__.update(self.__dict__)
381        rv.overlayed = True
382        rv.linked_to = self
383
384        for key, value in iteritems(args):
385            if value is not missing:
386                setattr(rv, key, value)
387
388        if cache_size is not missing:
389            rv.cache = create_cache(cache_size)
390        else:
391            rv.cache = copy_cache(self.cache)
392
393        rv.extensions = {}
394        for key, value in iteritems(self.extensions):
395            rv.extensions[key] = value.bind(rv)
396        if extensions is not missing:
397            rv.extensions.update(load_extensions(rv, extensions))
398
399        return _environment_sanity_check(rv)
400
401    lexer = property(get_lexer, doc="The lexer for this environment.")
402
403    def iter_extensions(self):
404        """Iterates over the extensions by priority."""
405        return iter(sorted(self.extensions.values(),
406                           key=lambda x: x.priority))
407
408    def getitem(self, obj, argument):
409        """Get an item or attribute of an object but prefer the item."""
410        try:
411            return obj[argument]
412        except (AttributeError, TypeError, LookupError):
413            if isinstance(argument, string_types):
414                try:
415                    attr = str(argument)
416                except Exception:
417                    pass
418                else:
419                    try:
420                        return getattr(obj, attr)
421                    except AttributeError:
422                        pass
423            return self.undefined(obj=obj, name=argument)
424
425    def getattr(self, obj, attribute):
426        """Get an item or attribute of an object but prefer the attribute.
427        Unlike :meth:`getitem` the attribute *must* be a bytestring.
428        """
429        try:
430            return getattr(obj, attribute)
431        except AttributeError:
432            pass
433        try:
434            return obj[attribute]
435        except (TypeError, LookupError, AttributeError):
436            return self.undefined(obj=obj, name=attribute)
437
438    def call_filter(self, name, value, args=None, kwargs=None,
439                    context=None, eval_ctx=None):
440        """Invokes a filter on a value the same way the compiler does it.
441
442        Note that on Python 3 this might return a coroutine in case the
443        filter is running from an environment in async mode and the filter
444        supports async execution.  It's your responsibility to await this
445        if needed.
446
447        .. versionadded:: 2.7
448        """
449        func = self.filters.get(name)
450        if func is None:
451            fail_for_missing_callable('no filter named %r', name)
452        args = [value] + list(args or ())
453        if getattr(func, 'contextfilter', False):
454            if context is None:
455                raise TemplateRuntimeError('Attempted to invoke context '
456                                           'filter without context')
457            args.insert(0, context)
458        elif getattr(func, 'evalcontextfilter', False):
459            if eval_ctx is None:
460                if context is not None:
461                    eval_ctx = context.eval_ctx
462                else:
463                    eval_ctx = EvalContext(self)
464            args.insert(0, eval_ctx)
465        elif getattr(func, 'environmentfilter', False):
466            args.insert(0, self)
467        return func(*args, **(kwargs or {}))
468
469    def call_test(self, name, value, args=None, kwargs=None):
470        """Invokes a test on a value the same way the compiler does it.
471
472        .. versionadded:: 2.7
473        """
474        func = self.tests.get(name)
475        if func is None:
476            fail_for_missing_callable('no test named %r', name)
477        return func(value, *(args or ()), **(kwargs or {}))
478
479    @internalcode
480    def parse(self, source, name=None, filename=None):
481        """Parse the sourcecode and return the abstract syntax tree.  This
482        tree of nodes is used by the compiler to convert the template into
483        executable source- or bytecode.  This is useful for debugging or to
484        extract information from templates.
485
486        If you are :ref:`developing Jinja2 extensions <writing-extensions>`
487        this gives you a good overview of the node tree generated.
488        """
489        try:
490            return self._parse(source, name, filename)
491        except TemplateSyntaxError:
492            exc_info = sys.exc_info()
493        self.handle_exception(exc_info, source_hint=source)
494
495    def _parse(self, source, name, filename):
496        """Internal parsing function used by `parse` and `compile`."""
497        return Parser(self, source, name, encode_filename(filename)).parse()
498
499    def lex(self, source, name=None, filename=None):
500        """Lex the given sourcecode and return a generator that yields
501        tokens as tuples in the form ``(lineno, token_type, value)``.
502        This can be useful for :ref:`extension development <writing-extensions>`
503        and debugging templates.
504
505        This does not perform preprocessing.  If you want the preprocessing
506        of the extensions to be applied you have to filter source through
507        the :meth:`preprocess` method.
508        """
509        source = text_type(source)
510        try:
511            return self.lexer.tokeniter(source, name, filename)
512        except TemplateSyntaxError:
513            exc_info = sys.exc_info()
514        self.handle_exception(exc_info, source_hint=source)
515
516    def preprocess(self, source, name=None, filename=None):
517        """Preprocesses the source with all extensions.  This is automatically
518        called for all parsing and compiling methods but *not* for :meth:`lex`
519        because there you usually only want the actual source tokenized.
520        """
521        return reduce(lambda s, e: e.preprocess(s, name, filename),
522                      self.iter_extensions(), text_type(source))
523
524    def _tokenize(self, source, name, filename=None, state=None):
525        """Called by the parser to do the preprocessing and filtering
526        for all the extensions.  Returns a :class:`~jinja2.lexer.TokenStream`.
527        """
528        source = self.preprocess(source, name, filename)
529        stream = self.lexer.tokenize(source, name, filename, state)
530        for ext in self.iter_extensions():
531            stream = ext.filter_stream(stream)
532            if not isinstance(stream, TokenStream):
533                stream = TokenStream(stream, name, filename)
534        return stream
535
536    def _generate(self, source, name, filename, defer_init=False):
537        """Internal hook that can be overridden to hook a different generate
538        method in.
539
540        .. versionadded:: 2.5
541        """
542        return generate(source, self, name, filename, defer_init=defer_init,
543                        optimized=self.optimized)
544
545    def _compile(self, source, filename):
546        """Internal hook that can be overridden to hook a different compile
547        method in.
548
549        .. versionadded:: 2.5
550        """
551        return compile(source, filename, 'exec')
552
553    @internalcode
554    def compile(self, source, name=None, filename=None, raw=False,
555                defer_init=False):
556        """Compile a node or template source code.  The `name` parameter is
557        the load name of the template after it was joined using
558        :meth:`join_path` if necessary, not the filename on the file system.
559        the `filename` parameter is the estimated filename of the template on
560        the file system.  If the template came from a database or memory this
561        can be omitted.
562
563        The return value of this method is a python code object.  If the `raw`
564        parameter is `True` the return value will be a string with python
565        code equivalent to the bytecode returned otherwise.  This method is
566        mainly used internally.
567
568        `defer_init` is use internally to aid the module code generator.  This
569        causes the generated code to be able to import without the global
570        environment variable to be set.
571
572        .. versionadded:: 2.4
573           `defer_init` parameter added.
574        """
575        source_hint = None
576        try:
577            if isinstance(source, string_types):
578                source_hint = source
579                source = self._parse(source, name, filename)
580            source = self._generate(source, name, filename,
581                                    defer_init=defer_init)
582            if raw:
583                return source
584            if filename is None:
585                filename = '<template>'
586            else:
587                filename = encode_filename(filename)
588            return self._compile(source, filename)
589        except TemplateSyntaxError:
590            exc_info = sys.exc_info()
591        self.handle_exception(exc_info, source_hint=source_hint)
592
593    def compile_expression(self, source, undefined_to_none=True):
594        """A handy helper method that returns a callable that accepts keyword
595        arguments that appear as variables in the expression.  If called it
596        returns the result of the expression.
597
598        This is useful if applications want to use the same rules as Jinja
599        in template "configuration files" or similar situations.
600
601        Example usage:
602
603        >>> env = Environment()
604        >>> expr = env.compile_expression('foo == 42')
605        >>> expr(foo=23)
606        False
607        >>> expr(foo=42)
608        True
609
610        Per default the return value is converted to `None` if the
611        expression returns an undefined value.  This can be changed
612        by setting `undefined_to_none` to `False`.
613
614        >>> env.compile_expression('var')() is None
615        True
616        >>> env.compile_expression('var', undefined_to_none=False)()
617        Undefined
618
619        .. versionadded:: 2.1
620        """
621        parser = Parser(self, source, state='variable')
622        exc_info = None
623        try:
624            expr = parser.parse_expression()
625            if not parser.stream.eos:
626                raise TemplateSyntaxError('chunk after expression',
627                                          parser.stream.current.lineno,
628                                          None, None)
629            expr.set_environment(self)
630        except TemplateSyntaxError:
631            exc_info = sys.exc_info()
632        if exc_info is not None:
633            self.handle_exception(exc_info, source_hint=source)
634        body = [nodes.Assign(nodes.Name('result', 'store'), expr, lineno=1)]
635        template = self.from_string(nodes.Template(body, lineno=1))
636        return TemplateExpression(template, undefined_to_none)
637
638    def compile_templates(self, target, extensions=None, filter_func=None,
639                          zip='deflated', log_function=None,
640                          ignore_errors=True, py_compile=False):
641        """Finds all the templates the loader can find, compiles them
642        and stores them in `target`.  If `zip` is `None`, instead of in a
643        zipfile, the templates will be stored in a directory.
644        By default a deflate zip algorithm is used. To switch to
645        the stored algorithm, `zip` can be set to ``'stored'``.
646
647        `extensions` and `filter_func` are passed to :meth:`list_templates`.
648        Each template returned will be compiled to the target folder or
649        zipfile.
650
651        By default template compilation errors are ignored.  In case a
652        log function is provided, errors are logged.  If you want template
653        syntax errors to abort the compilation you can set `ignore_errors`
654        to `False` and you will get an exception on syntax errors.
655
656        If `py_compile` is set to `True` .pyc files will be written to the
657        target instead of standard .py files.  This flag does not do anything
658        on pypy and Python 3 where pyc files are not picked up by itself and
659        don't give much benefit.
660
661        .. versionadded:: 2.4
662        """
663        from jinja2.loaders import ModuleLoader
664
665        if log_function is None:
666            log_function = lambda x: None
667
668        if py_compile:
669            if not PY2 or PYPY:
670                from warnings import warn
671                warn(Warning('py_compile has no effect on pypy or Python 3'))
672                py_compile = False
673            else:
674                import imp
675                import marshal
676                py_header = imp.get_magic() + \
677                    u'\xff\xff\xff\xff'.encode('iso-8859-15')
678
679                # Python 3.3 added a source filesize to the header
680                if sys.version_info >= (3, 3):
681                    py_header += u'\x00\x00\x00\x00'.encode('iso-8859-15')
682
683        def write_file(filename, data, mode):
684            if zip:
685                info = ZipInfo(filename)
686                info.external_attr = 0o755 << 16
687                zip_file.writestr(info, data)
688            else:
689                f = open(os.path.join(target, filename), mode)
690                try:
691                    f.write(data)
692                finally:
693                    f.close()
694
695        if zip is not None:
696            from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED, ZIP_STORED
697            zip_file = ZipFile(target, 'w', dict(deflated=ZIP_DEFLATED,
698                                                 stored=ZIP_STORED)[zip])
699            log_function('Compiling into Zip archive "%s"' % target)
700        else:
701            if not os.path.isdir(target):
702                os.makedirs(target)
703            log_function('Compiling into folder "%s"' % target)
704
705        try:
706            for name in self.list_templates(extensions, filter_func):
707                source, filename, _ = self.loader.get_source(self, name)
708                try:
709                    code = self.compile(source, name, filename, True, True)
710                except TemplateSyntaxError as e:
711                    if not ignore_errors:
712                        raise
713                    log_function('Could not compile "%s": %s' % (name, e))
714                    continue
715
716                filename = ModuleLoader.get_module_filename(name)
717
718                if py_compile:
719                    c = self._compile(code, encode_filename(filename))
720                    write_file(filename + 'c', py_header +
721                               marshal.dumps(c), 'wb')
722                    log_function('Byte-compiled "%s" as %s' %
723                                 (name, filename + 'c'))
724                else:
725                    write_file(filename, code, 'w')
726                    log_function('Compiled "%s" as %s' % (name, filename))
727        finally:
728            if zip:
729                zip_file.close()
730
731        log_function('Finished compiling templates')
732
733    def list_templates(self, extensions=None, filter_func=None):
734        """Returns a list of templates for this environment.  This requires
735        that the loader supports the loader's
736        :meth:`~BaseLoader.list_templates` method.
737
738        If there are other files in the template folder besides the
739        actual templates, the returned list can be filtered.  There are two
740        ways: either `extensions` is set to a list of file extensions for
741        templates, or a `filter_func` can be provided which is a callable that
742        is passed a template name and should return `True` if it should end up
743        in the result list.
744
745        If the loader does not support that, a :exc:`TypeError` is raised.
746
747        .. versionadded:: 2.4
748        """
749        x = self.loader.list_templates()
750        if extensions is not None:
751            if filter_func is not None:
752                raise TypeError('either extensions or filter_func '
753                                'can be passed, but not both')
754            filter_func = lambda x: '.' in x and \
755                                    x.rsplit('.', 1)[1] in extensions
756        if filter_func is not None:
757            x = list(ifilter(filter_func, x))
758        return x
759
760    def handle_exception(self, exc_info=None, rendered=False, source_hint=None):
761        """Exception handling helper.  This is used internally to either raise
762        rewritten exceptions or return a rendered traceback for the template.
763        """
764        global _make_traceback
765        if exc_info is None:
766            exc_info = sys.exc_info()
767
768        # the debugging module is imported when it's used for the first time.
769        # we're doing a lot of stuff there and for applications that do not
770        # get any exceptions in template rendering there is no need to load
771        # all of that.
772        if _make_traceback is None:
773            from jinja2.debug import make_traceback as _make_traceback
774        traceback = _make_traceback(exc_info, source_hint)
775        if rendered and self.exception_formatter is not None:
776            return self.exception_formatter(traceback)
777        if self.exception_handler is not None:
778            self.exception_handler(traceback)
779        exc_type, exc_value, tb = traceback.standard_exc_info
780        reraise(exc_type, exc_value, tb)
781
782    def join_path(self, template, parent):
783        """Join a template with the parent.  By default all the lookups are
784        relative to the loader root so this method returns the `template`
785        parameter unchanged, but if the paths should be relative to the
786        parent template, this function can be used to calculate the real
787        template name.
788
789        Subclasses may override this method and implement template path
790        joining here.
791        """
792        return template
793
794    @internalcode
795    def _load_template(self, name, globals):
796        if self.loader is None:
797            raise TypeError('no loader for this environment specified')
798        cache_key = (weakref.ref(self.loader), name)
799        if self.cache is not None:
800            template = self.cache.get(cache_key)
801            if template is not None and (not self.auto_reload or
802                                         template.is_up_to_date):
803                return template
804        template = self.loader.load(self, name, globals)
805        if self.cache is not None:
806            self.cache[cache_key] = template
807        return template
808
809    @internalcode
810    def get_template(self, name, parent=None, globals=None):
811        """Load a template from the loader.  If a loader is configured this
812        method asks the loader for the template and returns a :class:`Template`.
813        If the `parent` parameter is not `None`, :meth:`join_path` is called
814        to get the real template name before loading.
815
816        The `globals` parameter can be used to provide template wide globals.
817        These variables are available in the context at render time.
818
819        If the template does not exist a :exc:`TemplateNotFound` exception is
820        raised.
821
822        .. versionchanged:: 2.4
823           If `name` is a :class:`Template` object it is returned from the
824           function unchanged.
825        """
826        if isinstance(name, Template):
827            return name
828        if parent is not None:
829            name = self.join_path(name, parent)
830        return self._load_template(name, self.make_globals(globals))
831
832    @internalcode
833    def select_template(self, names, parent=None, globals=None):
834        """Works like :meth:`get_template` but tries a number of templates
835        before it fails.  If it cannot find any of the templates, it will
836        raise a :exc:`TemplatesNotFound` exception.
837
838        .. versionadded:: 2.3
839
840        .. versionchanged:: 2.4
841           If `names` contains a :class:`Template` object it is returned
842           from the function unchanged.
843        """
844        if not names:
845            raise TemplatesNotFound(message=u'Tried to select from an empty list '
846                                            u'of templates.')
847        globals = self.make_globals(globals)
848        for name in names:
849            if isinstance(name, Template):
850                return name
851            if parent is not None:
852                name = self.join_path(name, parent)
853            try:
854                return self._load_template(name, globals)
855            except TemplateNotFound:
856                pass
857        raise TemplatesNotFound(names)
858
859    @internalcode
860    def get_or_select_template(self, template_name_or_list,
861                               parent=None, globals=None):
862        """Does a typecheck and dispatches to :meth:`select_template`
863        if an iterable of template names is given, otherwise to
864        :meth:`get_template`.
865
866        .. versionadded:: 2.3
867        """
868        if isinstance(template_name_or_list, string_types):
869            return self.get_template(template_name_or_list, parent, globals)
870        elif isinstance(template_name_or_list, Template):
871            return template_name_or_list
872        return self.select_template(template_name_or_list, parent, globals)
873
874    def from_string(self, source, globals=None, template_class=None):
875        """Load a template from a string.  This parses the source given and
876        returns a :class:`Template` object.
877        """
878        globals = self.make_globals(globals)
879        cls = template_class or self.template_class
880        return cls.from_code(self, self.compile(source), globals, None)
881
882    def make_globals(self, d):
883        """Return a dict for the globals."""
884        if not d:
885            return self.globals
886        return dict(self.globals, **d)
887
888
889class Template(object):
890    """The central template object.  This class represents a compiled template
891    and is used to evaluate it.
892
893    Normally the template object is generated from an :class:`Environment` but
894    it also has a constructor that makes it possible to create a template
895    instance directly using the constructor.  It takes the same arguments as
896    the environment constructor but it's not possible to specify a loader.
897
898    Every template object has a few methods and members that are guaranteed
899    to exist.  However it's important that a template object should be
900    considered immutable.  Modifications on the object are not supported.
901
902    Template objects created from the constructor rather than an environment
903    do have an `environment` attribute that points to a temporary environment
904    that is probably shared with other templates created with the constructor
905    and compatible settings.
906
907    >>> template = Template('Hello {{ name }}!')
908    >>> template.render(name='John Doe') == u'Hello John Doe!'
909    True
910    >>> stream = template.stream(name='John Doe')
911    >>> next(stream) == u'Hello John Doe!'
912    True
913    >>> next(stream)
914    Traceback (most recent call last):
915        ...
916    StopIteration
917    """
918
919    def __new__(cls, source,
920                block_start_string=BLOCK_START_STRING,
921                block_end_string=BLOCK_END_STRING,
922                variable_start_string=VARIABLE_START_STRING,
923                variable_end_string=VARIABLE_END_STRING,
924                comment_start_string=COMMENT_START_STRING,
925                comment_end_string=COMMENT_END_STRING,
926                line_statement_prefix=LINE_STATEMENT_PREFIX,
927                line_comment_prefix=LINE_COMMENT_PREFIX,
928                trim_blocks=TRIM_BLOCKS,
929                lstrip_blocks=LSTRIP_BLOCKS,
930                newline_sequence=NEWLINE_SEQUENCE,
931                keep_trailing_newline=KEEP_TRAILING_NEWLINE,
932                extensions=(),
933                optimized=True,
934                undefined=Undefined,
935                finalize=None,
936                autoescape=False,
937                enable_async=False):
938        env = get_spontaneous_environment(
939            block_start_string, block_end_string, variable_start_string,
940            variable_end_string, comment_start_string, comment_end_string,
941            line_statement_prefix, line_comment_prefix, trim_blocks,
942            lstrip_blocks, newline_sequence, keep_trailing_newline,
943            frozenset(extensions), optimized, undefined, finalize, autoescape,
944            None, 0, False, None, enable_async)
945        return env.from_string(source, template_class=cls)
946
947    @classmethod
948    def from_code(cls, environment, code, globals, uptodate=None):
949        """Creates a template object from compiled code and the globals.  This
950        is used by the loaders and environment to create a template object.
951        """
952        namespace = {
953            'environment':  environment,
954            '__file__':     code.co_filename
955        }
956        exec(code, namespace)
957        rv = cls._from_namespace(environment, namespace, globals)
958        rv._uptodate = uptodate
959        return rv
960
961    @classmethod
962    def from_module_dict(cls, environment, module_dict, globals):
963        """Creates a template object from a module.  This is used by the
964        module loader to create a template object.
965
966        .. versionadded:: 2.4
967        """
968        return cls._from_namespace(environment, module_dict, globals)
969
970    @classmethod
971    def _from_namespace(cls, environment, namespace, globals):
972        t = object.__new__(cls)
973        t.environment = environment
974        t.globals = globals
975        t.name = namespace['name']
976        t.filename = namespace['__file__']
977        t.blocks = namespace['blocks']
978
979        # render function and module
980        t.root_render_func = namespace['root']
981        t._module = None
982
983        # debug and loader helpers
984        t._debug_info = namespace['debug_info']
985        t._uptodate = None
986
987        # store the reference
988        namespace['environment'] = environment
989        namespace['__jinja_template__'] = t
990
991        return t
992
993    def render(self, *args, **kwargs):
994        """This method accepts the same arguments as the `dict` constructor:
995        A dict, a dict subclass or some keyword arguments.  If no arguments
996        are given the context will be empty.  These two calls do the same::
997
998            template.render(knights='that say nih')
999            template.render({'knights': 'that say nih'})
1000
1001        This will return the rendered template as unicode string.
1002        """
1003        vars = dict(*args, **kwargs)
1004        try:
1005            return concat(self.root_render_func(self.new_context(vars)))
1006        except Exception:
1007            exc_info = sys.exc_info()
1008        return self.environment.handle_exception(exc_info, True)
1009
1010    def render_async(self, *args, **kwargs):
1011        """This works similar to :meth:`render` but returns a coroutine
1012        that when awaited returns the entire rendered template string.  This
1013        requires the async feature to be enabled.
1014
1015        Example usage::
1016
1017            await template.render_async(knights='that say nih; asynchronously')
1018        """
1019        # see asyncsupport for the actual implementation
1020        raise NotImplementedError('This feature is not available for this '
1021                                  'version of Python')
1022
1023    def stream(self, *args, **kwargs):
1024        """Works exactly like :meth:`generate` but returns a
1025        :class:`TemplateStream`.
1026        """
1027        return TemplateStream(self.generate(*args, **kwargs))
1028
1029    def generate(self, *args, **kwargs):
1030        """For very large templates it can be useful to not render the whole
1031        template at once but evaluate each statement after another and yield
1032        piece for piece.  This method basically does exactly that and returns
1033        a generator that yields one item after another as unicode strings.
1034
1035        It accepts the same arguments as :meth:`render`.
1036        """
1037        vars = dict(*args, **kwargs)
1038        try:
1039            for event in self.root_render_func(self.new_context(vars)):
1040                yield event
1041        except Exception:
1042            exc_info = sys.exc_info()
1043        else:
1044            return
1045        yield self.environment.handle_exception(exc_info, True)
1046
1047    def generate_async(self, *args, **kwargs):
1048        """An async version of :meth:`generate`.  Works very similarly but
1049        returns an async iterator instead.
1050        """
1051        # see asyncsupport for the actual implementation
1052        raise NotImplementedError('This feature is not available for this '
1053                                  'version of Python')
1054
1055    def new_context(self, vars=None, shared=False, locals=None):
1056        """Create a new :class:`Context` for this template.  The vars
1057        provided will be passed to the template.  Per default the globals
1058        are added to the context.  If shared is set to `True` the data
1059        is passed as it to the context without adding the globals.
1060
1061        `locals` can be a dict of local variables for internal usage.
1062        """
1063        return new_context(self.environment, self.name, self.blocks,
1064                           vars, shared, self.globals, locals)
1065
1066    def make_module(self, vars=None, shared=False, locals=None):
1067        """This method works like the :attr:`module` attribute when called
1068        without arguments but it will evaluate the template on every call
1069        rather than caching it.  It's also possible to provide
1070        a dict which is then used as context.  The arguments are the same
1071        as for the :meth:`new_context` method.
1072        """
1073        return TemplateModule(self, self.new_context(vars, shared, locals))
1074
1075    def make_module_async(self, vars=None, shared=False, locals=None):
1076        """As template module creation can invoke template code for
1077        asynchronous exections this method must be used instead of the
1078        normal :meth:`make_module` one.  Likewise the module attribute
1079        becomes unavailable in async mode.
1080        """
1081        # see asyncsupport for the actual implementation
1082        raise NotImplementedError('This feature is not available for this '
1083                                  'version of Python')
1084
1085    @internalcode
1086    def _get_default_module(self):
1087        if self._module is not None:
1088            return self._module
1089        self._module = rv = self.make_module()
1090        return rv
1091
1092    @property
1093    def module(self):
1094        """The template as module.  This is used for imports in the
1095        template runtime but is also useful if one wants to access
1096        exported template variables from the Python layer:
1097
1098        >>> t = Template('{% macro foo() %}42{% endmacro %}23')
1099        >>> str(t.module)
1100        '23'
1101        >>> t.module.foo() == u'42'
1102        True
1103
1104        This attribute is not available if async mode is enabled.
1105        """
1106        return self._get_default_module()
1107
1108    def get_corresponding_lineno(self, lineno):
1109        """Return the source line number of a line number in the
1110        generated bytecode as they are not in sync.
1111        """
1112        for template_line, code_line in reversed(self.debug_info):
1113            if code_line <= lineno:
1114                return template_line
1115        return 1
1116
1117    @property
1118    def is_up_to_date(self):
1119        """If this variable is `False` there is a newer version available."""
1120        if self._uptodate is None:
1121            return True
1122        return self._uptodate()
1123
1124    @property
1125    def debug_info(self):
1126        """The debug info mapping."""
1127        return [tuple(imap(int, x.split('='))) for x in
1128                self._debug_info.split('&')]
1129
1130    def __repr__(self):
1131        if self.name is None:
1132            name = 'memory:%x' % id(self)
1133        else:
1134            name = repr(self.name)
1135        return '<%s %s>' % (self.__class__.__name__, name)
1136
1137
1138@implements_to_string
1139class TemplateModule(object):
1140    """Represents an imported template.  All the exported names of the
1141    template are available as attributes on this object.  Additionally
1142    converting it into an unicode- or bytestrings renders the contents.
1143    """
1144
1145    def __init__(self, template, context, body_stream=None):
1146        if body_stream is None:
1147            if context.environment.is_async:
1148                raise RuntimeError('Async mode requires a body stream '
1149                                   'to be passed to a template module.  Use '
1150                                   'the async methods of the API you are '
1151                                   'using.')
1152            body_stream = list(template.root_render_func(context))
1153        self._body_stream = body_stream
1154        self.__dict__.update(context.get_exported())
1155        self.__name__ = template.name
1156
1157    def __html__(self):
1158        return Markup(concat(self._body_stream))
1159
1160    def __str__(self):
1161        return concat(self._body_stream)
1162
1163    def __repr__(self):
1164        if self.__name__ is None:
1165            name = 'memory:%x' % id(self)
1166        else:
1167            name = repr(self.__name__)
1168        return '<%s %s>' % (self.__class__.__name__, name)
1169
1170
1171class TemplateExpression(object):
1172    """The :meth:`jinja2.Environment.compile_expression` method returns an
1173    instance of this object.  It encapsulates the expression-like access
1174    to the template with an expression it wraps.
1175    """
1176
1177    def __init__(self, template, undefined_to_none):
1178        self._template = template
1179        self._undefined_to_none = undefined_to_none
1180
1181    def __call__(self, *args, **kwargs):
1182        context = self._template.new_context(dict(*args, **kwargs))
1183        consume(self._template.root_render_func(context))
1184        rv = context.vars['result']
1185        if self._undefined_to_none and isinstance(rv, Undefined):
1186            rv = None
1187        return rv
1188
1189
1190@implements_iterator
1191class TemplateStream(object):
1192    """A template stream works pretty much like an ordinary python generator
1193    but it can buffer multiple items to reduce the number of total iterations.
1194    Per default the output is unbuffered which means that for every unbuffered
1195    instruction in the template one unicode string is yielded.
1196
1197    If buffering is enabled with a buffer size of 5, five items are combined
1198    into a new unicode string.  This is mainly useful if you are streaming
1199    big templates to a client via WSGI which flushes after each iteration.
1200    """
1201
1202    def __init__(self, gen):
1203        self._gen = gen
1204        self.disable_buffering()
1205
1206    def dump(self, fp, encoding=None, errors='strict'):
1207        """Dump the complete stream into a file or file-like object.
1208        Per default unicode strings are written, if you want to encode
1209        before writing specify an `encoding`.
1210
1211        Example usage::
1212
1213            Template('Hello {{ name }}!').stream(name='foo').dump('hello.html')
1214        """
1215        close = False
1216        if isinstance(fp, string_types):
1217            if encoding is None:
1218                encoding = 'utf-8'
1219            fp = open(fp, 'wb')
1220            close = True
1221        try:
1222            if encoding is not None:
1223                iterable = (x.encode(encoding, errors) for x in self)
1224            else:
1225                iterable = self
1226            if hasattr(fp, 'writelines'):
1227                fp.writelines(iterable)
1228            else:
1229                for item in iterable:
1230                    fp.write(item)
1231        finally:
1232            if close:
1233                fp.close()
1234
1235    def disable_buffering(self):
1236        """Disable the output buffering."""
1237        self._next = partial(next, self._gen)
1238        self.buffered = False
1239
1240    def _buffered_generator(self, size):
1241        buf = []
1242        c_size = 0
1243        push = buf.append
1244
1245        while 1:
1246            try:
1247                while c_size < size:
1248                    c = next(self._gen)
1249                    push(c)
1250                    if c:
1251                        c_size += 1
1252            except StopIteration:
1253                if not c_size:
1254                    return
1255            yield concat(buf)
1256            del buf[:]
1257            c_size = 0
1258
1259    def enable_buffering(self, size=5):
1260        """Enable buffering.  Buffer `size` items before yielding them."""
1261        if size <= 1:
1262            raise ValueError('buffer size too small')
1263
1264        self.buffered = True
1265        self._next = partial(next, self._buffered_generator(size))
1266
1267    def __iter__(self):
1268        return self
1269
1270    def __next__(self):
1271        return self._next()
1272
1273
1274# hook in default template class.  if anyone reads this comment: ignore that
1275# it's possible to use custom templates ;-)
1276Environment.template_class = Template
1277