update jinja2 to get better python3 support

README.md

@@ -48,7 +48,7 @@ some unfinished backends for SPARC, ARM
 
 Prerequisites for the build:
 
-* python (>=2.4)
+* python (2.6.x, 2.7.x or >=3.3 are supported)
 * perl
 * an ANSI C99 compiler (gcc, clang, icc are known to work)
 

scripts/jinja2/__init__.py

@@ -27,7 +27,7 @@
     :license: BSD, see LICENSE for more details.
 """
 __docformat__ = 'restructuredtext en'
-__version__ = '2.6'
+__version__ = '2.7.2'
 
 # high level interface
 from jinja2.environment import Environment, Template

scripts/jinja2/_compat.py

+# -*- coding: utf-8 -*-
+"""
+    jinja2._compat
+    ~~~~~~~~~~~~~~
+
+    Some py2/py3 compatibility support based on a stripped down
+    version of six so we don't have to depend on a specific version
+    of it.
+
+    :copyright: Copyright 2013 by the Jinja team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import sys
+
+PY2 = sys.version_info[0] == 2
+PYPY = hasattr(sys, 'pypy_translation_info')
+_identity = lambda x: x
+
+
+if not PY2:
+    unichr = chr
+    range_type = range
+    text_type = str
+    string_types = (str,)
+
+    iterkeys = lambda d: iter(d.keys())
+    itervalues = lambda d: iter(d.values())
+    iteritems = lambda d: iter(d.items())
+
+    import pickle
+    from io import BytesIO, StringIO
+    NativeStringIO = StringIO
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+    ifilter = filter
+    imap = map
+    izip = zip
+    intern = sys.intern
+
+    implements_iterator = _identity
+    implements_to_string = _identity
+    encode_filename = _identity
+    get_next = lambda x: x.__next__
+
+else:
+    unichr = unichr
+    text_type = unicode
+    range_type = xrange
+    string_types = (str, unicode)
+
+    iterkeys = lambda d: d.iterkeys()
+    itervalues = lambda d: d.itervalues()
+    iteritems = lambda d: d.iteritems()
+
+    import cPickle as pickle
+    from cStringIO import StringIO as BytesIO, StringIO
+    NativeStringIO = BytesIO
+
+    exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
+
+    from itertools import imap, izip, ifilter
+    intern = intern
+
+    def implements_iterator(cls):
+        cls.next = cls.__next__
+        del cls.__next__
+        return cls
+
+    def implements_to_string(cls):
+        cls.__unicode__ = cls.__str__
+        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
+        return cls
+
+    get_next = lambda x: x.next
+
+    def encode_filename(filename):
+        if isinstance(filename, unicode):
+            return filename.encode('utf-8')
+        return filename
+
+try:
+    next = next
+except NameError:
+    def next(it):
+        return it.next()
+
+
+def with_metaclass(meta, *bases):
+    # This requires a bit of explanation: the basic idea is to make a
+    # dummy metaclass for one level of class instanciation that replaces
+    # itself with the actual metaclass.  Because of internal type checks
+    # we also need to make sure that we downgrade the custom metaclass
+    # for one level to something closer to type (that's why __call__ and
+    # __init__ comes back from type etc.).
+    #
+    # This has the advantage over six.with_metaclass in that it does not
+    # introduce dummy classes into the final MRO.
+    class metaclass(meta):
+        __call__ = type.__call__
+        __init__ = type.__init__
+        def __new__(cls, name, this_bases, d):
+            if this_bases is None:
+                return type.__new__(cls, name, (), d)
+            return meta(name, bases, d)
+    return metaclass('temporary_class', None, {})
+
+
+try:
+    from collections import Mapping as mapping_types
+except ImportError:
+    import UserDict
+    mapping_types = (UserDict.UserDict, UserDict.DictMixin, dict)
+
+
+# common types.  These do exist in the special types module too which however
+# does not exist in IronPython out of the box.  Also that way we don't have
+# to deal with implementation specific stuff here
+class _C(object):
+    def method(self): pass
+def _func():
+    yield None
+function_type = type(_func)
+generator_type = type(_func())
+method_type = type(_C().method)
+code_type = type(_C.method.__code__)
+try:
+    raise TypeError()
+except TypeError:
+    _tb = sys.exc_info()[2]
+    traceback_type = type(_tb)
+    frame_type = type(_tb.tb_frame)
+
+
+try:
+    from urllib.parse import quote_from_bytes as url_quote
+except ImportError:
+    from urllib import quote as url_quote
+
+
+try:
+    from thread import allocate_lock
+except ImportError:
+    try:
+        from threading import Lock as allocate_lock
+    except ImportError:
+        from dummy_thread import allocate_lock

scripts/jinja2/_markupsafe/_bundle.py

-# -*- coding: utf-8 -*-
-"""
-    jinja2._markupsafe._bundle
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    This script pulls in markupsafe from a source folder and
-    bundles it with Jinja2.  It does not pull in the speedups
-    module though.
-
-    :copyright: Copyright 2010 by the Jinja team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-import sys
-import os
-import re
-
-
-def rewrite_imports(lines):
-    for idx, line in enumerate(lines):
-        new_line = re.sub(r'(import|from)\s+markupsafe\b',
-                          r'\1 jinja2._markupsafe', line)
-        if new_line != line:
-            lines[idx] = new_line
-
-
-def main():
-    if len(sys.argv) != 2:
-        print('error: only argument is path to markupsafe')
-        sys.exit(1)
-    basedir = os.path.dirname(__file__)
-    markupdir = sys.argv[1]
-    for filename in os.listdir(markupdir):
-        if filename.endswith('.py'):
-            f = open(os.path.join(markupdir, filename))
-            try:
-                lines = list(f)
-            finally:
-                f.close()
-            rewrite_imports(lines)
-            f = open(os.path.join(basedir, filename), 'w')
-            try:
-                for line in lines:
-                    f.write(line)
-            finally:
-                f.close()
-
-
-if __name__ == '__main__':
-    main()

scripts/jinja2/_stringdefs.py

@@ -13,6 +13,8 @@
     :license: BSD, see LICENSE for details.
 """
 
+from jinja2._compat import unichr
+
 Cc = u'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f'
 
 Cf = u'\xad\u0600\u0601\u0602\u0603\u06dd\u070f\u17b4\u17b5\u200b\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2060\u2061\u2062\u2063\u206a\u206b\u206c\u206d\u206e\u206f\ufeff\ufff9\ufffa\ufffb'

scripts/jinja2/bccache.py

@@ -15,25 +15,22 @@
     :license: BSD.
 """
 from os import path, listdir
+import os
 import sys
+import errno
 import marshal
 import tempfile
-import cPickle as pickle
 import fnmatch
-try:
-    from hashlib import sha1
-except ImportError:
-    from sha import new as sha1
+from hashlib import sha1
 from jinja2.utils import open_if_exists
+from jinja2._compat import BytesIO, pickle, PY2, text_type
 
 
 # marshal works better on 3.x, one hack less required
-if sys.version_info > (3, 0):
-    from io import BytesIO
+if not PY2:
     marshal_dump = marshal.dump
     marshal_load = marshal.load
 else:
-    from cStringIO import StringIO as BytesIO
 
     def marshal_dump(code, f):
         if isinstance(f, file):
@@ -165,7 +162,7 @@ class BytecodeCache(object):
         hash = sha1(name.encode('utf-8'))
         if filename is not None:
             filename = '|' + filename
-            if isinstance(filename, unicode):
+            if isinstance(filename, text_type):
                 filename = filename.encode('utf-8')
             hash.update(filename)
         return hash.hexdigest()
@@ -194,7 +191,9 @@ class FileSystemBytecodeCache(BytecodeCache):
     two arguments: The directory where the cache items are stored and a
     pattern string that is used to build the filename.
 
-    If no directory is specified the system temporary items folder is used.
+    If no directory is specified a default cache directory is selected.  On
+    Windows the user's temp directory is used, on UNIX systems a directory
+    is created for the user in the system temp directory.
 
     The pattern can be used to have multiple separate caches operate on the
     same directory.  The default pattern is ``'__jinja2_%s.cache'``.  ``%s``
@@ -207,10 +206,32 @@ class FileSystemBytecodeCache(BytecodeCache):
 
     def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
         if directory is None:
-            directory = tempfile.gettempdir()
+            directory = self._get_default_cache_dir()
         self.directory = directory
         self.pattern = pattern
 
+    def _get_default_cache_dir(self):
+        tmpdir = tempfile.gettempdir()
+
+        # On windows the temporary directory is used specific unless
+        # explicitly forced otherwise.  We can just use that.
+        if os.name == 'n':
+            return tmpdir
+        if not hasattr(os, 'getuid'):
+            raise RuntimeError('Cannot determine safe temp directory.  You '
+                               'need to explicitly provide one.')
+
+        dirname = '_jinja2-cache-%d' % os.getuid()
+        actual_dir = os.path.join(tmpdir, dirname)
+        try:
+            # 448 == 0700
+            os.mkdir(actual_dir, 448)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        return actual_dir
+
     def _get_cache_filename(self, bucket):
         return path.join(self.directory, self.pattern % bucket.key)
 
@@ -282,15 +303,26 @@ class MemcachedBytecodeCache(BytecodeCache):
 
     This bytecode cache does not support clearing of used items in the cache.
     The clear method is a no-operation function.
+
+    .. versionadded:: 2.7
+       Added support for ignoring memcache errors through the
+       `ignore_memcache_errors` parameter.
     """
 
-    def __init__(self, client, prefix='jinja2/bytecode/', timeout=None):
+    def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
+                 ignore_memcache_errors=True):
         self.client = client
         self.prefix = prefix
         self.timeout = timeout
+        self.ignore_memcache_errors = ignore_memcache_errors
 
     def load_bytecode(self, bucket):
-        code = self.client.get(self.prefix + bucket.key)
+        try:
+            code = self.client.get(self.prefix + bucket.key)
+        except Exception:
+            if not self.ignore_memcache_errors:
+                raise
+            code = None
         if code is not None:
             bucket.bytecode_from_string(code)
 
@@ -298,4 +330,8 @@ class MemcachedBytecodeCache(BytecodeCache):
         args = (self.prefix + bucket.key, bucket.bytecode_to_string())
         if self.timeout is not None:
             args += (self.timeout,)
-        self.client.set(*args)
+        try:
+            self.client.set(*args)
+        except Exception:
+            if not self.ignore_memcache_errors:
+                raise

scripts/jinja2/compiler.py

@@ -8,14 +8,16 @@
     :copyright: (c) 2010 by the Jinja Team.
     :license: BSD, see LICENSE for more details.
 """
-from cStringIO import StringIO
 from itertools import chain
 from copy import deepcopy
+from keyword import iskeyword as is_python_keyword
 from jinja2 import nodes
 from jinja2.nodes import EvalContext
 from jinja2.visitor import NodeVisitor
 from jinja2.exceptions import TemplateAssertionError
-from jinja2.utils import Markup, concat, escape, is_python_keyword, next
+from jinja2.utils import Markup, concat, escape
+from jinja2._compat import range_type, next, text_type, string_types, \
+     iteritems, NativeStringIO, imap
 
 
 operators = {
@@ -29,14 +31,6 @@ operators = {
     'notin':    'not in'
 }
 
-try:
-    exec '(0 if 0 else 0)'
-except SyntaxError:
-    have_condexpr = False
-else:
-    have_condexpr = True
-
-
 # what method to iterate over items do we want to use for dict iteration
 # in generated code?  on 2.x let's go with iteritems, on 3.x with items
 if hasattr(dict, 'iteritems'):
@@ -51,7 +45,11 @@ def unoptimize_before_dead_code():
     def f():
         if 0: dummy(x)
     return f
-unoptimize_before_dead_code = bool(unoptimize_before_dead_code().func_closure)
+
+# The getattr is necessary for pypy which does not set this attribute if
+# no closure is on the function
+unoptimize_before_dead_code = bool(
+    getattr(unoptimize_before_dead_code(), '__closure__', None))
 
 
 def generate(node, environment, name, filename, stream=None,
@@ -69,8 +67,8 @@ def has_safe_repr(value):
     """Does the node have a safe representation?"""
     if value is None or value is NotImplemented or value is Ellipsis:
         return True
-    if isinstance(value, (bool, int, long, float, complex, basestring,
-                          xrange, Markup)):
+    if isinstance(value, (bool, int, float, complex, range_type,
+            Markup) + string_types):
         return True
     if isinstance(value, (tuple, list, set, frozenset)):
         for item in value:
@@ -78,7 +76,7 @@ def has_safe_repr(value):
                 return False
         return True
     elif isinstance(value, dict):
-        for key, value in value.iteritems():
+        for key, value in iteritems(value):
             if not has_safe_repr(key):
                 return False
             if not has_safe_repr(value):
@@ -368,7 +366,7 @@ class CodeGenerator(NodeVisitor):
     def __init__(self, environment, name, filename, stream=None,
                  defer_init=False):
         if stream is None:
-            stream = StringIO()
+            stream = NativeStringIO()
         self.environment = environment
         self.name = name
         self.filename = filename
@@ -542,7 +540,7 @@ class CodeGenerator(NodeVisitor):
                 self.write(', ')
                 self.visit(kwarg, frame)
             if extra_kwargs is not None:
-                for key, value in extra_kwargs.iteritems():
+                for key, value in iteritems(extra_kwargs):
                     self.write(', %s=%s' % (key, value))
         if node.dyn_args:
             self.write(', *')
@@ -558,7 +556,7 @@ class CodeGenerator(NodeVisitor):
                 self.visit(kwarg.value, frame)
                 self.write(', ')
             if extra_kwargs is not None:
-                for key, value in extra_kwargs.iteritems():
+                for key, value in iteritems(extra_kwargs):
                     self.write('%r: %s, ' % (key, value))
             if node.dyn_kwargs is not None:
                 self.write('}, **')
@@ -625,7 +623,7 @@ class CodeGenerator(NodeVisitor):
 
     def pop_scope(self, aliases, frame):
         """Restore all aliases and delete unused variables."""
-        for name, alias in aliases.iteritems():
+        for name, alias in iteritems(aliases):
             self.writeline('l_%s = %s' % (name, alias))
         to_delete = set()
         for name in frame.identifiers.declared_locally:
@@ -663,16 +661,16 @@ class CodeGenerator(NodeVisitor):
         # it without aliasing all the variables.
         # this could be fixed in Python 3 where we have the nonlocal
         # keyword or if we switch to bytecode generation
-        overriden_closure_vars = (
+        overridden_closure_vars = (
             func_frame.identifiers.undeclared &
             func_frame.identifiers.declared &
             (func_frame.identifiers.declared_locally |
              func_frame.identifiers.declared_parameter)
         )
-        if overriden_closure_vars:
+        if overridden_closure_vars:
             self.fail('It\'s not possible to set and access variables '
                       'derived from an outer scope! (affects: %s)' %
-                      ', '.join(sorted(overriden_closure_vars)), node.lineno)
+                      ', '.join(sorted(overridden_closure_vars)), node.lineno)
 
         # remove variables from a closure from the frame's undeclared
         # identifiers.
@@ -827,7 +825,7 @@ class CodeGenerator(NodeVisitor):
             self.outdent(2 + (not self.has_known_extends))
 
         # at this point we now have the blocks collected and can visit them too.
-        for name, block in self.blocks.iteritems():
+        for name, block in iteritems(self.blocks):
             block_frame = Frame(eval_ctx)
             block_frame.inspect(block.body)
             block_frame.block = name
@@ -894,12 +892,13 @@ class CodeGenerator(NodeVisitor):
                 self.indent()
             self.writeline('raise TemplateRuntimeError(%r)' %
                            'extended multiple times')
-            self.outdent()
 
             # if we have a known extends already we don't need that code here
             # as we know that the template execution will end here.
             if self.has_known_extends:
                 raise CompilerExit()
+            else:
+                self.outdent()
 
         self.writeline('parent_template = environment.get_template(', node)
         self.visit(node.template, frame)
@@ -930,7 +929,7 @@ class CodeGenerator(NodeVisitor):
 
         func_name = 'get_or_select_template'
         if isinstance(node.template, nodes.Const):
-            if isinstance(node.template.value, basestring):
+            if isinstance(node.template.value, string_types):
                 func_name = 'get_template'
             elif isinstance(node.template.value, (tuple, list)):
                 func_name = 'select_template'
@@ -1032,7 +1031,7 @@ class CodeGenerator(NodeVisitor):
                                discarded_names[0])
             else:
                 self.writeline('context.exported_vars.difference_'
-                               'update((%s))' % ', '.join(map(repr, discarded_names)))
+                               'update((%s))' % ', '.join(imap(repr, discarded_names)))
 
     def visit_For(self, node, frame):
         # when calculating the nodes for the inner frame we have to exclude
@@ -1060,7 +1059,7 @@ class CodeGenerator(NodeVisitor):
 
         # otherwise we set up a buffer and add a function def
         else:
-            self.writeline('def loop(reciter, loop_render_func):', node)
+            self.writeline('def loop(reciter, loop_render_func, depth=0):', node)
             self.indent()
             self.buffer(loop_frame)
             aliases = {}
@@ -1068,6 +1067,7 @@ class CodeGenerator(NodeVisitor):
         # make sure the loop variable is a special one and raise a template
         # assertion error if a loop tries to write to loop
         if extended_loop:
+            self.writeline('l_loop = missing')
             loop_frame.identifiers.add_special('loop')
         for name in node.find_all(nodes.Name):
             if name.ctx == 'store' and name.name == 'loop':
@@ -1118,7 +1118,7 @@ class CodeGenerator(NodeVisitor):
             self.visit(node.iter, loop_frame)
 
         if node.recursive:
-            self.write(', recurse=loop_render_func):')
+            self.write(', loop_render_func, depth):')
         else:
             self.write(extended_loop and '):' or ':')
 
@@ -1216,9 +1216,9 @@ class CodeGenerator(NodeVisitor):
             return
 
         if self.environment.finalize:
-            finalize = lambda x: unicode(self.environment.finalize(x))
+            finalize = lambda x: text_type(self.environment.finalize(x))
         else:
-            finalize = unicode
+            finalize = text_type
 
         # if we are inside a frame that requires output checking, we do so
         outdent_later = False
@@ -1367,7 +1367,7 @@ class CodeGenerator(NodeVisitor):
                                    public_names[0])
                 else:
                     self.writeline('context.exported_vars.update((%s))' %
-                                   ', '.join(map(repr, public_names)))
+                                   ', '.join(imap(repr, public_names)))
 
     # -- Expression Visitors
 
@@ -1555,22 +1555,13 @@ class CodeGenerator(NodeVisitor):
                        'expression on %s evaluated to false and '
                        'no else section was defined.' % self.position(node)))
 
-        if not have_condexpr:
-            self.write('((')
-            self.visit(node.test, frame)
-            self.write(') and (')
-            self.visit(node.expr1, frame)
-            self.write(',) or (')
-            write_expr2()
-            self.write(',))[0]')
-        else:
-            self.write('(')
-            self.visit(node.expr1, frame)
-            self.write(' if ')
-            self.visit(node.test, frame)
-            self.write(' else ')
-            write_expr2()
-            self.write(')')
+        self.write('(')
+        self.visit(node.expr1, frame)
+        self.write(' if ')
+        self.visit(node.test, frame)
+        self.write(' else ')
+        write_expr2()
+        self.write(')')
 
     def visit_Call(self, node, frame, forward_caller=False):
         if self.environment.sandboxed:

scripts/jinja2/debug.py

@@ -13,8 +13,9 @@
 import sys
 import traceback
 from types import TracebackType
-from jinja2.utils import CodeType, missing, internal_code
+from jinja2.utils import missing, internal_code
 from jinja2.exceptions import TemplateSyntaxError
+from jinja2._compat import iteritems, reraise, code_type
 
 # on pypy we can take advantage of transparent proxies
 try:
@@ -25,7 +26,7 @@ except ImportError:
 
 # how does the raise helper look like?
 try:
-    exec "raise TypeError, 'foo'"
+    exec("raise TypeError, 'foo'")
 except SyntaxError:
     raise_helper = 'raise __jinja_exception__[1]'
 except TypeError:
@@ -77,7 +78,7 @@ def make_frame_proxy(frame):
 
 
 class ProcessedTraceback(object):
-    """Holds a Jinja preprocessed traceback for priting or reraising."""
+    """Holds a Jinja preprocessed traceback for printing or reraising."""
 
     def __init__(self, exc_type, exc_value, frames):
         assert frames, 'no frames for this traceback?'
@@ -158,7 +159,7 @@ def translate_exception(exc_info, initial_skip=0):
     frames = []
 
     # skip some internal frames if wanted