Commit 001d52d2 authored by Matthias Braun's avatar Matthias Braun
Browse files

update jinja2 to get better python3 support

parent ecce8d30
......@@ -48,7 +48,7 @@ some unfinished backends for SPARC, ARM
Prerequisites for the build:
* python (>=2.4)
* python (2.6.x, 2.7.x or >=3.3 are supported)
* perl
* an ANSI C99 compiler (gcc, clang, icc are known to work)
......@@ -27,7 +27,7 @@
:license: BSD, see LICENSE for more details.
__docformat__ = 'restructuredtext en'
__version__ = '2.6'
__version__ = '2.7.2'
# high level interface
from jinja2.environment import Environment, Template
# -*- coding: utf-8 -*-
Some py2/py3 compatibility support based on a stripped down
version of six so we don't have to depend on a specific version
of it.
:copyright: Copyright 2013 by the Jinja team, see AUTHORS.
:license: BSD, see LICENSE for details.
import sys
PY2 = sys.version_info[0] == 2
PYPY = hasattr(sys, 'pypy_translation_info')
_identity = lambda x: x
if not PY2:
unichr = chr
range_type = range
text_type = str
string_types = (str,)
iterkeys = lambda d: iter(d.keys())
itervalues = lambda d: iter(d.values())
iteritems = lambda d: iter(d.items())
import pickle
from io import BytesIO, StringIO
NativeStringIO = StringIO
def reraise(tp, value, tb=None):
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
ifilter = filter
imap = map
izip = zip
intern = sys.intern
implements_iterator = _identity
implements_to_string = _identity
encode_filename = _identity
get_next = lambda x: x.__next__
unichr = unichr
text_type = unicode
range_type = xrange
string_types = (str, unicode)
iterkeys = lambda d: d.iterkeys()
itervalues = lambda d: d.itervalues()
iteritems = lambda d: d.iteritems()
import cPickle as pickle
from cStringIO import StringIO as BytesIO, StringIO
NativeStringIO = BytesIO
exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
from itertools import imap, izip, ifilter
intern = intern
def implements_iterator(cls): = cls.__next__
del cls.__next__
return cls
def implements_to_string(cls):
cls.__unicode__ = cls.__str__
cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
return cls
get_next = lambda x:
def encode_filename(filename):
if isinstance(filename, unicode):
return filename.encode('utf-8')
return filename
next = next
except NameError:
def next(it):
def with_metaclass(meta, *bases):
# This requires a bit of explanation: the basic idea is to make a
# dummy metaclass for one level of class instanciation that replaces
# itself with the actual metaclass. Because of internal type checks
# we also need to make sure that we downgrade the custom metaclass
# for one level to something closer to type (that's why __call__ and
# __init__ comes back from type etc.).
# This has the advantage over six.with_metaclass in that it does not
# introduce dummy classes into the final MRO.
class metaclass(meta):
__call__ = type.__call__
__init__ = type.__init__
def __new__(cls, name, this_bases, d):
if this_bases is None:
return type.__new__(cls, name, (), d)
return meta(name, bases, d)
return metaclass('temporary_class', None, {})
from collections import Mapping as mapping_types
except ImportError:
import UserDict
mapping_types = (UserDict.UserDict, UserDict.DictMixin, dict)
# common types. These do exist in the special types module too which however
# does not exist in IronPython out of the box. Also that way we don't have
# to deal with implementation specific stuff here
class _C(object):
def method(self): pass
def _func():
yield None
function_type = type(_func)
generator_type = type(_func())
method_type = type(_C().method)
code_type = type(_C.method.__code__)
raise TypeError()
except TypeError:
_tb = sys.exc_info()[2]
traceback_type = type(_tb)
frame_type = type(_tb.tb_frame)
from urllib.parse import quote_from_bytes as url_quote
except ImportError:
from urllib import quote as url_quote
from thread import allocate_lock
except ImportError:
from threading import Lock as allocate_lock
except ImportError:
from dummy_thread import allocate_lock
# -*- coding: utf-8 -*-
This script pulls in markupsafe from a source folder and
bundles it with Jinja2. It does not pull in the speedups
module though.
:copyright: Copyright 2010 by the Jinja team, see AUTHORS.
:license: BSD, see LICENSE for details.
import sys
import os
import re
def rewrite_imports(lines):
for idx, line in enumerate(lines):
new_line = re.sub(r'(import|from)\s+markupsafe\b',
r'\1 jinja2._markupsafe', line)
if new_line != line:
lines[idx] = new_line
def main():
if len(sys.argv) != 2:
print('error: only argument is path to markupsafe')
basedir = os.path.dirname(__file__)
markupdir = sys.argv[1]
for filename in os.listdir(markupdir):
if filename.endswith('.py'):
f = open(os.path.join(markupdir, filename))
lines = list(f)
f = open(os.path.join(basedir, filename), 'w')
for line in lines:
if __name__ == '__main__':
......@@ -13,6 +13,8 @@
:license: BSD, see LICENSE for details.
from jinja2._compat import unichr
Cc = u'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f'
Cf = u'\xad\u0600\u0601\u0602\u0603\u06dd\u070f\u17b4\u17b5\u200b\u200c\u200d\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2060\u2061\u2062\u2063\u206a\u206b\u206c\u206d\u206e\u206f\ufeff\ufff9\ufffa\ufffb'
......@@ -15,25 +15,22 @@
:license: BSD.
from os import path, listdir
import os
import sys
import errno
import marshal
import tempfile
import cPickle as pickle
import fnmatch
from hashlib import sha1
except ImportError:
from sha import new as sha1
from hashlib import sha1
from jinja2.utils import open_if_exists
from jinja2._compat import BytesIO, pickle, PY2, text_type
# marshal works better on 3.x, one hack less required
if sys.version_info > (3, 0):
from io import BytesIO
if not PY2:
marshal_dump = marshal.dump
marshal_load = marshal.load
from cStringIO import StringIO as BytesIO
def marshal_dump(code, f):
if isinstance(f, file):
......@@ -165,7 +162,7 @@ class BytecodeCache(object):
hash = sha1(name.encode('utf-8'))
if filename is not None:
filename = '|' + filename
if isinstance(filename, unicode):
if isinstance(filename, text_type):
filename = filename.encode('utf-8')
return hash.hexdigest()
......@@ -194,7 +191,9 @@ class FileSystemBytecodeCache(BytecodeCache):
two arguments: The directory where the cache items are stored and a
pattern string that is used to build the filename.
If no directory is specified the system temporary items folder is used.
If no directory is specified a default cache directory is selected. On
Windows the user's temp directory is used, on UNIX systems a directory
is created for the user in the system temp directory.
The pattern can be used to have multiple separate caches operate on the
same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s``
......@@ -207,10 +206,32 @@ class FileSystemBytecodeCache(BytecodeCache):
def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
if directory is None:
directory = tempfile.gettempdir()
directory = self._get_default_cache_dir() = directory
self.pattern = pattern
def _get_default_cache_dir(self):
tmpdir = tempfile.gettempdir()
# On windows the temporary directory is used specific unless
# explicitly forced otherwise. We can just use that.
if == 'n':
return tmpdir
if not hasattr(os, 'getuid'):
raise RuntimeError('Cannot determine safe temp directory. You '
'need to explicitly provide one.')
dirname = '_jinja2-cache-%d' % os.getuid()
actual_dir = os.path.join(tmpdir, dirname)
# 448 == 0700
os.mkdir(actual_dir, 448)
except OSError as e:
if e.errno != errno.EEXIST:
return actual_dir
def _get_cache_filename(self, bucket):
return path.join(, self.pattern % bucket.key)
......@@ -282,15 +303,26 @@ class MemcachedBytecodeCache(BytecodeCache):
This bytecode cache does not support clearing of used items in the cache.
The clear method is a no-operation function.
.. versionadded:: 2.7
Added support for ignoring memcache errors through the
`ignore_memcache_errors` parameter.
def __init__(self, client, prefix='jinja2/bytecode/', timeout=None):
def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
self.client = client
self.prefix = prefix
self.timeout = timeout
self.ignore_memcache_errors = ignore_memcache_errors
def load_bytecode(self, bucket):
code = self.client.get(self.prefix + bucket.key)
code = self.client.get(self.prefix + bucket.key)
except Exception:
if not self.ignore_memcache_errors:
code = None
if code is not None:
......@@ -298,4 +330,8 @@ class MemcachedBytecodeCache(BytecodeCache):
args = (self.prefix + bucket.key, bucket.bytecode_to_string())
if self.timeout is not None:
args += (self.timeout,)
except Exception:
if not self.ignore_memcache_errors:
......@@ -8,14 +8,16 @@
:copyright: (c) 2010 by the Jinja Team.
:license: BSD, see LICENSE for more details.
from cStringIO import StringIO
from itertools import chain
from copy import deepcopy
from keyword import iskeyword as is_python_keyword
from jinja2 import nodes
from jinja2.nodes import EvalContext
from jinja2.visitor import NodeVisitor
from jinja2.exceptions import TemplateAssertionError
from jinja2.utils import Markup, concat, escape, is_python_keyword, next
from jinja2.utils import Markup, concat, escape
from jinja2._compat import range_type, next, text_type, string_types, \
iteritems, NativeStringIO, imap
operators = {
......@@ -29,14 +31,6 @@ operators = {
'notin': 'not in'
exec '(0 if 0 else 0)'
except SyntaxError:
have_condexpr = False
have_condexpr = True
# what method to iterate over items do we want to use for dict iteration
# in generated code? on 2.x let's go with iteritems, on 3.x with items
if hasattr(dict, 'iteritems'):
......@@ -51,7 +45,11 @@ def unoptimize_before_dead_code():
def f():
if 0: dummy(x)
return f
unoptimize_before_dead_code = bool(unoptimize_before_dead_code().func_closure)
# The getattr is necessary for pypy which does not set this attribute if
# no closure is on the function
unoptimize_before_dead_code = bool(
getattr(unoptimize_before_dead_code(), '__closure__', None))
def generate(node, environment, name, filename, stream=None,
......@@ -69,8 +67,8 @@ def has_safe_repr(value):
"""Does the node have a safe representation?"""
if value is None or value is NotImplemented or value is Ellipsis:
return True
if isinstance(value, (bool, int, long, float, complex, basestring,
xrange, Markup)):
if isinstance(value, (bool, int, float, complex, range_type,
Markup) + string_types):
return True
if isinstance(value, (tuple, list, set, frozenset)):
for item in value:
......@@ -78,7 +76,7 @@ def has_safe_repr(value):
return False
return True
elif isinstance(value, dict):
for key, value in value.iteritems():
for key, value in iteritems(value):
if not has_safe_repr(key):
return False
if not has_safe_repr(value):
......@@ -368,7 +366,7 @@ class CodeGenerator(NodeVisitor):
def __init__(self, environment, name, filename, stream=None,
if stream is None:
stream = StringIO()
stream = NativeStringIO()
self.environment = environment = name
self.filename = filename
......@@ -542,7 +540,7 @@ class CodeGenerator(NodeVisitor):
self.write(', ')
self.visit(kwarg, frame)
if extra_kwargs is not None:
for key, value in extra_kwargs.iteritems():
for key, value in iteritems(extra_kwargs):
self.write(', %s=%s' % (key, value))
if node.dyn_args:
self.write(', *')
......@@ -558,7 +556,7 @@ class CodeGenerator(NodeVisitor):
self.visit(kwarg.value, frame)
self.write(', ')
if extra_kwargs is not None:
for key, value in extra_kwargs.iteritems():
for key, value in iteritems(extra_kwargs):
self.write('%r: %s, ' % (key, value))
if node.dyn_kwargs is not None:
self.write('}, **')
......@@ -625,7 +623,7 @@ class CodeGenerator(NodeVisitor):
def pop_scope(self, aliases, frame):
"""Restore all aliases and delete unused variables."""
for name, alias in aliases.iteritems():
for name, alias in iteritems(aliases):
self.writeline('l_%s = %s' % (name, alias))
to_delete = set()
for name in frame.identifiers.declared_locally:
......@@ -663,16 +661,16 @@ class CodeGenerator(NodeVisitor):
# it without aliasing all the variables.
# this could be fixed in Python 3 where we have the nonlocal
# keyword or if we switch to bytecode generation
overriden_closure_vars = (
overridden_closure_vars = (
func_frame.identifiers.undeclared &
func_frame.identifiers.declared &
(func_frame.identifiers.declared_locally |
if overriden_closure_vars:
if overridden_closure_vars:'It\'s not possible to set and access variables '
'derived from an outer scope! (affects: %s)' %
', '.join(sorted(overriden_closure_vars)), node.lineno)
', '.join(sorted(overridden_closure_vars)), node.lineno)
# remove variables from a closure from the frame's undeclared
# identifiers.
......@@ -827,7 +825,7 @@ class CodeGenerator(NodeVisitor):
self.outdent(2 + (not self.has_known_extends))
# at this point we now have the blocks collected and can visit them too.
for name, block in self.blocks.iteritems():
for name, block in iteritems(self.blocks):
block_frame = Frame(eval_ctx)
block_frame.block = name
......@@ -894,12 +892,13 @@ class CodeGenerator(NodeVisitor):
self.writeline('raise TemplateRuntimeError(%r)' %
'extended multiple times')
# if we have a known extends already we don't need that code here
# as we know that the template execution will end here.
if self.has_known_extends:
raise CompilerExit()
self.writeline('parent_template = environment.get_template(', node)
self.visit(node.template, frame)
......@@ -930,7 +929,7 @@ class CodeGenerator(NodeVisitor):
func_name = 'get_or_select_template'
if isinstance(node.template, nodes.Const):
if isinstance(node.template.value, basestring):
if isinstance(node.template.value, string_types):
func_name = 'get_template'
elif isinstance(node.template.value, (tuple, list)):
func_name = 'select_template'
......@@ -1032,7 +1031,7 @@ class CodeGenerator(NodeVisitor):
'update((%s))' % ', '.join(map(repr, discarded_names)))
'update((%s))' % ', '.join(imap(repr, discarded_names)))
def visit_For(self, node, frame):
# when calculating the nodes for the inner frame we have to exclude
......@@ -1060,7 +1059,7 @@ class CodeGenerator(NodeVisitor):
# otherwise we set up a buffer and add a function def
self.writeline('def loop(reciter, loop_render_func):', node)
self.writeline('def loop(reciter, loop_render_func, depth=0):', node)
aliases = {}
......@@ -1068,6 +1067,7 @@ class CodeGenerator(NodeVisitor):
# make sure the loop variable is a special one and raise a template
# assertion error if a loop tries to write to loop
if extended_loop:
self.writeline('l_loop = missing')
for name in node.find_all(nodes.Name):
if name.ctx == 'store' and == 'loop':
......@@ -1118,7 +1118,7 @@ class CodeGenerator(NodeVisitor):
self.visit(node.iter, loop_frame)
if node.recursive:
self.write(', recurse=loop_render_func):')
self.write(', loop_render_func, depth):')
self.write(extended_loop and '):' or ':')
......@@ -1216,9 +1216,9 @@ class CodeGenerator(NodeVisitor):
if self.environment.finalize:
finalize = lambda x: unicode(self.environment.finalize(x))
finalize = lambda x: text_type(self.environment.finalize(x))
finalize = unicode
finalize = text_type
# if we are inside a frame that requires output checking, we do so
outdent_later = False
......@@ -1367,7 +1367,7 @@ class CodeGenerator(NodeVisitor):
self.writeline('context.exported_vars.update((%s))' %
', '.join(map(repr, public_names)))
', '.join(imap(repr, public_names)))
# -- Expression Visitors
......@@ -1555,22 +1555,13 @@ class CodeGenerator(NodeVisitor):
'expression on %s evaluated to false and '
'no else section was defined.' % self.position(node)))
if not have_condexpr:
self.visit(node.test, frame)
self.write(') and (')
self.visit(node.expr1, frame)
self.write(',) or (')
self.visit(node.expr1, frame)
self.write(' if ')
self.visit(node.test, frame)
self.write(' else ')
self.visit(node.expr1, frame)
self.write(' if ')
self.visit(node.test, frame)
self.write(' else ')
def visit_Call(self, node, frame, forward_caller=False):
if self.environment.sandboxed:
......@@ -13,8 +13,9 @@
import sys
import traceback
from types import TracebackType
from jinja2.utils import CodeType, missing, internal_code
from jinja2.utils import missing, internal_code
from jinja2.exceptions import TemplateSyntaxError
from jinja2._compat import iteritems, reraise, code_type
# on pypy we can take advantage of transparent proxies
......@@ -25,7 +26,7 @@ except ImportError:
# how does the raise helper look like?
exec "raise TypeError, 'foo'"
exec("raise TypeError, 'foo'")
except SyntaxError:
raise_helper = 'raise __jinja_exception__[1]'
except TypeError:
......@@ -77,7 +78,7 @@ def make_frame_proxy(frame):
class ProcessedTraceback(object):
"""Holds a Jinja preprocessed traceback for priting or reraising."""
"""Holds a Jinja preprocessed traceback for printing or reraising."""
def __init__(self, exc_type, exc_value, frames):
assert frames, 'no frames for this traceback?'
......@@ -158,7 +159,7 @@ def translate_exception(exc_info, initial_skip=0):
frames = []
# skip some internal frames if wanted