From 40bc44c9943920190079e6339ef10e6702c1c898 Mon Sep 17 00:00:00 2001 From: James Tanner Date: Thu, 27 Apr 2017 17:14:42 -0400 Subject: [PATCH 1/3] Add support for the Environment to optionally return native types. This works by having an alternate CodeGenerator that avoids doing to_string after the yield statement and a new version of concat that handles the returned generator with a bit more "intelligence". --- CHANGES | 3 + docs/contents.rst.inc | 1 + docs/nativetypes.rst | 64 +++++++++++ jinja2/nativetypes.py | 218 ++++++++++++++++++++++++++++++++++++++ tests/test_nativetypes.py | 94 ++++++++++++++++ 5 files changed, 380 insertions(+) create mode 100644 docs/nativetypes.rst create mode 100644 jinja2/nativetypes.py create mode 100644 tests/test_nativetypes.py diff --git a/CHANGES b/CHANGES index 6276b1157..4b0f53475 100644 --- a/CHANGES +++ b/CHANGES @@ -28,11 +28,14 @@ Version 2.10 - Add ``min`` and ``max`` filters. (`#475`_) - Add tests for all comparison operators: ``eq``, ``ne``, ``lt``, ``le``, ``gt``, ``ge``. (`#665`_) +- Add a ``NativeEnvironment`` that renders templates to native Python types + instead of strings. (`#708`_) .. _#469: https://github.com/pallets/jinja/pull/469 .. _#475: https://github.com/pallets/jinja/pull/475 .. _#478: https://github.com/pallets/jinja/pull/478 .. _#665: https://github.com/pallets/jinja/pull/665 +.. _#708: https://github.com/pallets/jinja/pull/708 Version 2.9.6 ------------- diff --git a/docs/contents.rst.inc b/docs/contents.rst.inc index 7ee68703f..467d4cef4 100644 --- a/docs/contents.rst.inc +++ b/docs/contents.rst.inc @@ -7,6 +7,7 @@ Jinja2 Documentation intro api sandbox + nativetypes templates extensions integration diff --git a/docs/nativetypes.rst b/docs/nativetypes.rst new file mode 100644 index 000000000..1a08700b0 --- /dev/null +++ b/docs/nativetypes.rst @@ -0,0 +1,64 @@ +.. module:: jinja2.nativetypes + +.. _nativetypes: + +Native Python Types +=================== + +The default :class:`~jinja2.Environment` renders templates to strings. With +:class:`NativeEnvironment`, rendering a template produces a native Python type. +This is useful if you are using Jinja outside the context of creating text +files. For example, your code may have an intermediate step where users may use +templates to define values that will then be passed to a traditional string +environment. + +Examples +-------- + +Adding two values results in an integer, not a string with a number: + +>>> env = NativeEnvironment() +>>> t = env.from_string('{{ x + y }}') +>>> result = t.render(x=4, y=2) +>>> print(result) +6 +>>> print(type(result)) +int + +Rendering list syntax produces a list: + +>>> t = env.from_string('[{% for item in data %}{{ item + 1 }},{% endfor %}]') +>>> result = t.render(data=range(5)) +>>> print(result) +[1, 2, 3, 4, 5] +>>> print(type(result)) +list + +Rendering something that doesn't look like a Python literal produces a string: + +>>> t = env.from_string('{{ x }} * {{ y }}') +>>> result = t.render(x=4, y=2) +>>> print(result) +4 * 2 +>>> print(type(result)) +str + +Rendering a Python object produces that object as long as it is the only node: + +>>> class Foo: +... def __init__(self, value): +... self.value = value +... +>>> result = env.from_string('{{ x }}').render(x=Foo(15)) +>>> print(type(result).__name__) +Foo +>>> print(result.value) +15 + +API +--- + +.. autoclass:: NativeEnvironment([options]) + +.. autoclass:: NativeTemplate([options]) + :members: render diff --git a/jinja2/nativetypes.py b/jinja2/nativetypes.py new file mode 100644 index 000000000..3aca683cd --- /dev/null +++ b/jinja2/nativetypes.py @@ -0,0 +1,218 @@ +import sys +from ast import literal_eval +from itertools import islice, chain +from jinja2 import nodes +from jinja2._compat import text_type +from jinja2.compiler import CodeGenerator +from jinja2.environment import Environment, Template +from jinja2.utils import concat, escape + + +def native_concat(nodes): + """Return a native Python type from the list of compiled nodes. If the + result is a single node, its value is returned. Otherwise, the nodes are + concatenated as strings. If the result can be parsed with + :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the + string is returned. + """ + head = list(islice(nodes, 2)) + + if not head: + return None + + if len(head) == 1: + out = head[0] + else: + out = u''.join([text_type(v) for v in chain(head, nodes)]) + + try: + return literal_eval(out) + except (ValueError, SyntaxError, MemoryError): + return out + + +class NativeCodeGenerator(CodeGenerator): + """A code generator which avoids injecting ``to_string()`` calls around the + internal code Jinja uses to render templates. + """ + + def visit_Output(self, node, frame): + """Same as :meth:`CodeGenerator.visit_Output`, but do not call + ``to_string`` on output nodes in generated code. + """ + if self.has_known_extends and frame.require_output_check: + return + + finalize = self.environment.finalize + finalize_context = getattr(finalize, 'contextfunction', False) + finalize_eval = getattr(finalize, 'evalcontextfunction', False) + finalize_env = getattr(finalize, 'environmentfunction', False) + + if finalize is not None: + if finalize_context or finalize_eval: + const_finalize = None + elif finalize_env: + def const_finalize(x): + return finalize(self.environment, x) + else: + const_finalize = finalize + else: + def const_finalize(x): + return x + + # If we are inside a frame that requires output checking, we do so. + outdent_later = False + + if frame.require_output_check: + self.writeline('if parent_template is None:') + self.indent() + outdent_later = True + + # Try to evaluate as many chunks as possible into a static string at + # compile time. + body = [] + + for child in node.nodes: + try: + if const_finalize is None: + raise nodes.Impossible() + + const = child.as_const(frame.eval_ctx) + except nodes.Impossible: + body.append(child) + continue + + # the frame can't be volatile here, because otherwise the as_const + # function would raise an Impossible exception at that point + try: + if frame.eval_ctx.autoescape: + if hasattr(const, '__html__'): + const = const.__html__() + else: + const = escape(const) + + const = const_finalize(const) + except Exception: + # if something goes wrong here we evaluate the node at runtime + # for easier debugging + body.append(child) + continue + + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + # if we have less than 3 nodes or a buffer we yield or extend/append + if len(body) < 3 or frame.buffer is not None: + if frame.buffer is not None: + # for one item we append, for more we extend + if len(body) == 1: + self.writeline('%s.append(' % frame.buffer) + else: + self.writeline('%s.extend((' % frame.buffer) + + self.indent() + + for item in body: + if isinstance(item, list): + val = repr(native_concat(item)) + + if frame.buffer is None: + self.writeline('yield ' + val) + else: + self.writeline(val + ',') + else: + if frame.buffer is None: + self.writeline('yield ', item) + else: + self.newline(item) + + close = 0 + + if finalize is not None: + self.write('environment.finalize(') + + if finalize_context: + self.write('context, ') + + close += 1 + + self.visit(item, frame) + + if close > 0: + self.write(')' * close) + + if frame.buffer is not None: + self.write(',') + + if frame.buffer is not None: + # close the open parentheses + self.outdent() + self.writeline(len(body) == 1 and ')' or '))') + + # otherwise we create a format string as this is faster in that case + else: + format = [] + arguments = [] + + for item in body: + if isinstance(item, list): + format.append(native_concat(item).replace('%', '%%')) + else: + format.append('%s') + arguments.append(item) + + self.writeline('yield ') + self.write(repr(concat(format)) + ' % (') + self.indent() + + for argument in arguments: + self.newline(argument) + close = 0 + + if finalize is not None: + self.write('environment.finalize(') + + if finalize_context: + self.write('context, ') + elif finalize_eval: + self.write('context.eval_ctx, ') + elif finalize_env: + self.write('environment, ') + + close += 1 + + self.visit(argument, frame) + self.write(')' * close + ', ') + + self.outdent() + self.writeline(')') + + if outdent_later: + self.outdent() + + +class NativeTemplate(Template): + def render(self, *args, **kwargs): + """Render the template to produce a native Python type. If the result + is a single node, its value is returned. Otherwise, the nodes are + concatenated as strings. If the result can be parsed with + :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the + string is returned. + """ + vars = dict(*args, **kwargs) + + try: + return native_concat(self.root_render_func(self.new_context(vars))) + except Exception: + exc_info = sys.exc_info() + + return self.environment.handle_exception(exc_info, True) + + +class NativeEnvironment(Environment): + """An environment that renders templates to native Python types.""" + + code_generator_class = NativeCodeGenerator + template_class = NativeTemplate diff --git a/tests/test_nativetypes.py b/tests/test_nativetypes.py new file mode 100644 index 000000000..d295466c9 --- /dev/null +++ b/tests/test_nativetypes.py @@ -0,0 +1,94 @@ +import pytest + +from jinja2._compat import text_type +from jinja2.exceptions import UndefinedError +from jinja2.nativetypes import NativeEnvironment +from jinja2.runtime import Undefined + + +@pytest.fixture +def env(): + return NativeEnvironment() + + +class TestNativeEnvironment(object): + def test_is_defined_native_return(self, env): + t = env.from_string('{{ missing is defined }}') + assert not t.render() + + def test_undefined_native_return(self, env): + t = env.from_string('{{ missing }}') + assert isinstance(t.render(), Undefined) + + def test_adding_undefined_native_return(self, env): + t = env.from_string('{{ 3 + missing }}') + + with pytest.raises(UndefinedError): + t.render() + + def test_cast_int(self, env): + t = env.from_string("{{ anumber|int }}") + result = t.render(anumber='3') + assert isinstance(result, int) + assert result == 3 + + def test_list_add(self, env): + t = env.from_string("{{ listone + listtwo }}") + result = t.render(listone=['a', 'b'], listtwo=['c', 'd']) + assert isinstance(result, list) + assert result == ['a', 'b', 'c', 'd'] + + def test_multi_expression_add(self, env): + t = env.from_string("{{ listone }} + {{ listtwo }}") + result = t.render(listone=['a', 'b'], listtwo=['c', 'd']) + assert not isinstance(result, list) + assert result == "['a', 'b'] + ['c', 'd']" + + def test_loops(self, env): + t = env.from_string("{% for x in listone %}{{ x }}{% endfor %}") + result = t.render(listone=['a', 'b', 'c', 'd']) + assert isinstance(result, text_type) + assert result == 'abcd' + + def test_loops_with_ints(self, env): + t = env.from_string("{% for x in listone %}{{ x }}{% endfor %}") + result = t.render(listone=[1, 2, 3, 4]) + assert isinstance(result, int) + assert result == 1234 + + def test_loop_look_alike(self, env): + t = env.from_string("{% for x in listone %}{{ x }}{% endfor %}") + result = t.render(listone=[1]) + assert isinstance(result, int) + assert result == 1 + + def test_booleans(self, env): + t = env.from_string("{{ boolval }}") + result = t.render(boolval=True) + assert isinstance(result, bool) + assert result is True + + t = env.from_string("{{ boolval }}") + result = t.render(boolval=False) + assert isinstance(result, bool) + assert result is False + + t = env.from_string("{{ 1 == 1 }}") + result = t.render() + assert isinstance(result, bool) + assert result is True + + t = env.from_string("{{ 2 + 2 == 5 }}") + result = t.render() + assert isinstance(result, bool) + assert result is False + + t = env.from_string("{{ None == None }}") + result = t.render() + assert isinstance(result, bool) + assert result is True + + t = env.from_string("{{ '' == None }}") + result = t.render() + assert isinstance(result, bool) + assert result is False From 2ffeb693e337828e9955ab1371cf20a81bef0459 Mon Sep 17 00:00:00 2001 From: James Tanner Date: Tue, 25 Jul 2017 21:48:12 -0400 Subject: [PATCH 2/3] Add requested patch for safe_repr and nodes.Impossible --- jinja2/nativetypes.py | 4 +++- tests/test_nativetypes.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/jinja2/nativetypes.py b/jinja2/nativetypes.py index 3aca683cd..fe17e4138 100644 --- a/jinja2/nativetypes.py +++ b/jinja2/nativetypes.py @@ -3,7 +3,7 @@ from itertools import islice, chain from jinja2 import nodes from jinja2._compat import text_type -from jinja2.compiler import CodeGenerator +from jinja2.compiler import CodeGenerator, has_safe_repr from jinja2.environment import Environment, Template from jinja2.utils import concat, escape @@ -78,6 +78,8 @@ def const_finalize(x): raise nodes.Impossible() const = child.as_const(frame.eval_ctx) + if not has_safe_repr(const): + raise nodes.Impossible() except nodes.Impossible: body.append(child) continue diff --git a/tests/test_nativetypes.py b/tests/test_nativetypes.py index d295466c9..8bff50380 100644 --- a/tests/test_nativetypes.py +++ b/tests/test_nativetypes.py @@ -92,3 +92,19 @@ def test_booleans(self, env): result = t.render() assert isinstance(result, bool) assert result is False + + def test_variable_dunder(self, env): + t = env.from_string("{{ x.__class__ }}") + result = t.render(x=True) + assert isinstance(result, type) + + def test_constant_dunder(self, env): + t = env.from_string("{{ true.__class__ }}") + result = t.render() + assert isinstance(result, type) + + def test_constant_dunder_to_string(self, env): + t = env.from_string("{{ true.__class__|string }}") + result = t.render() + assert not isinstance(result, type) + assert result == "" From 68820c39765bb54b548a90746b75484694ad507c Mon Sep 17 00:00:00 2001 From: James Tanner Date: Tue, 25 Jul 2017 22:11:22 -0400 Subject: [PATCH 3/3] quickly fix py3 tests --- tests/test_nativetypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_nativetypes.py b/tests/test_nativetypes.py index 8bff50380..aec1a3b8f 100644 --- a/tests/test_nativetypes.py +++ b/tests/test_nativetypes.py @@ -107,4 +107,4 @@ def test_constant_dunder_to_string(self, env): t = env.from_string("{{ true.__class__|string }}") result = t.render() assert not isinstance(result, type) - assert result == "" + assert result in ["", ""]