From 8a88f5c7c1b0f38e069772b214083a8aca0400d7 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Fri, 7 Oct 2011 19:54:43 +0200 Subject: [PATCH 1/9] Cleaned up sort and dictsort filters, by moving creation of the sort_func into a factory function. --- jinja2/filters.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/jinja2/filters.py b/jinja2/filters.py index 352b16685..3419cb6c9 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -62,6 +62,22 @@ def attrgetter(item): return item return attrgetter +def make_sort_func(func, case_sensitive): + """Returns a callable that is usable as key function for the sorted(), + built-in function, respecting the case sensitivity. + """ + if case_sensitive: + return func + + def sort_func(item): + if func is not None: + item = func(item) + + if isinstance(item, basestring): + return item.lower() + return item + + return sort_func def do_forceescape(value): """Enforce HTML escaping. This will probably double escape variables.""" @@ -182,13 +198,8 @@ def do_dictsort(value, case_sensitive=False, by='key'): else: raise FilterArgumentError('You can only sort by either ' '"key" or "value"') - def sort_func(item): - value = item[pos] - if isinstance(value, basestring) and not case_sensitive: - value = value.lower() - return value - return sorted(value.items(), key=sort_func) + return sorted(value.items(), key=make_sort_func(itemgetter(pos), case_sensitive)) @environmentfilter @@ -219,18 +230,11 @@ def do_sort(environment, value, reverse=False, case_sensitive=False, .. versionchanged:: 2.6 The `attribute` parameter was added. """ - if not case_sensitive: - def sort_func(item): - if isinstance(item, basestring): - item = item.lower() - return item - else: - sort_func = None if attribute is not None: getter = make_attrgetter(environment, attribute) - def sort_func(item, processor=sort_func or (lambda x: x)): - return processor(getter(item)) - return sorted(value, key=sort_func, reverse=reverse) + else: + getter = None + return sorted(value, key=make_sort_func(getter, case_sensitive), reverse=reverse) def do_default(value, default_value=u'', boolean=False): From 0645a8e83ff95fa1ddda94c7ce818e20685732a8 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Mon, 10 Oct 2011 16:43:30 +0200 Subject: [PATCH 2/9] Added 'min' and 'max' filter. --- jinja2/filters.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++- jinja2/utils.py | 33 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/jinja2/filters.py b/jinja2/filters.py index 3419cb6c9..68f2d4852 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -13,7 +13,7 @@ from random import choice from operator import itemgetter from itertools import imap, groupby -from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode +from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, min, max from jinja2.runtime import Undefined from jinja2.exceptions import FilterArgumentError @@ -341,6 +341,64 @@ def do_random(environment, seq): return environment.undefined('No random item, sequence was empty.') +@environmentfilter +def do_min(environment, seq, attribute=None, case_sensitive=False): + """Return the smallest item from the sequence. + + .. sourcecode:: jinja + + {{ [1, 2, 3]|min }} + -> 1 + + It is also possible to get the item providing the smallest value for a + certain attribute: + + .. sourcecode:: jinja + + {{ users|min('last_login') }} + """ + if attribute is not None: + getter = make_attrgetter(environment, attribute) + else: + getter = None + key = make_sort_func(getter, case_sensitive) + try: + if key is None: + return min(seq) + return min(seq, key=key) + except ValueError: + return environment.undefined('No smallest item, sequence was empty.') + + +@environmentfilter +def do_max(environment, seq, attribute=None, case_sensitive=False): + """Return the largest item from the sequence. + + .. sourcecode:: jinja + + {{ [1, 2, 3]|max }} + -> 3 + + It is also possible to get the item providing the largest value for a + certain attribute: + + .. sourcecode:: jinja + + {{ users|max('last_login') }} + """ + if attribute is not None: + getter = make_attrgetter(environment, attribute) + else: + getter = None + key = make_sort_func(getter, case_sensitive) + try: + if key is None: + return max(seq) + return max(seq, key=key) + except ValueError: + return environment.undefined('No largest item, sequence was empty.') + + def do_filesizeformat(value, binary=False): """Format the value like a 'human-readable' file size (i.e. 13 kB, 4.1 MB, 102 Bytes, etc). Per default decimal prefixes are used (Mega, @@ -781,6 +839,8 @@ def do_attr(environment, obj, name): 'first': do_first, 'last': do_last, 'random': do_random, + 'min': do_min, + 'max': do_max, 'filesizeformat': do_filesizeformat, 'pprint': do_pprint, 'truncate': do_truncate, diff --git a/jinja2/utils.py b/jinja2/utils.py index 49e9e9ae0..7f3a79238 100644 --- a/jinja2/utils.py +++ b/jinja2/utils.py @@ -11,6 +11,7 @@ import re import sys import errno +import operator try: from thread import allocate_lock except ImportError: @@ -72,6 +73,38 @@ def next(x): return x.next() +# for python 2.4 we wrap the min and max builtins in order to support +# the 'key' argument. +def _make_min_or_max(builtin, op): + try: + builtin([1], key=lambda x: x) + except TypeError: + def min_or_max(*seq, **kwargs): + if 'key' not in kwargs: + return builtin(*seq, **kwargs) + + if len(seq) == 1: + seq = seq[0] + key = kwargs['key'] + + def func(x, y): + k = key(y) + if x is None or op(k, x[1]): + return (y, k) + return x + + result = reduce(func, seq, None) + if result is None: + raise ValueError(builtin.__name__ + '() arg is an empty sequence') + return result[0] + else: + min_or_max = builtin + return min_or_max +min = _make_min_or_max(min, operator.lt) +max = _make_min_or_max(max, operator.gt) +del _make_min_or_max + + # if this python version is unable to deal with unicode filenames # when passed to encode we let this function encode it properly. # This is used in a couple of places. As far as Jinja is concerned From 59d633afd1b861f8ff479a1cdd6991f559b4a0b6 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Mon, 10 Oct 2011 19:32:03 +0200 Subject: [PATCH 3/9] Added 'filter' filter. --- jinja2/filters.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/jinja2/filters.py b/jinja2/filters.py index 68f2d4852..85878a4e5 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -646,6 +646,54 @@ def do_batch(value, linecount, fill_with=None): yield tmp +@environmentfilter +def do_filter(environment, iterable, test=None, *args, **kwargs): + + """Filter an iterable using tests and attribute lookups. + + .. sourcecode:: jinja + + {{ range(10)|filter('even') }} + -> [0,2,4,6,8] + + If you don't specify a ``'test'``, all values that evaluate to False (e.g. + none, false, '', 0) are excluded: + + .. sourcecode:: jinja + + {{ [none,true,false,'foo','',42,0]|filter }} + -> [true,'foo',42] + + It is also possible to filter by a certain attribute: + + .. sourcecode:: jinja + + {{ users|filter(attribute='is_staff') }} + + And you can also invert the the filter condition: + + .. sourcecode:: jinja + + {{ [none,true,false,'foo','',42,0]|filter(invert=false) }} + -> [none, false, '', 0] + """ + if test is not None: + try: + test = environment.tests[test] + except KeyError: + raise FilterArgumentError("there is no test '%s'" % test) + else: + test = bool + + attribute = kwargs.pop('attribute', None) + invert = kwargs.pop('invert', False) + + if attribute is not None: + iterable = imap(make_attrgetter(environment, attribute), iterable) + + return [x for x in iterable if test(x, *args) != invert] + + def do_round(value, precision=0, method='common'): """Round the number to a given precision. The first parameter specifies the precision (default is ``0``), the @@ -856,6 +904,7 @@ def do_attr(environment, obj, name): 'striptags': do_striptags, 'slice': do_slice, 'batch': do_batch, + 'filter': do_filter, 'sum': do_sum, 'abs': abs, 'round': do_round, From 31f87d3bbcc8c06cb4386d76880f15a2d9f04462 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Tue, 11 Oct 2011 12:32:15 +0200 Subject: [PATCH 4/9] Added a fallback for dealing with non-sequence iterables to the 'last' filter. --- jinja2/filters.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/jinja2/filters.py b/jinja2/filters.py index 85878a4e5..5bf2eb5ae 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -327,7 +327,17 @@ def do_first(environment, seq): def do_last(environment, seq): """Return the last item of a sequence.""" try: - return iter(reversed(seq)).next() + try: + rv = reversed(seq).next() + except TypeError: + it = iter(seq) + rv = it.next() + while True: + try: + rv = it.next() + except StopIteration: + break + return rv except StopIteration: return environment.undefined('No last item, sequence was empty.') From 2d9a89607d7c831c57828980f21aaaf0200b1270 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Tue, 11 Oct 2011 14:29:37 +0200 Subject: [PATCH 5/9] Added 'unique' filter. --- jinja2/filters.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/jinja2/filters.py b/jinja2/filters.py index 5bf2eb5ae..abefa7a43 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -237,6 +237,50 @@ def do_sort(environment, value, reverse=False, case_sensitive=False, return sorted(value, key=make_sort_func(getter, case_sensitive), reverse=reverse) +@environmentfilter +def do_unique(environment, iterable, case_sensitive=False): + """Return a list of unique items from the the given iterable. + + .. sourcecode:: jinja + + {{ ['foo', 'bar', 'foobar', 'FooBar']|unique }} + -> ['foo', 'bar', 'foobar'] + + This filter complements the `groupby` filter, which sorts and groups an + iterable by a certain attribute. The `unique` filter groups the items + from the iterable by themself instead and always returns a flat list of + unique items. That can be useuful for example when you need to concatenate + that items: + + .. sourcecode:: jinja + + {{ ['foo', 'bar', 'foobar', 'FooBar']|unique|join(',') }} + -> foo,bar,foobar + + Also note that the resulting list contains the items in the same order + as their first occurence in the iterable passed to the filter. If sorting + is needed you can still chain the `unique` and `sort` filter: + + .. sourcecode:: jinja + + {{ ['foo', 'bar', 'foobar', 'FooBar']|unique|sort }} + -> ['bar', 'foo', 'foobar'] + """ + sort_func = make_sort_func(lambda x: x, case_sensitive) + + uniq_items = [] + norm_items = [] + + for item in iterable: + norm_item = sort_func(item) + + if norm_item not in norm_items: + norm_items.append(norm_item) + uniq_items.append(item) + + return uniq_items + + def do_default(value, default_value=u'', boolean=False): """If the value is undefined it will return the passed default value, otherwise the value of the variable: @@ -888,6 +932,7 @@ def do_attr(environment, obj, name): 'count': len, 'dictsort': do_dictsort, 'sort': do_sort, + 'unique': do_unique, 'length': len, 'reverse': do_reverse, 'center': do_center, From 0abc3654b784bf1e25905e76dfb4e1e00d84c263 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Tue, 11 Oct 2011 15:51:30 +0200 Subject: [PATCH 6/9] Got rid of redundant sorted() in 'groupby' filter. --- jinja2/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jinja2/filters.py b/jinja2/filters.py index abefa7a43..a03f9f03c 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -822,7 +822,7 @@ def do_groupby(environment, value, attribute): attribute of another attribute. """ expr = make_attrgetter(environment, attribute) - return sorted(map(_GroupTuple, groupby(sorted(value, key=expr), expr))) + return map(_GroupTuple, groupby(sorted(value, key=expr), expr)) class _GroupTuple(tuple): From a8dabbed2694b4b099116275488ea28e15713aa9 Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Tue, 11 Oct 2011 17:21:42 +0200 Subject: [PATCH 7/9] Made 'groupby' filter case insensitive by default, like the 'sort' and 'dictsort' filter. --- jinja2/filters.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/jinja2/filters.py b/jinja2/filters.py index a03f9f03c..a06fcd8a1 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -783,7 +783,7 @@ def do_round(value, precision=0, method='common'): @environmentfilter -def do_groupby(environment, value, attribute): +def do_groupby(environment, value, attribute, case_sensitive=False): """Group a sequence of objects by a common attribute. If you for example have a list of dicts or objects that represent persons @@ -821,8 +821,14 @@ def do_groupby(environment, value, attribute): It's now possible to use dotted notation to group by the child attribute of another attribute. """ - expr = make_attrgetter(environment, attribute) - return map(_GroupTuple, groupby(sorted(value, key=expr), expr)) + getter = make_attrgetter(environment, attribute) + expr = make_sort_func(getter, case_sensitive) + + rv = [] + for _, list_ in groupby(sorted(value, key=expr), expr): + list_ = list(list_) + rv.append(_GroupTuple(getter(list_[0]), list_)) + return rv class _GroupTuple(tuple): @@ -830,8 +836,8 @@ class _GroupTuple(tuple): grouper = property(itemgetter(0)) list = property(itemgetter(1)) - def __new__(cls, (key, value)): - return tuple.__new__(cls, (key, list(value))) + def __new__(cls, key, list_): + return tuple.__new__(cls, (key, list_)) @environmentfilter From b8be8c9ce318f30b7ab3f69a8e0a0b7c6b6e24ea Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Thu, 27 Oct 2011 12:38:26 +0200 Subject: [PATCH 8/9] Refactored 'filter' filter and added 'map' filter. * Keyword arguments are passed in addition to positional arguments to the test function, now. * The 'invert' keyword argument was removed in favor of a new filter called 'filterfalse'. * The new filter 'map' was added. It shares most of its code with the 'filter' and 'filterfalse' filters, but expects a filter instead of test and calls map() instead of filter(). --- jinja2/filters.py | 108 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/jinja2/filters.py b/jinja2/filters.py index a06fcd8a1..2d5c9b23f 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -11,7 +11,7 @@ import re import math from random import choice -from operator import itemgetter +from operator import itemgetter, not_ from itertools import imap, groupby from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, min, max from jinja2.runtime import Undefined @@ -79,6 +79,39 @@ def sort_func(item): return sort_func +def make_map_and_filter_func(environment, filters_or_tests, *args, **kwargs): + """Parses arguments of filters like `map` and `filter` and returns a + callable usable for Python's map() and filter() built-in. Those filters + can be called in following ways: + + * With the name of a filter (in the case of `map`) or a test (in the case + of `filter` and `filterfalse`) as first positional argument. + All additional arguments are passed to the filter/test. + * With the keyword argument `attribute`. So an attribute lookup is performed + on the items in the sequence, instead of a filter or test. + * With no arguments at all. In that case this function returns None, so when + passed to the filter() built-in for exmpale, only items that evaluate to + True are returned. + """ + if args: + name, args = args[0], args[1:] + + try: + func = filters_or_tests[name] + except KeyError: + raise FilterArgumentError("there is no filter/test '%s'" % name) + + return lambda x: func(x, *args, **kwargs) + + attribute = kwargs.pop('attribute', None) + + if kwargs: + raise FilterArgumentError('got an unexpected keyword argument') + + if attribute is not None: + return make_attrgetter(environment, attribute) + + def do_forceescape(value): """Enforce HTML escaping. This will probably double escape variables.""" if hasattr(value, '__html__'): @@ -701,51 +734,80 @@ def do_batch(value, linecount, fill_with=None): @environmentfilter -def do_filter(environment, iterable, test=None, *args, **kwargs): +def do_map(environment, iterable, *args, **kwargs): + """Map an iterable using filters or attribute lookups, returning a list + that holds the values processed by the given filter or provided by the + given attribute. + + .. sourcecode:: jinja + + {{ ['foo', 'bar']|map('upper') }} + -> ['FOO','BAR'] + + It is also possible to map the items to a certain attribute: + + .. sourcecode:: jinja + + {{ users|map(attribute='full_name') }} + """ + func = make_map_and_filter_func(environment, environment.filters, *args, **kwargs) + return map(func, iterable) + - """Filter an iterable using tests and attribute lookups. +@environmentfilter +def do_filter(environment, iterable, *args, **kwargs): + """Filter an iterable using tests or attribute lookups, returning only + items where the tests passes or the given attribute evaluates to True. .. sourcecode:: jinja {{ range(10)|filter('even') }} -> [0,2,4,6,8] - If you don't specify a ``'test'``, all values that evaluate to False (e.g. - none, false, '', 0) are excluded: + If you don't specify a ``'test'``, all items that evaluate to False + (e.g. none, false, '', 0) are excluded: .. sourcecode:: jinja {{ [none,true,false,'foo','',42,0]|filter }} -> [true,'foo',42] - It is also possible to filter by a certain attribute: + It is also possible to filter the items by a certain attribute: .. sourcecode:: jinja {{ users|filter(attribute='is_staff') }} + """ + func = make_map_and_filter_func(environment, environment.tests, *args, **kwargs) + return filter(func, iterable) + - And you can also invert the the filter condition: +@environmentfilter +def do_filterfalse(environment, iterable, *args, **kwargs): + """Filter an iterable using tests or attribute lookups, returning only + items where the tests fails or the given attribute evalutes to False. .. sourcecode:: jinja - {{ [none,true,false,'foo','',42,0]|filter(invert=false) }} - -> [none, false, '', 0] - """ - if test is not None: - try: - test = environment.tests[test] - except KeyError: - raise FilterArgumentError("there is no test '%s'" % test) - else: - test = bool + {{ range(10)|filterfalse('even') }} + -> [1,3,5,7,9] - attribute = kwargs.pop('attribute', None) - invert = kwargs.pop('invert', False) + If you don't specify a ``'test'``, only items that evaluate to False + (e.g. none, false, '', 0) are included: - if attribute is not None: - iterable = imap(make_attrgetter(environment, attribute), iterable) + .. sourcecode:: jinja + + {{ [none,true,false,'foo','',42,0]|filterfalse }} + -> [none,false,'',0] - return [x for x in iterable if test(x, *args) != invert] + It is also possible to filter the items by a certain attribute: + + .. sourcecode:: jinja + + {{ users|filterfalse(attribute='is_staff') }} + """ + func = make_map_and_filter_func(environment, environment.tests, *args, **kwargs) + return filter(func and (lambda x: not func(x)) or not_, iterable) def do_round(value, precision=0, method='common'): @@ -965,7 +1027,9 @@ def do_attr(environment, obj, name): 'striptags': do_striptags, 'slice': do_slice, 'batch': do_batch, + 'map': do_map, 'filter': do_filter, + 'filterfalse': do_filterfalse, 'sum': do_sum, 'abs': abs, 'round': do_round, From a0ef3a111fb1bef48704d88ea92f037c50737abb Mon Sep 17 00:00:00 2001 From: Sebastian Noack Date: Thu, 27 Oct 2011 14:06:11 +0200 Subject: [PATCH 9/9] Added 'strict' keyword argument to Environment.getitem and Environment.getattr. That way we don't need to repeat the logic in the `attr` filter. Also it simplifies the implementation of the derived SandboxedEnvironment. --- jinja2/environment.py | 17 ++++++----------- jinja2/filters.py | 16 +--------------- jinja2/sandbox.py | 38 +++++++------------------------------- 3 files changed, 14 insertions(+), 57 deletions(-) diff --git a/jinja2/environment.py b/jinja2/environment.py index ebb54548e..3941af745 100644 --- a/jinja2/environment.py +++ b/jinja2/environment.py @@ -347,34 +347,29 @@ def iter_extensions(self): return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) - def getitem(self, obj, argument): + def getitem(self, obj, argument, strict=False): """Get an item or attribute of an object but prefer the item.""" try: return obj[argument] except (TypeError, LookupError): - if isinstance(argument, basestring): + if not strict and isinstance(argument, basestring): try: attr = str(argument) except Exception: pass else: - try: - return getattr(obj, attr) - except AttributeError: - pass + return self.getattr(obj, attr, strict=True) return self.undefined(obj=obj, name=argument) - def getattr(self, obj, attribute): + def getattr(self, obj, attribute, strict=False): """Get an item or attribute of an object but prefer the attribute. Unlike :meth:`getitem` the attribute *must* be a bytestring. """ try: return getattr(obj, attribute) except AttributeError: - pass - try: - return obj[attribute] - except (TypeError, LookupError, AttributeError): + if not strict: + return self.getitem(obj, attribute, strict=True) return self.undefined(obj=obj, name=attribute) @internalcode diff --git a/jinja2/filters.py b/jinja2/filters.py index 2d5c9b23f..04eff2efd 100644 --- a/jinja2/filters.py +++ b/jinja2/filters.py @@ -967,21 +967,7 @@ def do_attr(environment, obj, name): See :ref:`Notes on subscriptions ` for more details. """ - try: - name = str(name) - except UnicodeError: - pass - else: - try: - value = getattr(obj, name) - except AttributeError: - pass - else: - if environment.sandboxed and not \ - environment.is_safe_attribute(obj, name, value): - return environment.unsafe_undefined(obj, name) - return value - return environment.undefined(obj=obj, name=name) + return environment.getattr(obj, name, strict=True) FILTERS = { diff --git a/jinja2/sandbox.py b/jinja2/sandbox.py index a1cbb29ac..1896123ba 100644 --- a/jinja2/sandbox.py +++ b/jinja2/sandbox.py @@ -294,43 +294,19 @@ def call_unop(self, context, operator, arg): """ return self.unop_table[operator](arg) - def getitem(self, obj, argument): - """Subscribe an object from sandboxed code.""" - try: - return obj[argument] - except (TypeError, LookupError): - if isinstance(argument, basestring): - try: - attr = str(argument) - except Exception: - pass - else: - try: - value = getattr(obj, attr) - except AttributeError: - pass - else: - if self.is_safe_attribute(obj, argument, value): - return value - return self.unsafe_undefined(obj, argument) - return self.undefined(obj=obj, name=argument) - - def getattr(self, obj, attribute): + def getattr(self, obj, attribute, strict=False): """Subscribe an object from sandboxed code and prefer the attribute. The attribute passed *must* be a bytestring. """ try: value = getattr(obj, attribute) except AttributeError: - try: - return obj[attribute] - except (TypeError, LookupError): - pass - else: - if self.is_safe_attribute(obj, attribute, value): - return value - return self.unsafe_undefined(obj, attribute) - return self.undefined(obj=obj, name=attribute) + if not strict: + return self.getitem(obj, attribute, strict=True) + return self.undefined(obj=obj, name=attribute) + if self.is_safe_attribute(obj, attribute, value): + return value + return self.unsafe_undefined(obj, attribute) def unsafe_undefined(self, obj, attribute): """Return an undefined object for unsafe attributes."""