From 96d9bfec0058b7c91709822d314679bc1660368a Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 17 Sep 2022 14:53:03 +0200 Subject: [PATCH] Added support for slicing --- minijinja/src/compiler/ast.rs | 11 +++ minijinja/src/compiler/codegen.rs | 21 ++++++ minijinja/src/compiler/instructions.rs | 3 + minijinja/src/compiler/parser.rs | 55 ++++++++++++--- minijinja/src/syntax.rs | 2 + minijinja/src/value/ops.rs | 69 ++++++++++++++++++- minijinja/src/vm/mod.rs | 7 ++ minijinja/tests/inputs/slicing.txt | 17 +++++ .../test_templates__vm@slicing.txt.snap | 31 +++++++++ 9 files changed, 207 insertions(+), 9 deletions(-) create mode 100644 minijinja/tests/inputs/slicing.txt create mode 100644 minijinja/tests/snapshots/test_templates__vm@slicing.txt.snap diff --git a/minijinja/src/compiler/ast.rs b/minijinja/src/compiler/ast.rs index af7ff3ea..f84c5677 100644 --- a/minijinja/src/compiler/ast.rs +++ b/minijinja/src/compiler/ast.rs @@ -91,6 +91,7 @@ impl<'a> fmt::Debug for Stmt<'a> { pub enum Expr<'a> { Var(Spanned>), Const(Spanned), + Slice(Spanned>), UnaryOp(Spanned>), BinOp(Spanned>), IfExpr(Spanned>), @@ -109,6 +110,7 @@ impl<'a> fmt::Debug for Expr<'a> { match self { Expr::Var(s) => fmt::Debug::fmt(s, f), Expr::Const(s) => fmt::Debug::fmt(s, f), + Expr::Slice(s) => fmt::Debug::fmt(s, f), Expr::UnaryOp(s) => fmt::Debug::fmt(s, f), Expr::BinOp(s) => fmt::Debug::fmt(s, f), Expr::IfExpr(s) => fmt::Debug::fmt(s, f), @@ -228,6 +230,15 @@ pub struct Const { pub value: Value, } +/// Represents a slice. +#[cfg_attr(feature = "internal_debug", derive(Debug))] +pub struct Slice<'a> { + pub expr: Expr<'a>, + pub start: Option>, + pub stop: Option>, + pub step: Option>, +} + /// A kind of unary operator. #[cfg_attr(feature = "internal_debug", derive(Debug))] pub enum UnaryOpKind { diff --git a/minijinja/src/compiler/codegen.rs b/minijinja/src/compiler/codegen.rs index e54f245e..76451a41 100644 --- a/minijinja/src/compiler/codegen.rs +++ b/minijinja/src/compiler/codegen.rs @@ -402,6 +402,27 @@ impl<'source> CodeGenerator<'source> { self.set_line_from_span(v.span()); self.add(Instruction::LoadConst(v.value.clone())); } + ast::Expr::Slice(s) => { + self.push_span(s.span()); + self.compile_expr(&s.expr)?; + if let Some(ref start) = s.start { + self.compile_expr(start)?; + } else { + self.add(Instruction::LoadConst(Value::from(0))); + } + if let Some(ref stop) = s.stop { + self.compile_expr(stop)?; + } else { + self.add(Instruction::LoadConst(Value::from(()))); + } + if let Some(ref step) = s.step { + self.compile_expr(step)?; + } else { + self.add(Instruction::LoadConst(Value::from(1))); + } + self.add(Instruction::Slice); + self.pop_span(); + } ast::Expr::UnaryOp(c) => { self.set_line_from_span(c.span()); self.compile_expr(&c.expr)?; diff --git a/minijinja/src/compiler/instructions.rs b/minijinja/src/compiler/instructions.rs index 9c5df19e..ab4fd831 100644 --- a/minijinja/src/compiler/instructions.rs +++ b/minijinja/src/compiler/instructions.rs @@ -31,6 +31,9 @@ pub enum Instruction<'source> { /// Looks up an item. GetItem, + /// Performs a slice operation. + Slice, + /// Loads a constant value. LoadConst(Value), diff --git a/minijinja/src/compiler/parser.rs b/minijinja/src/compiler/parser.rs index 87e4bdcd..bb634ed6 100644 --- a/minijinja/src/compiler/parser.rs +++ b/minijinja/src/compiler/parser.rs @@ -315,15 +315,54 @@ impl<'a> Parser<'a> { } Some((Token::BracketOpen, _)) => { self.stream.next()?; - let subscript_expr = self.parse_expr()?; + + let mut start = None; + let mut stop = None; + let mut step = None; + let mut is_slice = false; + + if !matches!(self.stream.current()?, Some((Token::Colon, _))) { + start = Some(self.parse_expr()?); + } + if matches!(self.stream.current()?, Some((Token::Colon, _))) { + is_slice = true; + self.stream.next()?; + if !matches!( + self.stream.current()?, + Some((Token::BracketClose | Token::Colon, _)) + ) { + stop = Some(self.parse_expr()?); + } + if matches!(self.stream.current()?, Some((Token::Colon, _))) { + self.stream.next()?; + if !matches!(self.stream.current()?, Some((Token::BracketClose, _))) { + step = Some(self.parse_expr()?); + } + } + } expect_token!(self, Token::BracketClose, "`]`")?; - expr = ast::Expr::GetItem(Spanned::new( - ast::GetItem { - expr, - subscript_expr, - }, - self.stream.expand_span(span), - )); + + if !is_slice { + expr = ast::Expr::GetItem(Spanned::new( + ast::GetItem { + expr, + subscript_expr: start.ok_or_else(|| { + Error::new(ErrorKind::SyntaxError, "empty subscript") + })?, + }, + self.stream.expand_span(span), + )); + } else { + expr = ast::Expr::Slice(Spanned::new( + ast::Slice { + expr, + start, + stop, + step, + }, + self.stream.expand_span(span), + )); + } } Some((Token::ParenOpen, _)) => { let args = self.parse_args()?; diff --git a/minijinja/src/syntax.rs b/minijinja/src/syntax.rs index 3796f8a5..fac5dc22 100644 --- a/minijinja/src/syntax.rs +++ b/minijinja/src/syntax.rs @@ -136,6 +136,8 @@ //! which are treated like a dict syntax. Eg: `foo(a=1, b=2)` is the same as //! `foo({"a": 1, "b": 2})`. //! - ``.`` / ``[]``: Get an attribute of an object. +//! - ``[start:stop]`` / ``[start:stop:step]``: slices a list or string. All three expressions +//! are optional (`start`, `stop`, `step`). //! //! ### If Expressions //! diff --git a/minijinja/src/value/ops.rs b/minijinja/src/value/ops.rs index db4bf8c0..c7e74446 100644 --- a/minijinja/src/value/ops.rs +++ b/minijinja/src/value/ops.rs @@ -1,4 +1,4 @@ -use std::convert::TryFrom; +use std::convert::{TryFrom, TryInto}; use std::fmt::Write; use crate::error::{Error, ErrorKind}; @@ -48,6 +48,73 @@ pub fn coerce(a: &Value, b: &Value) -> Option { } } +fn get_offset_and_len usize>( + start: i64, + stop: Option, + end: F, +) -> (usize, usize) { + if start < 0 || stop.map_or(true, |x| x < 0) { + let end = end(); + let start = if start < 0 { + (end as i64 + start) as usize + } else { + start as usize + }; + let stop = match stop { + None => end, + Some(x) if x < 0 => (end as i64 + x) as usize, + Some(x) => x as usize, + }; + (start, stop.saturating_sub(start)) + } else { + ( + start as usize, + (stop.unwrap() as usize).saturating_sub(start as usize), + ) + } +} + +pub fn slice(value: Value, start: Value, stop: Value, step: Value) -> Result { + let start: i64 = if start.is_none() { + 0 + } else { + start.try_into()? + }; + let stop: Option = if stop.is_none() { + None + } else { + Some(stop.try_into()?) + }; + let step = if step.is_none() { + 1 + } else { + u64::try_from(step)? as usize + }; + + if let Some(s) = value.as_str() { + let (start, len) = get_offset_and_len(start, stop, || s.chars().count()); + return Ok(Value::from( + s.chars() + .skip(start) + .take(len) + .step_by(step) + .collect::(), + )); + } + + let slice = value.as_slice()?; + let (start, len) = get_offset_and_len(start, stop, || slice.len()); + Ok(Value::from( + slice + .iter() + .skip(start) + .take(len) + .step_by(step) + .cloned() + .collect::>(), + )) +} + fn int_as_value(val: i128) -> Value { if val as i64 as i128 == val { (val as i64).into() diff --git a/minijinja/src/vm/mod.rs b/minijinja/src/vm/mod.rs index ee269ae3..1c4a25e2 100644 --- a/minijinja/src/vm/mod.rs +++ b/minijinja/src/vm/mod.rs @@ -146,6 +146,13 @@ impl<'env> Vm<'env> { let value = stack.pop(); stack.push(try_ctx!(value.get_item(&attr))); } + Instruction::Slice => { + let step = stack.pop(); + let stop = stack.pop(); + let start = stack.pop(); + let value = stack.pop(); + stack.push(try_ctx!(ops::slice(value, start, stop, step))); + } Instruction::LoadConst(value) => { stack.push(value.clone()); } diff --git a/minijinja/tests/inputs/slicing.txt b/minijinja/tests/inputs/slicing.txt new file mode 100644 index 00000000..8f18dc35 --- /dev/null +++ b/minijinja/tests/inputs/slicing.txt @@ -0,0 +1,17 @@ +{ + "hello": "Hello World", + "intrange": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +} +--- +{{ hello[:] }} +{{ hello[1:] }} +{{ hello[1:-1] }} +{{ hello[::2] }} +{{ hello[2:10] }} +{{ hello[2:10:2] }} +{{ intrange[:] }} +{{ intrange[1:] }} +{{ intrange[1:-1] }} +{{ intrange[::2] }} +{{ intrange[2:10] }} +{{ intrange[2:10:2] }} diff --git a/minijinja/tests/snapshots/test_templates__vm@slicing.txt.snap b/minijinja/tests/snapshots/test_templates__vm@slicing.txt.snap new file mode 100644 index 00000000..f9a3534b --- /dev/null +++ b/minijinja/tests/snapshots/test_templates__vm@slicing.txt.snap @@ -0,0 +1,31 @@ +--- +source: minijinja/tests/test_templates.rs +description: "{{ hello[:] }}\n{{ hello[1:] }}\n{{ hello[1:-1] }}\n{{ hello[::2] }}\n{{ hello[2:10] }}\n{{ hello[2:10:2] }}\n{{ intrange[:] }}\n{{ intrange[1:] }}\n{{ intrange[1:-1] }}\n{{ intrange[::2] }}\n{{ intrange[2:10] }}\n{{ intrange[2:10:2] }}" +info: + hello: Hello World + intrange: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 +input_file: minijinja/tests/inputs/slicing.txt +--- +Hello World +ello World +ello Worl +HloWrd +llo Worl +loWr +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +[1, 2, 3, 4, 5, 6, 7, 8, 9] +[1, 2, 3, 4, 5, 6, 7, 8] +[0, 2, 4, 6, 8] +[2, 3, 4, 5, 6, 7, 8, 9] +[2, 4, 6, 8] +