From 3ded060d478176937092681f4681073078417a4b Mon Sep 17 00:00:00 2001
From: lovasoa <pere.jobs@gmail.com>
Date: Mon, 5 Sep 2022 21:49:55 +0200
Subject: [PATCH] Fix parse error on some prepared statement placeholders

sqlparser can now parse all the prepared statement placeholders supported by SQLite:

 - ?
 - ?NNN
 - @VVV
 - :VVV
 - $VVV

See: https://www.sqlite.org/lang_expr.html#varparam

This does not break existing support for postgresql's '@' operator

Fixes #603
---
 src/parser.rs             | 13 +++++++++----
 src/tokenizer.rs          | 11 ++++++-----
 tests/sqlparser_common.rs | 12 ++++++++++++
 tests/sqlparser_sqlite.rs | 21 ++++++++++++++++++---
 4 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/src/parser.rs b/src/parser.rs
index 877c47303..822cabc95 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -570,7 +570,7 @@ impl<'a> Parser<'a> {
                     })
                 }
             }
-            Token::Placeholder(_) => {
+            Token::Placeholder(_) | Token::Colon | Token::AtSign => {
                 self.prev_token();
                 Ok(Expr::Value(self.parse_value()?))
             }
@@ -1774,7 +1774,7 @@ impl<'a> Parser<'a> {
                             .iter()
                             .any(|d| kw.keyword == *d) =>
                     {
-                        break
+                        break;
                     }
                     Token::RParen | Token::EOF => break,
                     _ => continue,
@@ -3026,6 +3026,11 @@ impl<'a> Parser<'a> {
             Token::EscapedStringLiteral(ref s) => Ok(Value::EscapedStringLiteral(s.to_string())),
             Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
             Token::Placeholder(ref s) => Ok(Value::Placeholder(s.to_string())),
+            tok @ Token::Colon | tok @ Token::AtSign => {
+                let ident = self.parse_identifier()?;
+                let placeholder = tok.to_string() + &ident.value;
+                Ok(Value::Placeholder(placeholder))
+            }
             unexpected => self.expected("a value", unexpected),
         }
     }
@@ -4844,12 +4849,12 @@ impl<'a> Parser<'a> {
                     Some(_) => {
                         return Err(ParserError::ParserError(
                             "expected UPDATE, DELETE or INSERT in merge clause".to_string(),
-                        ))
+                        ));
                     }
                     None => {
                         return Err(ParserError::ParserError(
                             "expected UPDATE, DELETE or INSERT in merge clause".to_string(),
-                        ))
+                        ));
                     }
                 },
             );
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index fdd066f61..cda5140e9 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -677,13 +677,14 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 '@' => self.consume_and_return(chars, Token::AtSign),
-                '?' => self.consume_and_return(chars, Token::Placeholder(String::from("?"))),
+                '?' => {
+                    chars.next();
+                    let s = peeking_take_while(chars, |ch| ch.is_numeric());
+                    Ok(Some(Token::Placeholder(String::from("?") + &s)))
+                }
                 '$' => {
                     chars.next();
-                    let s = peeking_take_while(
-                        chars,
-                        |ch| matches!(ch, '0'..='9' | 'A'..='Z' | 'a'..='z'),
-                    );
+                    let s = peeking_take_while(chars, |ch| ch.is_alphanumeric() || ch == '_');
                     Ok(Some(Token::Placeholder(String::from("$") + &s)))
                 }
                 //whitespace check (including unicode chars) should be last as it covers some of the chars above
diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs
index 6fee5e88b..59f9623fd 100644
--- a/tests/sqlparser_common.rs
+++ b/tests/sqlparser_common.rs
@@ -22,6 +22,7 @@
 mod test_utils;
 
 use matches::assert_matches;
+use sqlparser::ast::SelectItem::UnnamedExpr;
 use sqlparser::ast::*;
 use sqlparser::dialect::{
     AnsiDialect, BigQueryDialect, ClickHouseDialect, GenericDialect, HiveDialect, MsSqlDialect,
@@ -5213,6 +5214,17 @@ fn test_placeholder() {
             rows: OffsetRows::None,
         }),
     );
+
+    let sql = "SELECT $fromage_français, :x, ?123";
+    let ast = dialects.verified_only_select(sql);
+    assert_eq!(
+        ast.projection,
+        vec![
+            UnnamedExpr(Expr::Value(Value::Placeholder("$fromage_français".into()))),
+            UnnamedExpr(Expr::Value(Value::Placeholder(":x".into()))),
+            UnnamedExpr(Expr::Value(Value::Placeholder("?123".into()))),
+        ]
+    );
 }
 
 #[test]
diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs
index 61436cb51..3f60e16d1 100644
--- a/tests/sqlparser_sqlite.rs
+++ b/tests/sqlparser_sqlite.rs
@@ -16,8 +16,10 @@
 
 #[macro_use]
 mod test_utils;
+
 use test_utils::*;
 
+use sqlparser::ast::SelectItem::UnnamedExpr;
 use sqlparser::ast::*;
 use sqlparser::dialect::{GenericDialect, SQLiteDialect};
 use sqlparser::tokenizer::Token;
@@ -73,14 +75,14 @@ fn parse_create_table_auto_increment() {
                     options: vec![
                         ColumnOptionDef {
                             name: None,
-                            option: ColumnOption::Unique { is_primary: true }
+                            option: ColumnOption::Unique { is_primary: true },
                         },
                         ColumnOptionDef {
                             name: None,
                             option: ColumnOption::DialectSpecific(vec![Token::make_keyword(
                                 "AUTOINCREMENT"
-                            )])
-                        }
+                            )]),
+                        },
                     ],
                 }],
                 columns
@@ -118,6 +120,19 @@ fn parse_create_sqlite_quote() {
     }
 }
 
+#[test]
+fn test_placeholder() {
+    // In postgres, this would be the absolute value operator '@' applied to the column 'xxx'
+    // But in sqlite, this is a named parameter.
+    // see https://www.sqlite.org/lang_expr.html#varparam
+    let sql = "SELECT @xxx";
+    let ast = sqlite().verified_only_select(sql);
+    assert_eq!(
+        ast.projection[0],
+        UnnamedExpr(Expr::Value(Value::Placeholder("@xxx".into()))),
+    );
+}
+
 fn sqlite() -> TestedDialects {
     TestedDialects {
         dialects: vec![Box::new(SQLiteDialect {})],