From 7e7471e1b8588ef52a34b672df16436bbb3a455a Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Thu, 27 Oct 2022 14:44:27 +0300 Subject: [PATCH 1/8] Support parse json in snowflake --- src/parser.rs | 7 +++++++ tests/sqlparser_common.rs | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/src/parser.rs b/src/parser.rs index 108427889..a9512c328 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1426,6 +1426,12 @@ impl<'a> Parser<'a> { return self.parse_array_index(expr); } self.parse_map_access(expr) + } else if Token::Colon == tok { + Ok(Expr::JsonAccess { + left: Box::new(expr), + operator: JsonOperator::LongArrow, + right: Box::new(self.parse_subexpr(51)?), + }) } else if Token::Arrow == tok || Token::LongArrow == tok || Token::HashArrow == tok @@ -1627,6 +1633,7 @@ impl<'a> Parser<'a> { Token::Plus | Token::Minus => Ok(Self::PLUS_MINUS_PREC), Token::Mul | Token::Div | Token::Mod | Token::StringConcat => Ok(40), Token::DoubleColon => Ok(50), + Token::Colon => Ok(50), Token::ExclamationMark => Ok(50), Token::LBracket | Token::LongArrow diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index bae310ef0..3608cb5c4 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1594,6 +1594,15 @@ fn parse_limit_accepts_all() { ); } +#[test] +fn parse_json_using_colon() { + one_statement_parses_to("SELECT field:key FROM t", "SELECT field ->> key FROM t"); + one_statement_parses_to( + "SELECT field:key::int FROM t", + "SELECT CAST(field ->> key AS INT) FROM t", + ); +} + #[test] fn parse_cast() { let sql = "SELECT CAST(id AS BIGINT) FROM customer"; From 2b43f3054b095a6b0060d78b39d8151987ce3d40 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Sun, 30 Oct 2022 11:26:55 +0200 Subject: [PATCH 2/8] MR Review --- src/ast/mod.rs | 12 +++++++++++- src/parser.rs | 2 +- tests/sqlparser_common.rs | 9 --------- tests/sqlparser_snowflake.rs | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index c83ead544..62b647b19 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -187,6 +187,8 @@ pub enum JsonOperator { HashArrow, /// #>> Extracts JSON sub-object at the specified path as text HashLongArrow, + /// : Colon is used by Snowflake (Which is similar to LongArrow) + Colon, } impl fmt::Display for JsonOperator { @@ -204,6 +206,9 @@ impl fmt::Display for JsonOperator { JsonOperator::HashLongArrow => { write!(f, "#>>") } + JsonOperator::Colon => { + write!(f, ":") + } } } } @@ -757,7 +762,12 @@ impl fmt::Display for Expr { operator, right, } => { - write!(f, "{} {} {}", left, operator, right) + if operator == &JsonOperator::Colon { + write!(f, "{}{}{}", left, operator, right) + } else { + write!(f, "{} {} {}", left, operator, right) + } + } Expr::CompositeAccess { expr, key } => { write!(f, "{}.{}", expr, key) diff --git a/src/parser.rs b/src/parser.rs index a9512c328..546827472 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1429,7 +1429,7 @@ impl<'a> Parser<'a> { } else if Token::Colon == tok { Ok(Expr::JsonAccess { left: Box::new(expr), - operator: JsonOperator::LongArrow, + operator: JsonOperator::Colon, right: Box::new(self.parse_subexpr(51)?), }) } else if Token::Arrow == tok diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 3608cb5c4..bae310ef0 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1594,15 +1594,6 @@ fn parse_limit_accepts_all() { ); } -#[test] -fn parse_json_using_colon() { - one_statement_parses_to("SELECT field:key FROM t", "SELECT field ->> key FROM t"); - one_statement_parses_to( - "SELECT field:key::int FROM t", - "SELECT CAST(field ->> key AS INT) FROM t", - ); -} - #[test] fn parse_cast() { let sql = "SELECT CAST(id AS BIGINT) FROM customer"; diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 7c089a935..8f0c6f462 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -143,6 +143,25 @@ fn test_single_table_in_parenthesis_with_alias() { ); } +#[test] +fn parse_json_using_colon() { + let sql = "SELECT field:key FROM t"; + let select = snowflake().verified_only_select(sql); + assert_eq!( + SelectItem::UnnamedExpr(Expr::JsonAccess { + left: Box::new(Expr::Identifier(Ident::new("field"))), + operator: JsonOperator::Colon, + right: Box::new(Expr::Identifier(Ident::new("key"))), + }), + select.projection[0] + ); + + snowflake().one_statement_parses_to( + "SELECT field:key::int FROM t", + "SELECT CAST(field:key AS INT) FROM t", + ); +} + fn snowflake() -> TestedDialects { TestedDialects { dialects: vec![Box::new(SnowflakeDialect {})], From 4c1629d90c50beeeddad2f9eeb97d340ee3aff5c Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Sun, 30 Oct 2022 11:29:14 +0200 Subject: [PATCH 3/8] Lint --- src/ast/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index 62b647b19..cda0d4e29 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -762,12 +762,11 @@ impl fmt::Display for Expr { operator, right, } => { - if operator == &JsonOperator::Colon { + if operator == &JsonOperator::Colon { write!(f, "{}{}{}", left, operator, right) } else { write!(f, "{} {} {}", left, operator, right) } - } Expr::CompositeAccess { expr, key } => { write!(f, "{}.{}", expr, key) From 498519a0825754f5966b51ab14cfcffcd7929686 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Mon, 31 Oct 2022 16:37:34 +0200 Subject: [PATCH 4/8] Try to fix right as value --- src/ast/value.rs | 3 +++ src/parser.rs | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ast/value.rs b/src/ast/value.rs index 3861ab008..2162dd219 100644 --- a/src/ast/value.rs +++ b/src/ast/value.rs @@ -45,6 +45,8 @@ pub enum Value { Null, /// `?` or `$` Prepared statement arg placeholder Placeholder(String), + /// Add support of snowflake field:key - key should be a value + UnQuotedString(String), } impl fmt::Display for Value { @@ -59,6 +61,7 @@ impl fmt::Display for Value { Value::Boolean(v) => write!(f, "{}", v), Value::Null => write!(f, "NULL"), Value::Placeholder(v) => write!(f, "{}", v), + Value::UnQuotedString(v) => write!(f, "{}", v), } } } diff --git a/src/parser.rs b/src/parser.rs index 546827472..c3c736df8 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1430,7 +1430,7 @@ impl<'a> Parser<'a> { Ok(Expr::JsonAccess { left: Box::new(expr), operator: JsonOperator::Colon, - right: Box::new(self.parse_subexpr(51)?), + right: Box::new(Expr::Value(self.parse_value()?)), }) } else if Token::Arrow == tok || Token::LongArrow == tok @@ -3448,6 +3448,9 @@ impl<'a> Parser<'a> { Some('\'') => Ok(Value::SingleQuotedString(w.value)), _ => self.expected("A value?", Token::Word(w))?, }, + Keyword::NoKeyword if dialect_of!(self is SnowflakeDialect) => { + Ok(Value::UnQuotedString(w.value)) + } _ => self.expected("a concrete value", Token::Word(w)), }, // The call to n.parse() returns a bigdecimal when the From 283aa0a7648ed345f6fdb931529019d3792c53b7 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Mon, 31 Oct 2022 23:44:28 +0200 Subject: [PATCH 5/8] Fix tests --- tests/sqlparser_snowflake.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 8f0c6f462..3989b87cc 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -145,20 +145,20 @@ fn test_single_table_in_parenthesis_with_alias() { #[test] fn parse_json_using_colon() { - let sql = "SELECT field:key FROM t"; + let sql = "SELECT a:b FROM t"; let select = snowflake().verified_only_select(sql); assert_eq!( SelectItem::UnnamedExpr(Expr::JsonAccess { - left: Box::new(Expr::Identifier(Ident::new("field"))), + left: Box::new(Expr::Identifier(Ident::new("a"))), operator: JsonOperator::Colon, - right: Box::new(Expr::Identifier(Ident::new("key"))), + right: Box::new(Expr::Value(Value::UnQuotedString("b".to_string()))), }), select.projection[0] ); snowflake().one_statement_parses_to( - "SELECT field:key::int FROM t", - "SELECT CAST(field:key AS INT) FROM t", + "SELECT a:b::int FROM t", + "SELECT CAST(a:b AS INT) FROM t", ); } From 7dde4b716ba267ef32513a069373068e59cade40 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Mon, 31 Oct 2022 23:47:34 +0200 Subject: [PATCH 6/8] Fix lint --- tests/sqlparser_snowflake.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 3989b87cc..64fff62f9 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -156,10 +156,7 @@ fn parse_json_using_colon() { select.projection[0] ); - snowflake().one_statement_parses_to( - "SELECT a:b::int FROM t", - "SELECT CAST(a:b AS INT) FROM t", - ); + snowflake().one_statement_parses_to("SELECT a:b::int FROM t", "SELECT CAST(a:b AS INT) FROM t"); } fn snowflake() -> TestedDialects { From 3598734b1569549abf8f076b198ff2cee1f334fa Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Tue, 1 Nov 2022 10:31:03 +0200 Subject: [PATCH 7/8] Add generic dialect --- src/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser.rs b/src/parser.rs index c3c736df8..8f629b7fd 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3448,7 +3448,7 @@ impl<'a> Parser<'a> { Some('\'') => Ok(Value::SingleQuotedString(w.value)), _ => self.expected("A value?", Token::Word(w))?, }, - Keyword::NoKeyword if dialect_of!(self is SnowflakeDialect) => { + Keyword::NoKeyword if dialect_of!(self is SnowflakeDialect | GenericDialect) => { Ok(Value::UnQuotedString(w.value)) } _ => self.expected("a concrete value", Token::Word(w)), From 00d27d04a0164b4740cd56e346329526dbeaa479 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 2 Nov 2022 10:27:32 +0200 Subject: [PATCH 8/8] Add support in key location --- src/parser.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/parser.rs b/src/parser.rs index 8f629b7fd..5621e67c6 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3448,7 +3448,8 @@ impl<'a> Parser<'a> { Some('\'') => Ok(Value::SingleQuotedString(w.value)), _ => self.expected("A value?", Token::Word(w))?, }, - Keyword::NoKeyword if dialect_of!(self is SnowflakeDialect | GenericDialect) => { + // Case when Snowflake Semi-structured data like key:value + Keyword::NoKeyword | Keyword::LOCATION if dialect_of!(self is SnowflakeDialect | GenericDialect) => { Ok(Value::UnQuotedString(w.value)) } _ => self.expected("a concrete value", Token::Word(w)),