From ffdc75eb5e26d5718599516fec3426f6d2029eff Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 2 Nov 2022 14:42:33 +0200 Subject: [PATCH 1/6] Snowflake Array --- src/ast/data_type.rs | 3 +++ src/parser.rs | 18 +++++++++++------- tests/sqlparser_common.rs | 22 +++++++++++++++------- tests/sqlparser_snowflake.rs | 13 +++++++++++++ 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index a66761167..b030aae8d 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -136,6 +136,8 @@ pub enum DataType { Enum(Vec), /// Set Set(Vec), + // SnowflakeArray (Not Typed, without []) + SnowflakeArray, } impl fmt::Display for DataType { @@ -216,6 +218,7 @@ impl fmt::Display for DataType { DataType::Text => write!(f, "TEXT"), DataType::String => write!(f, "STRING"), DataType::Bytea => write!(f, "BYTEA"), + DataType::SnowflakeArray => write!(f, "ARRAY"), DataType::Array(ty) => write!(f, "{}[]", ty), DataType::Custom(ty, modifiers) => { if modifiers.is_empty() { diff --git a/src/parser.rs b/src/parser.rs index 108427889..e482578f2 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3650,13 +3650,17 @@ impl<'a> Parser<'a> { Keyword::ENUM => Ok(DataType::Enum(self.parse_string_values()?)), Keyword::SET => Ok(DataType::Set(self.parse_string_values()?)), Keyword::ARRAY => { - // Hive array syntax. Note that nesting arrays - or other Hive syntax - // that ends with > will fail due to "C++" problem - >> is parsed as - // Token::ShiftRight - self.expect_token(&Token::Lt)?; - let inside_type = self.parse_data_type()?; - self.expect_token(&Token::Gt)?; - Ok(DataType::Array(Box::new(inside_type))) + if dialect_of!(self is SnowflakeDialect) { + Ok(DataType::SnowflakeArray) + } else { + // Hive array syntax. Note that nesting arrays - or other Hive syntax + // that ends with > will fail due to "C++" problem - >> is parsed as + // Token::ShiftRight + self.expect_token(&Token::Lt)?; + let inside_type = self.parse_data_type()?; + self.expect_token(&Token::Gt)?; + Ok(DataType::Array(Box::new(inside_type))) + } } _ => { self.prev_token(); diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index bae310ef0..fa7efcd39 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -26,7 +26,7 @@ use sqlparser::ast::SelectItem::UnnamedExpr; use sqlparser::ast::*; use sqlparser::dialect::{ AnsiDialect, BigQueryDialect, ClickHouseDialect, GenericDialect, HiveDialect, MsSqlDialect, - PostgreSqlDialect, SQLiteDialect, SnowflakeDialect, + MySqlDialect, PostgreSqlDialect, SQLiteDialect, SnowflakeDialect, }; use sqlparser::keywords::ALL_KEYWORDS; use sqlparser::parser::{Parser, ParserError}; @@ -2123,12 +2123,20 @@ fn parse_create_table_hive_array() { _ => unreachable!(), } - let res = - parse_sql_statements("CREATE TABLE IF NOT EXISTS something (name int, val array, found: )")); + // SnowflakeDialect using array diffrent + let dialects = TestedDialects { + dialects: vec![ + Box::new(PostgreSqlDialect {}), + Box::new(HiveDialect {}), + Box::new(MySqlDialect {}), + ], + }; + let sql = "CREATE TABLE IF NOT EXISTS something (name int, val array, found: )".to_string()) + ); } #[test] diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index 7c089a935..d13d98f40 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -143,6 +143,19 @@ fn test_single_table_in_parenthesis_with_alias() { ); } +#[test] +fn parse_array() { + let sql = "SELECT CAST(a AS ARRAY) FROM customer"; + let select = snowflake().verified_only_select(sql); + assert_eq!( + &Expr::Cast { + expr: Box::new(Expr::Identifier(Ident::new("a"))), + data_type: DataType::SnowflakeArray, + }, + expr_from_projection(only(&select.projection)) + ); +} + fn snowflake() -> TestedDialects { TestedDialects { dialects: vec![Box::new(SnowflakeDialect {})], From c4160d3f85acf892aae6a508c0e8126076313048 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 2 Nov 2022 15:00:47 +0200 Subject: [PATCH 2/6] Use the array data type --- src/ast/data_type.rs | 14 ++++++++++---- src/parser.rs | 5 ++++- tests/sqlparser_snowflake.rs | 5 ++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index b030aae8d..ce72ca9cf 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -136,8 +136,6 @@ pub enum DataType { Enum(Vec), /// Set Set(Vec), - // SnowflakeArray (Not Typed, without []) - SnowflakeArray, } impl fmt::Display for DataType { @@ -218,8 +216,16 @@ impl fmt::Display for DataType { DataType::Text => write!(f, "TEXT"), DataType::String => write!(f, "STRING"), DataType::Bytea => write!(f, "BYTEA"), - DataType::SnowflakeArray => write!(f, "ARRAY"), - DataType::Array(ty) => write!(f, "{}[]", ty), + DataType::Array(ty) => { + if ty.to_string() + == Box::new(DataType::Custom(ObjectName(vec!["VARAINT".into()]), vec![])) + .to_string() + { + write!(f, "ARRAY") + } else { + write!(f, "{}[]", ty) + } + } DataType::Custom(ty, modifiers) => { if modifiers.is_empty() { write!(f, "{}", ty) diff --git a/src/parser.rs b/src/parser.rs index e482578f2..708afe496 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3651,7 +3651,10 @@ impl<'a> Parser<'a> { Keyword::SET => Ok(DataType::Set(self.parse_string_values()?)), Keyword::ARRAY => { if dialect_of!(self is SnowflakeDialect) { - Ok(DataType::SnowflakeArray) + Ok(DataType::Array(Box::new(DataType::Custom( + ObjectName(vec!["VARAINT".into()]), + vec![], + )))) } else { // Hive array syntax. Note that nesting arrays - or other Hive syntax // that ends with > will fail due to "C++" problem - >> is parsed as diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index d13d98f40..a45de8e06 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -150,7 +150,10 @@ fn parse_array() { assert_eq!( &Expr::Cast { expr: Box::new(Expr::Identifier(Ident::new("a"))), - data_type: DataType::SnowflakeArray, + data_type: DataType::Array(Box::new(DataType::Custom( + ObjectName(vec!["VARAINT".into()]), + vec![] + ))), }, expr_from_projection(only(&select.projection)) ); From 677579e1bd1b4da6604eef27827ba7caa15f71f3 Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 2 Nov 2022 18:26:01 +0200 Subject: [PATCH 3/6] Try to fix build --- src/ast/data_type.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index ce72ca9cf..14c7e2bc2 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -217,10 +217,7 @@ impl fmt::Display for DataType { DataType::String => write!(f, "STRING"), DataType::Bytea => write!(f, "BYTEA"), DataType::Array(ty) => { - if ty.to_string() - == Box::new(DataType::Custom(ObjectName(vec!["VARAINT".into()]), vec![])) - .to_string() - { + if ty == &Box::new(DataType::Custom(ObjectName(vec!["VARAINT".into()]), vec![])) { write!(f, "ARRAY") } else { write!(f, "{}[]", ty) From 6cf47253808117d8f5266397527e4854f29a404a Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Wed, 2 Nov 2022 18:36:40 +0200 Subject: [PATCH 4/6] Try to fix build --- src/ast/data_type.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index 30d295bd3..4fcd31158 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -11,7 +11,7 @@ // limitations under the License. #[cfg(not(feature = "std"))] -use alloc::{boxed::Box, string::String, vec::Vec}; +use alloc::{boxed::Box, string::String, vec, vec::Vec}; use core::fmt; #[cfg(feature = "serde")] From 9ac1540afa37e77ee0e570510786e4c6708e58cf Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Thu, 3 Nov 2022 23:43:40 +0200 Subject: [PATCH 5/6] Change Array to option --- src/ast/data_type.rs | 8 ++++---- src/parser.rs | 9 +++------ tests/sqlparser_common.rs | 2 +- tests/sqlparser_postgres.rs | 6 +++--- tests/sqlparser_snowflake.rs | 5 +---- 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index 4fcd31158..ceabe16e2 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -141,7 +141,7 @@ pub enum DataType { /// Custom type such as enums Custom(ObjectName, Vec), /// Arrays - Array(Box), + Array(Option>), /// Enums Enum(Vec), /// Set @@ -233,10 +233,10 @@ impl fmt::Display for DataType { DataType::String => write!(f, "STRING"), DataType::Bytea => write!(f, "BYTEA"), DataType::Array(ty) => { - if ty == &Box::new(DataType::Custom(ObjectName(vec!["VARAINT".into()]), vec![])) { - write!(f, "ARRAY") + if let Some(t) = &ty { + write!(f, "{}[]", t) } else { - write!(f, "{}[]", ty) + write!(f, "ARRAY") } } DataType::Custom(ty, modifiers) => { diff --git a/src/parser.rs b/src/parser.rs index 99fcd705e..f7cadbef1 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3673,10 +3673,7 @@ impl<'a> Parser<'a> { Keyword::SET => Ok(DataType::Set(self.parse_string_values()?)), Keyword::ARRAY => { if dialect_of!(self is SnowflakeDialect) { - Ok(DataType::Array(Box::new(DataType::Custom( - ObjectName(vec!["VARAINT".into()]), - vec![], - )))) + Ok(DataType::Array(None)) } else { // Hive array syntax. Note that nesting arrays - or other Hive syntax // that ends with > will fail due to "C++" problem - >> is parsed as @@ -3684,7 +3681,7 @@ impl<'a> Parser<'a> { self.expect_token(&Token::Lt)?; let inside_type = self.parse_data_type()?; self.expect_token(&Token::Gt)?; - Ok(DataType::Array(Box::new(inside_type))) + Ok(DataType::Array(Some(Box::new(inside_type)))) } } _ => { @@ -3704,7 +3701,7 @@ impl<'a> Parser<'a> { // Keyword::ARRAY syntax from above while self.consume_token(&Token::LBracket) { self.expect_token(&Token::RBracket)?; - data = DataType::Array(Box::new(data)) + data = DataType::Array(Some(Box::new(data))) } Ok(data) } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 4eae11a21..a3d6ea766 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -2099,7 +2099,7 @@ fn parse_create_table_hive_array() { }, ColumnDef { name: Ident::new("val"), - data_type: DataType::Array(Box::new(DataType::Int(None))), + data_type: DataType::Array(Some(Box::new(DataType::Int(None)))), collation: None, options: vec![], }, diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index c5d4bd0fc..869106220 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -1281,9 +1281,9 @@ fn parse_array_index_expr() { })], named: true, })), - data_type: DataType::Array(Box::new(DataType::Array(Box::new(DataType::Int( - None - ))))) + data_type: DataType::Array(Some(Box::new(DataType::Array(Some(Box::new( + DataType::Int(None) + )))))) }))), indexes: vec![num[1].clone(), num[2].clone()], }, diff --git a/tests/sqlparser_snowflake.rs b/tests/sqlparser_snowflake.rs index e3f23427a..b182becb0 100644 --- a/tests/sqlparser_snowflake.rs +++ b/tests/sqlparser_snowflake.rs @@ -150,10 +150,7 @@ fn parse_array() { assert_eq!( &Expr::Cast { expr: Box::new(Expr::Identifier(Ident::new("a"))), - data_type: DataType::Array(Box::new(DataType::Custom( - ObjectName(vec!["VARAINT".into()]), - vec![] - ))), + data_type: DataType::Array(None), }, expr_from_projection(only(&select.projection)) ); From d860141a7e345421298c14873437172dc3476ebd Mon Sep 17 00:00:00 2001 From: Yuval Shkolar Date: Thu, 3 Nov 2022 23:51:04 +0200 Subject: [PATCH 6/6] Remove unused import --- src/ast/data_type.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast/data_type.rs b/src/ast/data_type.rs index ceabe16e2..c0b2f977b 100644 --- a/src/ast/data_type.rs +++ b/src/ast/data_type.rs @@ -11,7 +11,7 @@ // limitations under the License. #[cfg(not(feature = "std"))] -use alloc::{boxed::Box, string::String, vec, vec::Vec}; +use alloc::{boxed::Box, string::String, vec::Vec}; use core::fmt; #[cfg(feature = "serde")]