Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce dependencies of datafusion-sql crate #3566

Merged
merged 7 commits into from
Sep 21, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 6 additions & 6 deletions datafusion/common/Cargo.toml
Expand Up @@ -34,16 +34,16 @@ path = "src/lib.rs"

[features]
avro = ["apache-avro"]
default = []
jit = ["cranelift-module"]
pyarrow = ["pyo3"]
pyarrow = ["pyo3", "arrow/pyarrow"]

[dependencies]
apache-avro = { version = "0.14", features = ["snappy"], optional = true }
arrow = { version = "23.0.0", features = ["prettyprint"] }
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
arrow = { version = "23.0.0", default-features = false }
cranelift-module = { version = "0.87.0", optional = true }
object_store = { version = "0.5.0", optional = true }
object_store = { version = "0.5.0", default-features = false, optional = true }
ordered-float = "3.0"
parquet = { version = "23.0.0", features = ["arrow"], optional = true }
parquet = { version = "23.0.0", default-features = false, optional = true }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given the only reason for these dependencies is deriving From conversions for ArrowError, perhaps we could/should just remove them. This is what I have been doing when splitting apart arrow-rs - see apache/arrow-rs#2711.

In general the error handling is a little bit unfortunate, so perhaps this will all get addressed eventually by apache/arrow-rs#2725

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes absolutely. I was planning on revisiting this after the arrow-rs split.

pyo3 = { version = "0.17.1", optional = true }
sqlparser = "0.23"
2 changes: 1 addition & 1 deletion datafusion/expr/Cargo.toml
Expand Up @@ -36,6 +36,6 @@ path = "src/lib.rs"

[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", default-features = false }
datafusion-common = { path = "../common", version = "12.0.0" }
sqlparser = "0.23"
2 changes: 1 addition & 1 deletion datafusion/jit/Cargo.toml
Expand Up @@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []

[dependencies]
arrow = "23.0.0"
arrow = { version = "23.0.0", default-features = false }
cranelift = "0.87.0"
cranelift-jit = "0.87.0"
cranelift-module = "0.87.0"
Expand Down
5 changes: 1 addition & 4 deletions datafusion/sql/Cargo.toml
Expand Up @@ -37,10 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []

[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", default-features = false }
datafusion-common = { path = "../common", version = "12.0.0" }
datafusion-expr = { path = "../expr", version = "12.0.0" }
hashbrown = "0.12"
sqlparser = "0.23"
tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] }
31 changes: 15 additions & 16 deletions datafusion/sql/src/planner.rs
Expand Up @@ -41,8 +41,7 @@ use datafusion_expr::{
use datafusion_expr::{
window_function::WindowFunction, BuiltinScalarFunction, TableSource,
};
use hashbrown::HashMap;
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use std::sync::Arc;
use std::{convert::TryInto, vec};
Expand Down Expand Up @@ -4995,8 +4994,8 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn subquery_references_cte() {
#[test]
fn subquery_references_cte() {
let sql = "WITH \
cte AS (SELECT * FROM person) \
SELECT * FROM person WHERE EXISTS (SELECT * FROM cte WHERE id = person.id)";
Expand All @@ -5013,17 +5012,17 @@ mod tests {
quick_test(sql, expected)
}

#[tokio::test]
async fn aggregate_with_rollup() {
#[test]
fn aggregate_with_rollup() {
let sql = "SELECT id, state, age, COUNT(*) FROM person GROUP BY id, ROLLUP (state, age)";
let expected = "Projection: #person.id, #person.state, #person.age, #COUNT(UInt8(1))\
\n Aggregate: groupBy=[[#person.id, ROLLUP (#person.state, #person.age)]], aggr=[[COUNT(UInt8(1))]]\
\n TableScan: person";
quick_test(sql, expected);
}

#[tokio::test]
async fn aggregate_with_rollup_with_grouping() {
#[test]
fn aggregate_with_rollup_with_grouping() {
let sql = "SELECT id, state, age, grouping(state), grouping(age), grouping(state) + grouping(age), COUNT(*) \
FROM person GROUP BY id, ROLLUP (state, age)";
let expected = "Projection: #person.id, #person.state, #person.age, #GROUPING(person.state), #GROUPING(person.age), #GROUPING(person.state) + #GROUPING(person.age), #COUNT(UInt8(1))\
Expand All @@ -5032,8 +5031,8 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn rank_partition_grouping() {
#[test]
fn rank_partition_grouping() {
let sql = "select
sum(age) as total_sum,
state,
Expand All @@ -5054,8 +5053,8 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn aggregate_with_cube() {
#[test]
fn aggregate_with_cube() {
let sql =
"SELECT id, state, age, COUNT(*) FROM person GROUP BY id, CUBE (state, age)";
let expected = "Projection: #person.id, #person.state, #person.age, #COUNT(UInt8(1))\
Expand All @@ -5064,17 +5063,17 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn round_decimal() {
#[test]
fn round_decimal() {
let sql = "SELECT round(price/3, 2) FROM test_decimal";
let expected = "Projection: round(#test_decimal.price / Int64(3), Int64(2))\
\n TableScan: test_decimal";
quick_test(sql, expected);
}

#[ignore] // see https://github.com/apache/arrow-datafusion/issues/2469
#[tokio::test]
async fn aggregate_with_grouping_sets() {
#[test]
fn aggregate_with_grouping_sets() {
let sql = "SELECT id, state, age, COUNT(*) FROM person GROUP BY id, GROUPING SETS ((state), (state, age), (id, state))";
let expected = "TBD";
quick_test(sql, expected);
Expand Down