Update to arrow 27

Cargo.toml

@@ -50,3 +50,10 @@ opt-level = 3
 overflow-checks = false
 panic = 'unwind'
 rpath = false
+
+[patch.crates-io]
+arrow = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }
+arrow-buffer = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }
+arrow-flight = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }
+parquet = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }

benchmarks/Cargo.toml

@@ -33,14 +33,14 @@ simd = ["datafusion/simd"]
 snmalloc = ["snmalloc-rs"]
 
 [dependencies]
-arrow = "26.0.0"
+arrow = "27.0.0"
 datafusion = { path = "../datafusion/core", version = "14.0.0" }
 env_logger = "0.9"
 futures = "0.3"
 mimalloc = { version = "0.1", optional = true, default-features = false }
 num_cpus = "1.13.0"
 object_store = "0.5.0"
-parquet = "26.0.0"
+parquet = "27.0.0"
 parquet-test-utils = { path = "../parquet-test-utils/", version = "0.1.0" }
 rand = "0.8.4"
 serde = { version = "1.0.136", features = ["derive"] }

datafusion-cli/Cargo.toml

@@ -29,7 +29,7 @@ rust-version = "1.62"
 readme = "README.md"
 
 [dependencies]
-arrow = "26.0.0"
+arrow = "27.0.0"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "14.0.0" }
 dirs = "4.0.0"
@@ -39,3 +39,9 @@ object_store = { version = "0.5.0", features = ["aws", "gcp"] }
 rustyline = "10.0"
 tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot"] }
 url = "2.2"
+
+[patch.crates-io]
+arrow = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }
+arrow-buffer = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }
+parquet = { git = "https://github.com/apache/arrow-rs", rev = "27.0.0" }

datafusion-examples/Cargo.toml

@@ -34,8 +34,8 @@ path = "examples/avro_sql.rs"
 required-features = ["datafusion/avro"]
 
 [dev-dependencies]
-arrow = "26.0.0"
-arrow-flight = "26.0.0"
+arrow = "27.0.0"
+arrow-flight = "27.0.0"
 async-trait = "0.1.41"
 datafusion = { path = "../datafusion/core" }
 datafusion-common = { path = "../datafusion/common" }

datafusion/common/Cargo.toml

@@ -40,10 +40,10 @@ pyarrow = ["pyo3", "arrow/pyarrow"]
 
 [dependencies]
 apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
-arrow = { version = "26.0.0", default-features = false }
+arrow = { version = "27.0.0", default-features = false }
 chrono = { version = "0.4", default-features = false }
 cranelift-module = { version = "0.89.0", optional = true }
 object_store = { version = "0.5.0", default-features = false, optional = true }
-parquet = { version = "26.0.0", default-features = false, optional = true }
+parquet = { version = "27.0.0", default-features = false, optional = true }
 pyo3 = { version = "0.17.1", optional = true }
 sqlparser = "0.26"

datafusion/core/Cargo.toml

@@ -56,7 +56,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
 apache-avro = { version = "0.14", optional = true }
-arrow = { version = "26.0.0", features = ["prettyprint"] }
+arrow = { version = "27.0.0", features = ["prettyprint"] }
 async-compression = { version = "0.3.14", features = ["bzip2", "gzip", "futures-io", "tokio"] }
 async-trait = "0.1.41"
 bytes = "1.1"
@@ -81,7 +81,7 @@ num-traits = { version = "0.2", optional = true }
 num_cpus = "1.13.0"
 object_store = "0.5.0"
 parking_lot = "0.12"
-parquet = { version = "26.0.0", features = ["arrow", "async"] }
+parquet = { version = "27.0.0", features = ["arrow", "async"] }
 paste = "^1.0"
 percent-encoding = "2.2.0"
 pin-project-lite = "^0.2.7"
@@ -98,7 +98,7 @@ url = "2.2"
 uuid = { version = "1.0", features = ["v4"] }
 
 [dev-dependencies]
-arrow = { version = "26.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { version = "27.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
 async-trait = "0.1.53"
 criterion = "0.4"
 csv = "1.1.6"

datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs

@@ -356,7 +356,7 @@ impl SortPreservingMergeStream {
                 Ok(SortField::new_with_options(data_type, expr.options))
             })
             .collect::<Result<Vec<_>>>()?;
-        let row_converter = RowConverter::new(sort_fields);
+        let row_converter = RowConverter::new(sort_fields)?;
 
         Ok(Self {
             schema,

datafusion/core/tests/sort_key_cursor.rs

@@ -26,7 +26,7 @@ use datafusion::physical_plan::sorts::{RowIndex, SortKeyCursor};
 
 #[test]
 fn test_single_column() {
-    let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]);
+    let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]).unwrap();
     let batch1 = int64_batch(vec![Some(1), Some(2), Some(5), Some(6)]);
     let batch2 = int64_batch(vec![Some(3), Some(4), Some(8), Some(9)]);
 
@@ -56,7 +56,7 @@ fn test_single_column() {
 
 #[test]
 fn test_stable_compare() {
-    let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]);
+    let mut converter = RowConverter::new(vec![SortField::new(DataType::Int64)]).unwrap();
     // Validate ties are broken by the lower stream idx to ensure stable sort
     let batch1 = int64_batch(vec![Some(3), Some(4)]);
     let batch2 = int64_batch(vec![Some(3)]);

datafusion/expr/Cargo.toml

@@ -36,7 +36,7 @@ path = "src/lib.rs"
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "26.0.0", default-features = false }
+arrow = { version = "27.0.0", default-features = false }
 datafusion-common = { path = "../common", version = "14.0.0" }
 log = "^0.4"
 sqlparser = "0.26"

datafusion/jit/Cargo.toml

@@ -36,7 +36,7 @@ path = "src/lib.rs"
 jit = []
 
 [dependencies]
-arrow = { version = "26.0.0", default-features = false }
+arrow = { version = "27.0.0", default-features = false }
 cranelift = "0.89.0"
 cranelift-jit = "0.89.0"
 cranelift-module = "0.89.0"