From cd3164918b0415b072a3109f3ccf654da7518ec1 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 27 Jul 2022 14:55:37 -0400 Subject: [PATCH] Update to arrow `19.0.0` (#2955) * Update to use arrow 19.0.0 * Update for infallable builder and Decimal128Array * Update avro for new API * Update more APIs for arrow 19.0.0 * Fix tests and clippy --- datafusion-cli/Cargo.toml | 2 +- datafusion-examples/Cargo.toml | 2 +- .../examples/custom_datasource.rs | 4 +- datafusion/common/Cargo.toml | 4 +- datafusion/common/src/scalar.rs | 149 +++++++-------- datafusion/core/Cargo.toml | 4 +- datafusion/core/fuzz-utils/Cargo.toml | 2 +- .../src/avro_to_arrow/arrow_array_reader.rs | 102 ++++++----- datafusion/core/src/avro_to_arrow/schema.rs | 1 + .../core/src/catalog/information_schema.rs | 86 +++------ .../core/src/datasource/listing/helpers.rs | 8 +- .../core/src/physical_optimizer/pruning.rs | 6 +- .../core/src/physical_plan/aggregates/hash.rs | 2 +- .../src/physical_plan/aggregates/row_hash.rs | 2 +- datafusion/core/src/physical_plan/analyze.rs | 18 +- datafusion/core/src/physical_plan/explain.rs | 6 +- .../core/src/physical_plan/hash_join.rs | 43 +++-- .../core/src/physical_plan/hash_utils.rs | 6 +- .../core/src/physical_plan/repartition.rs | 3 +- .../core/src/physical_plan/sort_merge_join.rs | 20 +-- datafusion/core/src/test/mod.rs | 4 +- .../core/tests/provider_filter_pushdown.rs | 2 +- datafusion/core/tests/sql/mod.rs | 16 +- datafusion/core/tests/sql/select.rs | 14 +- datafusion/expr/Cargo.toml | 2 +- datafusion/jit/Cargo.toml | 2 +- datafusion/optimizer/Cargo.toml | 2 +- datafusion/physical-expr/Cargo.toml | 2 +- .../physical-expr/src/aggregate/average.rs | 6 +- .../src/aggregate/count_distinct.rs | 16 +- .../physical-expr/src/aggregate/min_max.rs | 24 +-- datafusion/physical-expr/src/aggregate/sum.rs | 14 +- .../src/aggregate/sum_distinct.rs | 2 +- .../physical-expr/src/array_expressions.rs | 23 +-- .../physical-expr/src/datetime_expressions.rs | 10 +- .../physical-expr/src/expressions/binary.rs | 170 ++++++++++-------- .../physical-expr/src/expressions/cast.rs | 47 ++--- .../physical-expr/src/expressions/datetime.rs | 2 +- .../src/expressions/get_indexed_field.rs | 13 +- .../physical-expr/src/expressions/in_list.rs | 12 +- .../physical-expr/src/expressions/try_cast.rs | 48 ++--- .../physical-expr/src/regex_expressions.rs | 32 ++-- datafusion/proto/Cargo.toml | 2 +- datafusion/proto/src/to_proto.rs | 6 +- datafusion/row/Cargo.toml | 2 +- datafusion/row/src/reader.rs | 16 +- datafusion/sql/Cargo.toml | 2 +- 47 files changed, 469 insertions(+), 492 deletions(-) diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 1684ecf5001..6ac3a30c824 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -29,7 +29,7 @@ rust-version = "1.59" readme = "README.md" [dependencies] -arrow = { version = "18.0.0" } +arrow = { version = "19.0.0" } clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "10.0.0" } dirs = "4.0.0" diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 00d2a68c994..ed23512f3e2 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -34,7 +34,7 @@ path = "examples/avro_sql.rs" required-features = ["datafusion/avro"] [dev-dependencies] -arrow-flight = { version = "18.0.0" } +arrow-flight = { version = "19.0.0" } async-trait = "0.1.41" datafusion = { path = "../datafusion/core" } futures = "0.3" diff --git a/datafusion-examples/examples/custom_datasource.rs b/datafusion-examples/examples/custom_datasource.rs index b68936a7c6e..2be4f194c79 100644 --- a/datafusion-examples/examples/custom_datasource.rs +++ b/datafusion-examples/examples/custom_datasource.rs @@ -247,8 +247,8 @@ impl ExecutionPlan for CustomExec { let mut account_array = UInt64Builder::new(users.len()); for user in users { - id_array.append_value(user.id)?; - account_array.append_value(user.bank_account)?; + id_array.append_value(user.id); + account_array.append_value(user.bank_account); } Ok(Box::pin(MemoryStream::try_new( diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 0e1a7944514..e121a55a4ea 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -38,11 +38,11 @@ jit = ["cranelift-module"] pyarrow = ["pyo3"] [dependencies] -arrow = { version = "18.0.0", features = ["prettyprint"] } +arrow = { version = "19.0.0", features = ["prettyprint"] } avro-rs = { version = "0.13", features = ["snappy"], optional = true } cranelift-module = { version = "0.86.1", optional = true } object_store = { version = "0.3", optional = true } ordered-float = "3.0" -parquet = { version = "18.0.0", features = ["arrow"], optional = true } +parquet = { version = "19.0.0", features = ["arrow"], optional = true } pyo3 = { version = "0.16", optional = true } sqlparser = "0.18" diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index c2b0e875b9f..b30238762a3 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -527,15 +527,15 @@ macro_rules! build_values_list { for scalar_value in $VALUES { match scalar_value { ScalarValue::$SCALAR_TY(Some(v)) => { - builder.values().append_value(v.clone()).unwrap() + builder.values().append_value(v.clone()); } ScalarValue::$SCALAR_TY(None) => { - builder.values().append_null().unwrap(); + builder.values().append_null(); } _ => panic!("Incompatible ScalarValue for list"), }; } - builder.append(true).unwrap(); + builder.append(true); } builder.finish() @@ -550,15 +550,15 @@ macro_rules! build_values_list_tz { for scalar_value in $VALUES { match scalar_value { ScalarValue::$SCALAR_TY(Some(v), _) => { - builder.values().append_value(v.clone()).unwrap() + builder.values().append_value(v.clone()); } ScalarValue::$SCALAR_TY(None, _) => { - builder.values().append_null().unwrap(); + builder.values().append_null(); } _ => panic!("Incompatible ScalarValue for list"), }; } - builder.append(true).unwrap(); + builder.append(true); } builder.finish() @@ -879,10 +879,10 @@ impl ScalarValue { for s in xs { match s { ScalarValue::$SCALAR_TY(Some(val)) => { - builder.values().append_value(val)?; + builder.values().append_value(val); } ScalarValue::$SCALAR_TY(None) => { - builder.values().append_null()?; + builder.values().append_null(); } sv => { return Err(DataFusionError::Internal(format!( @@ -893,10 +893,10 @@ impl ScalarValue { } } } - builder.append(true)?; + builder.append(true); } ScalarValue::List(None, _) => { - builder.append(false)?; + builder.append(false); } sv => { return Err(DataFusionError::Internal(format!( @@ -917,6 +917,11 @@ impl ScalarValue { ScalarValue::iter_to_decimal_array(scalars, precision, scale)?; Arc::new(decimal_array) } + DataType::Decimal256(_, _) => { + return Err(DataFusionError::Internal( + "Decimal256 is not supported for ScalarValue".to_string(), + )) + } DataType::Null => ScalarValue::iter_to_null_array(scalars), DataType::Boolean => build_array_primitive!(BooleanArray, Boolean), DataType::Float32 => build_array_primitive!(Float32Array, Float32), @@ -1112,14 +1117,14 @@ impl ScalarValue { scalars: impl IntoIterator, precision: &usize, scale: &usize, - ) -> Result { + ) -> Result { let array = scalars .into_iter() .map(|element: ScalarValue| match element { ScalarValue::Decimal128(v1, _, _) => v1, _ => unreachable!(), }) - .collect::() + .collect::() .with_precision_and_scale(*precision, *scale)?; Ok(array) } @@ -1129,9 +1134,7 @@ impl ScalarValue { data_type: &DataType, ) -> Result> { let mut offsets = Int32Array::builder(0); - if let Err(err) = offsets.append_value(0) { - return Err(DataFusionError::ArrowError(err)); - } + offsets.append_value(0); let mut elements: Vec = Vec::new(); let mut valid = BooleanBufferBuilder::new(0); @@ -1144,9 +1147,7 @@ impl ScalarValue { // Add new offset index flat_len += element_array.len() as i32; - if let Err(err) = offsets.append_value(flat_len) { - return Err(DataFusionError::ArrowError(err)); - } + offsets.append_value(flat_len); elements.push(element_array); @@ -1155,9 +1156,7 @@ impl ScalarValue { } None => { // Repeat previous offset index - if let Err(err) = offsets.append_value(flat_len) { - return Err(DataFusionError::ArrowError(err)); - } + offsets.append_value(flat_len); // Element is null valid.append(false); @@ -1196,10 +1195,10 @@ impl ScalarValue { precision: &usize, scale: &usize, size: usize, - ) -> DecimalArray { + ) -> Decimal128Array { std::iter::repeat(value) .take(size) - .collect::() + .collect::() .with_precision_and_scale(*precision, *scale) .unwrap() } @@ -1405,7 +1404,7 @@ impl ScalarValue { precision: &usize, scale: &usize, ) -> ScalarValue { - let array = array.as_any().downcast_ref::().unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); if array.is_null(index) { ScalarValue::Decimal128(None, *precision, *scale) } else { @@ -1593,7 +1592,7 @@ impl ScalarValue { precision: usize, scale: usize, ) -> bool { - let array = array.as_any().downcast_ref::().unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); if array.precision() != precision || array.scale() != scale { return false; } @@ -2139,14 +2138,14 @@ mod tests { // decimal scalar to array let array = decimal_value.to_array(); - let array = array.as_any().downcast_ref::().unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); assert_eq!(1, array.len()); assert_eq!(DataType::Decimal(10, 1), array.data_type().clone()); assert_eq!(123i128, array.value(0).as_i128()); // decimal scalar to array with size let array = decimal_value.to_array_of_size(10); - let array_decimal = array.as_any().downcast_ref::().unwrap(); + let array_decimal = array.as_any().downcast_ref::().unwrap(); assert_eq!(10, array.len()); assert_eq!(DataType::Decimal(10, 1), array.data_type().clone()); assert_eq!(123i128, array_decimal.value(0).as_i128()); @@ -3106,116 +3105,100 @@ mod tests { .values() .field_builder::(0) .unwrap() - .append_value("First") - .unwrap(); + .append_value("First"); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(1) - .unwrap(); + .append_value(1); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(2) - .unwrap(); + .append_value(2); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(3) - .unwrap(); + .append_value(3); list_builder .values() .field_builder::>>(1) .unwrap() - .append(true) - .unwrap(); - list_builder.values().append(true).unwrap(); + .append(true); + list_builder.values().append(true); list_builder .values() .field_builder::(0) .unwrap() - .append_value("Second") - .unwrap(); + .append_value("Second"); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(4) - .unwrap(); + .append_value(4); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(5) - .unwrap(); + .append_value(5); list_builder .values() .field_builder::>>(1) .unwrap() - .append(true) - .unwrap(); - list_builder.values().append(true).unwrap(); - list_builder.append(true).unwrap(); + .append(true); + list_builder.values().append(true); + list_builder.append(true); list_builder .values() .field_builder::(0) .unwrap() - .append_value("Third") - .unwrap(); + .append_value("Third"); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(6) - .unwrap(); + .append_value(6); list_builder .values() .field_builder::>>(1) .unwrap() - .append(true) - .unwrap(); - list_builder.values().append(true).unwrap(); - list_builder.append(true).unwrap(); + .append(true); + list_builder.values().append(true); + list_builder.append(true); list_builder .values() .field_builder::(0) .unwrap() - .append_value("Second") - .unwrap(); + .append_value("Second"); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(4) - .unwrap(); + .append_value(4); list_builder .values() .field_builder::>>(1) .unwrap() .values() - .append_value(5) - .unwrap(); + .append_value(5); list_builder .values() .field_builder::>>(1) .unwrap() - .append(true) - .unwrap(); - list_builder.values().append(true).unwrap(); - list_builder.append(true).unwrap(); + .append(true); + list_builder.values().append(true); + list_builder.append(true); let expected = list_builder.finish(); @@ -3285,27 +3268,27 @@ mod tests { let middle_builder = ListBuilder::new(inner_builder); let mut outer_builder = ListBuilder::new(middle_builder); - outer_builder.values().values().append_value(1).unwrap(); - outer_builder.values().values().append_value(2).unwrap(); - outer_builder.values().values().append_value(3).unwrap(); - outer_builder.values().append(true).unwrap(); + outer_builder.values().values().append_value(1); + outer_builder.values().values().append_value(2); + outer_builder.values().values().append_value(3); + outer_builder.values().append(true); - outer_builder.values().values().append_value(4).unwrap(); - outer_builder.values().values().append_value(5).unwrap(); - outer_builder.values().append(true).unwrap(); - outer_builder.append(true).unwrap(); + outer_builder.values().values().append_value(4); + outer_builder.values().values().append_value(5); + outer_builder.values().append(true); + outer_builder.append(true); - outer_builder.values().values().append_value(6).unwrap(); - outer_builder.values().append(true).unwrap(); + outer_builder.values().values().append_value(6); + outer_builder.values().append(true); - outer_builder.values().values().append_value(7).unwrap(); - outer_builder.values().values().append_value(8).unwrap(); - outer_builder.values().append(true).unwrap(); - outer_builder.append(true).unwrap(); + outer_builder.values().values().append_value(7); + outer_builder.values().values().append_value(8); + outer_builder.values().append(true); + outer_builder.append(true); - outer_builder.values().values().append_value(9).unwrap(); - outer_builder.values().append(true).unwrap(); - outer_builder.append(true).unwrap(); + outer_builder.values().values().append_value(9); + outer_builder.values().append(true); + outer_builder.append(true); let expected = outer_builder.finish(); diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 351e72fc2a5..4385badb12d 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "18.0.0", features = ["prettyprint"] } +arrow = { version = "19.0.0", features = ["prettyprint"] } async-trait = "0.1.41" avro-rs = { version = "0.13", features = ["snappy"], optional = true } bytes = "1.1" @@ -78,7 +78,7 @@ num_cpus = "1.13.0" object_store = "0.3.0" ordered-float = "3.0" parking_lot = "0.12" -parquet = { version = "18.0.0", features = ["arrow", "async"] } +parquet = { version = "19.0.0", features = ["arrow", "async"] } paste = "^1.0" pin-project-lite = "^0.2.7" pyo3 = { version = "0.16", optional = true } diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml index 8544d88f1a0..0d66a699999 100644 --- a/datafusion/core/fuzz-utils/Cargo.toml +++ b/datafusion/core/fuzz-utils/Cargo.toml @@ -23,6 +23,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { version = "18.0.0", features = ["prettyprint"] } +arrow = { version = "19.0.0", features = ["prettyprint"] } env_logger = "0.9.0" rand = "0.8" diff --git a/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs b/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs index e82f732aa56..358a6d0485e 100644 --- a/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs +++ b/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs @@ -130,58 +130,52 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { arrays.and_then(|arr| RecordBatch::try_new(projected_schema, arr).map(Some)) } - fn build_boolean_array( - &self, - rows: RecordSlice, - col_name: &str, - ) -> ArrowResult { + fn build_boolean_array(&self, rows: RecordSlice, col_name: &str) -> ArrayRef { let mut builder = BooleanBuilder::new(rows.len()); for row in rows { if let Some(value) = self.field_lookup(col_name, row) { if let Some(boolean) = resolve_boolean(value) { - builder.append_value(boolean)? + builder.append_value(boolean) } else { - builder.append_null()?; + builder.append_null(); } } else { - builder.append_null()?; + builder.append_null(); } } - Ok(Arc::new(builder.finish())) + Arc::new(builder.finish()) } - #[allow(clippy::unnecessary_wraps)] fn build_primitive_array( &self, rows: RecordSlice, col_name: &str, - ) -> ArrowResult + ) -> ArrayRef where T: ArrowNumericType, T::Native: num_traits::cast::NumCast, { - Ok(Arc::new( + Arc::new( rows.iter() .map(|row| { self.field_lookup(col_name, row) .and_then(|value| resolve_item::(value)) }) .collect::>(), - )) + ) } #[inline(always)] - #[allow(clippy::unnecessary_wraps)] fn build_string_dictionary_builder( &self, row_len: usize, - ) -> ArrowResult> + ) -> StringDictionaryBuilder where T: ArrowPrimitiveType + ArrowDictionaryKeyType, { let key_builder = PrimitiveBuilder::::new(row_len); let values_builder = StringBuilder::new(row_len * 5); - Ok(StringDictionaryBuilder::new(key_builder, values_builder)) + StringDictionaryBuilder::new(key_builder, values_builder) } fn build_wrapped_list_array( @@ -271,7 +265,7 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { } DataType::Dictionary(_, _) => { let values_builder = - self.build_string_dictionary_builder::(rows.len() * 5)?; + self.build_string_dictionary_builder::(rows.len() * 5); Box::new(ListBuilder::new(values_builder)) } e => { @@ -316,14 +310,14 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { ))?; for val in vals { if let Some(v) = val { - builder.values().append_value(&v)? + builder.values().append_value(&v) } else { - builder.values().append_null()? + builder.values().append_null() }; } // Append to the list - builder.append(true)?; + builder.append(true); } DataType::Dictionary(_, _) => { let builder = builder.as_any_mut().downcast_mut::>>().ok_or_else(||ArrowError::SchemaError( @@ -333,12 +327,12 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { if let Some(v) = val { let _ = builder.values().append(&v)?; } else { - builder.values().append_null()? + builder.values().append_null() }; } // Append to the list - builder.append(true)?; + builder.append(true); } e => { return Err(SchemaError(format!( @@ -364,16 +358,16 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { T: ArrowPrimitiveType + ArrowDictionaryKeyType, { let mut builder: StringDictionaryBuilder = - self.build_string_dictionary_builder(rows.len())?; + self.build_string_dictionary_builder(rows.len()); for row in rows { if let Some(value) = self.field_lookup(col_name, row) { if let Ok(str_v) = resolve_string(value) { builder.append(str_v).map(drop)? } else { - builder.append_null()? + builder.append_null() } } else { - builder.append_null()? + builder.append_null() } } Ok(Arc::new(builder.finish()) as ArrayRef) @@ -609,10 +603,8 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { .iter() .filter(|field| projection.is_empty() || projection.contains(field.name())) .map(|field| { - match field.data_type() { - DataType::Null => { - Ok(Arc::new(NullArray::new(rows.len())) as ArrayRef) - } + let arr = match field.data_type() { + DataType::Null => Arc::new(NullArray::new(rows.len())) as ArrayRef, DataType::Boolean => self.build_boolean_array(rows, field.name()), DataType::Float64 => { self.build_primitive_array::(rows, field.name()) @@ -684,10 +676,12 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { rows, field.name(), ), - t => Err(ArrowError::SchemaError(format!( - "TimeUnit {:?} not supported with Time64", - t - ))), + t => { + return Err(ArrowError::SchemaError(format!( + "TimeUnit {:?} not supported with Time64", + t + ))) + } }, DataType::Time32(unit) => match unit { TimeUnit::Second => self @@ -700,12 +694,14 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { rows, field.name(), ), - t => Err(ArrowError::SchemaError(format!( - "TimeUnit {:?} not supported with Time32", - t - ))), + t => { + return Err(ArrowError::SchemaError(format!( + "TimeUnit {:?} not supported with Time32", + t + ))) + } }, - DataType::Utf8 | DataType::LargeUtf8 => Ok(Arc::new( + DataType::Utf8 | DataType::LargeUtf8 => Arc::new( rows.iter() .map(|row| { let maybe_value = self.field_lookup(field.name(), row); @@ -713,8 +709,8 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { }) .collect::>()?, ) - as ArrayRef), - DataType::Binary | DataType::LargeBinary => Ok(Arc::new( + as ArrayRef, + DataType::Binary | DataType::LargeBinary => Arc::new( rows.iter() .map(|row| { let maybe_value = self.field_lookup(field.name(), row); @@ -722,11 +718,11 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { }) .collect::(), ) - as ArrayRef), + as ArrayRef, DataType::List(ref list_field) => { match list_field.data_type() { DataType::Dictionary(ref key_ty, _) => { - self.build_wrapped_list_array(rows, field.name(), key_ty) + self.build_wrapped_list_array(rows, field.name(), key_ty)? } _ => { // extract rows by name @@ -740,7 +736,7 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { self.build_nested_list_array::( extracted_rows.as_slice(), list_field, - ) + )? } } } @@ -750,7 +746,7 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { field.name(), key_ty, val_ty, - ), + )?, DataType::Struct(fields) => { let len = rows.len(); let num_bytes = bit_util::ceil(len, 8); @@ -778,15 +774,17 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> { .child_data( arrays.into_iter().map(|a| a.data().clone()).collect(), ) - .build() - .unwrap(); - Ok(make_array(data)) + .build()?; + make_array(data) } - _ => Err(ArrowError::SchemaError(format!( - "type {:?} not supported", - field.data_type() - ))), - } + _ => { + return Err(ArrowError::SchemaError(format!( + "type {:?} not supported", + field.data_type() + ))) + } + }; + Ok(arr) }) .collect(); arrays diff --git a/datafusion/core/src/avro_to_arrow/schema.rs b/datafusion/core/src/avro_to_arrow/schema.rs index dd8b90776bc..c5b4febbb93 100644 --- a/datafusion/core/src/avro_to_arrow/schema.rs +++ b/datafusion/core/src/avro_to_arrow/schema.rs @@ -217,6 +217,7 @@ fn default_field_name(dt: &DataType) -> &str { DataType::Dictionary(_, _) => "map", DataType::Map(_, _) => unimplemented!("Map support not implemented"), DataType::Decimal(_, _) => "decimal", + DataType::Decimal256(_, _) => "decimal", } } diff --git a/datafusion/core/src/catalog/information_schema.rs b/datafusion/core/src/catalog/information_schema.rs index 0467dffa702..d4944c2d544 100644 --- a/datafusion/core/src/catalog/information_schema.rs +++ b/datafusion/core/src/catalog/information_schema.rs @@ -266,20 +266,14 @@ impl InformationSchemaTablesBuilder { table_type: TableType, ) { // Note: append_value is actually infallable. - self.catalog_names - .append_value(catalog_name.as_ref()) - .unwrap(); - self.schema_names - .append_value(schema_name.as_ref()) - .unwrap(); - self.table_names.append_value(table_name.as_ref()).unwrap(); - self.table_types - .append_value(match table_type { - TableType::Base => "BASE TABLE", - TableType::View => "VIEW", - TableType::Temporary => "LOCAL TEMPORARY", - }) - .unwrap(); + self.catalog_names.append_value(catalog_name.as_ref()); + self.schema_names.append_value(schema_name.as_ref()); + self.table_names.append_value(table_name.as_ref()); + self.table_types.append_value(match table_type { + TableType::Base => "BASE TABLE", + TableType::View => "VIEW", + TableType::Temporary => "LOCAL TEMPORARY", + }); } } @@ -347,14 +341,10 @@ impl InformationSchemaViewBuilder { definition: Option>, ) { // Note: append_value is actually infallable. - self.catalog_names - .append_value(catalog_name.as_ref()) - .unwrap(); - self.schema_names - .append_value(schema_name.as_ref()) - .unwrap(); - self.table_names.append_value(table_name.as_ref()).unwrap(); - self.definitions.append_option(definition.as_ref()).unwrap(); + self.catalog_names.append_value(catalog_name.as_ref()); + self.schema_names.append_value(schema_name.as_ref()); + self.table_names.append_value(table_name.as_ref()); + self.definitions.append_option(definition.as_ref()); } } @@ -450,33 +440,23 @@ impl InformationSchemaColumnsBuilder { use DataType::*; // Note: append_value is actually infallable. - self.catalog_names - .append_value(catalog_name.as_ref()) - .unwrap(); - self.schema_names - .append_value(schema_name.as_ref()) - .unwrap(); - self.table_names.append_value(table_name.as_ref()).unwrap(); - - self.column_names - .append_value(column_name.as_ref()) - .unwrap(); - - self.ordinal_positions - .append_value(column_position as u64) - .unwrap(); + self.catalog_names.append_value(catalog_name.as_ref()); + self.schema_names.append_value(schema_name.as_ref()); + self.table_names.append_value(table_name.as_ref()); + + self.column_names.append_value(column_name.as_ref()); + + self.ordinal_positions.append_value(column_position as u64); // DataFusion does not support column default values, so null - self.column_defaults.append_null().unwrap(); + self.column_defaults.append_null(); // "YES if the column is possibly nullable, NO if it is known not nullable. " let nullable_str = if is_nullable { "YES" } else { "NO" }; - self.is_nullables.append_value(nullable_str).unwrap(); + self.is_nullables.append_value(nullable_str); // "System supplied type" --> Use debug format of the datatype - self.data_types - .append_value(format!("{:?}", data_type)) - .unwrap(); + self.data_types.append_value(format!("{:?}", data_type)); // "If data_type identifies a character or bit string type, the // declared maximum length; null for all other data types or @@ -484,9 +464,7 @@ impl InformationSchemaColumnsBuilder { // // Arrow has no equivalent of VARCHAR(20), so we leave this as Null let max_chars = None; - self.character_maximum_lengths - .append_option(max_chars) - .unwrap(); + self.character_maximum_lengths.append_option(max_chars); // "Maximum length, in bytes, for binary data, character data, // or text and image data." @@ -495,9 +473,7 @@ impl InformationSchemaColumnsBuilder { LargeBinary | LargeUtf8 => Some(i64::MAX as u64), _ => None, }; - self.character_octet_lengths - .append_option(char_len) - .unwrap(); + self.character_octet_lengths.append_option(char_len); // numeric_precision: "If data_type identifies a numeric type, this column // contains the (declared or implicit) precision of the type @@ -538,16 +514,12 @@ impl InformationSchemaColumnsBuilder { _ => (None, None, None), }; - self.numeric_precisions - .append_option(numeric_precision) - .unwrap(); - self.numeric_precision_radixes - .append_option(numeric_radix) - .unwrap(); - self.numeric_scales.append_option(numeric_scale).unwrap(); + self.numeric_precisions.append_option(numeric_precision); + self.numeric_precision_radixes.append_option(numeric_radix); + self.numeric_scales.append_option(numeric_scale); - self.datetime_precisions.append_option(None).unwrap(); - self.interval_types.append_null().unwrap(); + self.datetime_precisions.append_option(None); + self.interval_types.append_null(); } } diff --git a/datafusion/core/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs index b4b93d5fc17..80fece68440 100644 --- a/datafusion/core/src/datasource/listing/helpers.rs +++ b/datafusion/core/src/datasource/listing/helpers.rs @@ -247,11 +247,11 @@ fn paths_to_batch( &file_meta.location, table_partition_cols, ) { - key_builder.append_value(file_meta.location.as_ref())?; - length_builder.append_value(file_meta.size as u64)?; - modified_builder.append_value(file_meta.last_modified.timestamp_millis())?; + key_builder.append_value(file_meta.location.as_ref()); + length_builder.append_value(file_meta.size as u64); + modified_builder.append_value(file_meta.last_modified.timestamp_millis()); for (i, part_val) in partition_values.iter().enumerate() { - partition_builders[i].append_value(part_val)?; + partition_builders[i].append_value(part_val); } } else { debug!("No partitioning for path {}", file_meta.location); diff --git a/datafusion/core/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs index 2265675b300..b1242b15032 100644 --- a/datafusion/core/src/physical_optimizer/pruning.rs +++ b/datafusion/core/src/physical_optimizer/pruning.rs @@ -800,7 +800,7 @@ mod tests { use crate::from_slice::FromSlice; use crate::logical_plan::{col, lit}; use crate::{assert_batches_eq, physical_optimizer::pruning::StatisticsType}; - use arrow::array::DecimalArray; + use arrow::array::Decimal128Array; use arrow::{ array::{BinaryArray, Int32Array, Int64Array, StringArray}, datatypes::{DataType, TimeUnit}, @@ -825,13 +825,13 @@ mod tests { Self { min: Arc::new( min.into_iter() - .collect::() + .collect::() .with_precision_and_scale(precision, scale) .unwrap(), ), max: Arc::new( max.into_iter() - .collect::() + .collect::() .with_precision_and_scale(precision, scale) .unwrap(), ), diff --git a/datafusion/core/src/physical_plan/aggregates/hash.rs b/datafusion/core/src/physical_plan/aggregates/hash.rs index ddf9af18fd7..c21109495e2 100644 --- a/datafusion/core/src/physical_plan/aggregates/hash.rs +++ b/datafusion/core/src/physical_plan/aggregates/hash.rs @@ -273,7 +273,7 @@ fn group_aggregate_batch( let mut offset_so_far = 0; for group_idx in groups_with_rows.iter() { let indices = &accumulators.group_states[*group_idx].indices; - batch_indices.append_slice(indices)?; + batch_indices.append_slice(indices); offset_so_far += indices.len(); offsets.push(offset_so_far); } diff --git a/datafusion/core/src/physical_plan/aggregates/row_hash.rs b/datafusion/core/src/physical_plan/aggregates/row_hash.rs index 5353bc745c1..2ef1d16d4fa 100644 --- a/datafusion/core/src/physical_plan/aggregates/row_hash.rs +++ b/datafusion/core/src/physical_plan/aggregates/row_hash.rs @@ -288,7 +288,7 @@ fn group_aggregate_batch( let mut offset_so_far = 0; for group_idx in groups_with_rows.iter() { let indices = &aggr_state.group_states[*group_idx].indices; - batch_indices.append_slice(indices)?; + batch_indices.append_slice(indices); offset_so_far += indices.len(); offsets.push(offset_so_far); } diff --git a/datafusion/core/src/physical_plan/analyze.rs b/datafusion/core/src/physical_plan/analyze.rs index c2f08c69a4b..46273aa893a 100644 --- a/datafusion/core/src/physical_plan/analyze.rs +++ b/datafusion/core/src/physical_plan/analyze.rs @@ -160,32 +160,30 @@ impl ExecutionPlan for AnalyzeExec { let mut plan_builder = StringBuilder::new(1); // TODO use some sort of enum rather than strings? - type_builder.append_value("Plan with Metrics").unwrap(); + type_builder.append_value("Plan with Metrics"); let annotated_plan = DisplayableExecutionPlan::with_metrics(captured_input.as_ref()) .indent() .to_string(); - plan_builder.append_value(annotated_plan).unwrap(); + plan_builder.append_value(annotated_plan); // Verbose output // TODO make this more sophisticated if verbose { - type_builder.append_value("Plan with Full Metrics").unwrap(); + type_builder.append_value("Plan with Full Metrics"); let annotated_plan = DisplayableExecutionPlan::with_full_metrics(captured_input.as_ref()) .indent() .to_string(); - plan_builder.append_value(annotated_plan).unwrap(); + plan_builder.append_value(annotated_plan); - type_builder.append_value("Output Rows").unwrap(); - plan_builder.append_value(total_rows.to_string()).unwrap(); + type_builder.append_value("Output Rows"); + plan_builder.append_value(total_rows.to_string()); - type_builder.append_value("Duration").unwrap(); - plan_builder - .append_value(format!("{:?}", end - start)) - .unwrap(); + type_builder.append_value("Duration"); + plan_builder.append_value(format!("{:?}", end - start)); } let maybe_batch = RecordBatch::try_new( diff --git a/datafusion/core/src/physical_plan/explain.rs b/datafusion/core/src/physical_plan/explain.rs index fdc139a7e90..555b9e1e7e1 100644 --- a/datafusion/core/src/physical_plan/explain.rs +++ b/datafusion/core/src/physical_plan/explain.rs @@ -133,13 +133,13 @@ impl ExecutionPlan for ExplainExec { let mut prev: Option<&StringifiedPlan> = None; for p in plans_to_print { - type_builder.append_value(p.plan_type.to_string())?; + type_builder.append_value(p.plan_type.to_string()); match prev { Some(prev) if !should_show(prev, p) => { - plan_builder.append_value("SAME TEXT AS ABOVE")?; + plan_builder.append_value("SAME TEXT AS ABOVE"); } Some(_) | None => { - plan_builder.append_value(&*p.plan)?; + plan_builder.append_value(&*p.plan); } } prev = Some(p); diff --git a/datafusion/core/src/physical_plan/hash_join.rs b/datafusion/core/src/physical_plan/hash_join.rs index 6db50403452..6540cc1301c 100644 --- a/datafusion/core/src/physical_plan/hash_join.rs +++ b/datafusion/core/src/physical_plan/hash_join.rs @@ -23,7 +23,7 @@ use ahash::RandomState; use arrow::{ array::{ as_dictionary_array, as_string_array, ArrayData, ArrayRef, BasicDecimalArray, - BooleanArray, Date32Array, Date64Array, DecimalArray, DictionaryArray, + BooleanArray, Date32Array, Date64Array, Decimal128Array, DictionaryArray, LargeStringArray, PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampSecondArray, UInt32BufferBuilder, UInt32Builder, UInt64BufferBuilder, UInt64Builder, @@ -787,8 +787,8 @@ fn build_join_indexes( &keys_values, *null_equals_null, )? { - left_indices.append_value(i)?; - right_indices.append_value(row as u32)?; + left_indices.append_value(i); + right_indices.append_value(row as u32); } } }; @@ -811,22 +811,22 @@ fn build_join_indexes( &keys_values, *null_equals_null, )? { - left_indices.append_value(i)?; - right_indices.append_value(row as u32)?; + left_indices.append_value(i); + right_indices.append_value(row as u32); no_match = false; } } // If no rows matched left, still must keep the right // with all nulls for left if no_match { - left_indices.append_null()?; - right_indices.append_value(row as u32)?; + left_indices.append_null(); + right_indices.append_value(row as u32); } } None => { // when no match, add the row with None for the left side - left_indices.append_null()?; - right_indices.append_value(row as u32)?; + left_indices.append_null(); + right_indices.append_value(row as u32); } } } @@ -901,13 +901,13 @@ fn apply_join_filter( // If row index changes and row doesnt have match // append (idx, null) if right_indices.value(pos) != state.0 && !state.1 { - right_rebuilt.append_value(state.0)?; - left_rebuilt.append_null()?; + right_rebuilt.append_value(state.0); + left_rebuilt.append_null(); } // If has match append matched row indices if mask.value(pos) { - right_rebuilt.append_value(right_indices.value(pos))?; - left_rebuilt.append_value(left_indices.value(pos))?; + right_rebuilt.append_value(right_indices.value(pos)); + left_rebuilt.append_value(left_indices.value(pos)); }; // Calculate if current row index has match @@ -921,12 +921,11 @@ fn apply_join_filter( }, ) // Append last row from right side if no match found - .and_then(|(row_idx, has_match)| { + .map(|(row_idx, has_match)| { if !has_match { - right_rebuilt.append_value(row_idx)?; - left_rebuilt.append_null()?; + right_rebuilt.append_value(row_idx); + left_rebuilt.append_null(); } - Ok(()) })?; Ok((left_rebuilt.finish(), right_rebuilt.finish())) @@ -1103,7 +1102,7 @@ fn equal_rows( DataType::Decimal(_, rscale) => { if lscale == rscale { equal_rows_elem!( - DecimalArray, + Decimal128Array, l, r, left, @@ -2378,12 +2377,12 @@ mod tests { )?; let mut left_ids = UInt64Builder::new(0); - left_ids.append_value(0)?; - left_ids.append_value(1)?; + left_ids.append_value(0); + left_ids.append_value(1); let mut right_ids = UInt32Builder::new(0); - right_ids.append_value(0)?; - right_ids.append_value(1)?; + right_ids.append_value(0); + right_ids.append_value(1); assert_eq!(left_ids.finish(), l); diff --git a/datafusion/core/src/physical_plan/hash_utils.rs b/datafusion/core/src/physical_plan/hash_utils.rs index 81bf5c1f3d2..b9c34ec9b40 100644 --- a/datafusion/core/src/physical_plan/hash_utils.rs +++ b/datafusion/core/src/physical_plan/hash_utils.rs @@ -21,7 +21,7 @@ use crate::error::{DataFusionError, Result}; use ahash::{CallHasher, RandomState}; use arrow::array::{ Array, ArrayRef, BasicDecimalArray, BooleanArray, Date32Array, Date64Array, - DecimalArray, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array, + Decimal128Array, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array, UInt64Array, UInt8Array, @@ -58,7 +58,7 @@ fn hash_decimal128<'a>( hashes_buffer: &'a mut [u64], mul_col: bool, ) { - let array = array.as_any().downcast_ref::().unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); if array.null_count() == 0 { if mul_col { for (i, hash) in hashes_buffer.iter_mut().enumerate() { @@ -626,7 +626,7 @@ mod tests { let array = vec![1, 2, 3, 4] .into_iter() .map(Some) - .collect::() + .collect::() .with_precision_and_scale(20, 3) .unwrap(); let array_ref = Arc::new(array); diff --git a/datafusion/core/src/physical_plan/repartition.rs b/datafusion/core/src/physical_plan/repartition.rs index 82efe2c4fdb..552e1820a73 100644 --- a/datafusion/core/src/physical_plan/repartition.rs +++ b/datafusion/core/src/physical_plan/repartition.rs @@ -156,8 +156,7 @@ impl BatchPartitioner { for (index, hash) in hash_buffer.iter().enumerate() { indices[(*hash % *partitions as u64) as usize] - .append_value(index as u64) - .unwrap(); + .append_value(index as u64); } for (partition, mut indices) in indices.into_iter().enumerate() { diff --git a/datafusion/core/src/physical_plan/sort_merge_join.rs b/datafusion/core/src/physical_plan/sort_merge_join.rs index 39ff4c2ecaa..4eb1616c9a6 100644 --- a/datafusion/core/src/physical_plan/sort_merge_join.rs +++ b/datafusion/core/src/physical_plan/sort_merge_join.rs @@ -353,7 +353,7 @@ impl StreamedBatch { &mut self, buffered_batch_idx: Option, buffered_idx: Option, - ) -> ArrowResult<()> { + ) { if self.output_indices.is_empty() || self.buffered_batch_idx != buffered_batch_idx { self.output_indices.push(StreamedJoinedChunk { @@ -365,16 +365,12 @@ impl StreamedBatch { }; let current_chunk = self.output_indices.last_mut().unwrap(); - current_chunk - .streamed_indices - .append_value(self.idx as u64)?; + current_chunk.streamed_indices.append_value(self.idx as u64); if let Some(idx) = buffered_idx { - current_chunk.buffered_indices.append_value(idx as u64)?; + current_chunk.buffered_indices.append_value(idx as u64); } else { - current_chunk.buffered_indices.append_null()?; + current_chunk.buffered_indices.append_null(); } - - Ok(()) } } @@ -808,7 +804,7 @@ impl SMJStream { self.streamed_batch.append_output_pair( Some(self.buffered_data.scanning_batch_idx), Some(scanning_idx), - )?; + ); } else { self.buffered_data .scanning_batch_mut() @@ -832,7 +828,7 @@ impl SMJStream { }; self.streamed_batch - .append_output_pair(scanning_batch_idx, None)?; + .append_output_pair(scanning_batch_idx, None); self.output_size += 1; self.buffered_data.scanning_finish(); self.streamed_joined = true; @@ -1102,7 +1098,7 @@ fn compare_join_arrays( DataType::Float64 => compare_value!(Float64Array), DataType::Utf8 => compare_value!(StringArray), DataType::LargeUtf8 => compare_value!(LargeStringArray), - DataType::Decimal(..) => compare_value!(DecimalArray), + DataType::Decimal(..) => compare_value!(Decimal128Array), DataType::Timestamp(time_unit, None) => match time_unit { TimeUnit::Second => compare_value!(TimestampSecondArray), TimeUnit::Millisecond => compare_value!(TimestampMillisecondArray), @@ -1168,7 +1164,7 @@ fn is_join_arrays_equal( DataType::Float64 => compare_value!(Float64Array), DataType::Utf8 => compare_value!(StringArray), DataType::LargeUtf8 => compare_value!(LargeStringArray), - DataType::Decimal(..) => compare_value!(DecimalArray), + DataType::Decimal(..) => compare_value!(Decimal128Array), DataType::Timestamp(time_unit, None) => match time_unit { TimeUnit::Second => compare_value!(TimestampSecondArray), TimeUnit::Millisecond => compare_value!(TimestampMillisecondArray), diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rs index 598d818796c..6c107f66939 100644 --- a/datafusion/core/src/test/mod.rs +++ b/datafusion/core/src/test/mod.rs @@ -27,7 +27,7 @@ use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; use crate::test::object_store::local_unpartitioned_file; use crate::test_util::aggr_test_schema; use array::{Array, ArrayRef}; -use arrow::array::{self, DecimalBuilder, Int32Array}; +use arrow::array::{self, Decimal128Builder, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use futures::{Future, FutureExt}; @@ -200,7 +200,7 @@ pub fn table_with_decimal() -> Arc { } fn make_decimal() -> RecordBatch { - let mut decimal_builder = DecimalBuilder::new(20, 10, 3); + let mut decimal_builder = Decimal128Builder::new(20, 10, 3); for i in 110000..110010 { decimal_builder.append_value(i as i128).unwrap(); } diff --git a/datafusion/core/tests/provider_filter_pushdown.rs b/datafusion/core/tests/provider_filter_pushdown.rs index 9b9ba84d3a5..3ebfec996e6 100644 --- a/datafusion/core/tests/provider_filter_pushdown.rs +++ b/datafusion/core/tests/provider_filter_pushdown.rs @@ -36,7 +36,7 @@ use std::sync::Arc; fn create_batch(value: i32, num_rows: usize) -> Result { let mut builder = Int32Builder::new(num_rows); for _ in 0..num_rows { - builder.append_value(value)?; + builder.append_value(value); } Ok(RecordBatch::try_new( diff --git a/datafusion/core/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rs index 186584aebd1..7f235b1ba62 100644 --- a/datafusion/core/tests/sql/mod.rs +++ b/datafusion/core/tests/sql/mod.rs @@ -290,7 +290,7 @@ fn create_hashjoin_datatype_context() -> Result { None, ])), Arc::new( - DecimalArray::from_iter_values([123, 45600, 78900, -12312]) + Decimal128Array::from_iter_values([123, 45600, 78900, -12312]) .with_precision_and_scale(5, 2) .unwrap(), ), @@ -323,7 +323,7 @@ fn create_hashjoin_datatype_context() -> Result { None, ])), Arc::new( - DecimalArray::from_iter_values([-12312, 10000000, 0, 78900]) + Decimal128Array::from_iter_values([-12312, 10000000, 0, 78900]) .with_precision_and_scale(10, 2) .unwrap(), ), @@ -571,25 +571,25 @@ async fn register_tpch_csv_data( match field.data_type() { DataType::Utf8 => { let sb = col.as_any_mut().downcast_mut::().unwrap(); - sb.append_value(val)?; + sb.append_value(val); } DataType::Date32 => { let sb = col.as_any_mut().downcast_mut::().unwrap(); let dt = NaiveDate::parse_from_str(val.trim(), "%Y-%m-%d").unwrap(); let dt = dt.sub(NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32; - sb.append_value(dt)?; + sb.append_value(dt); } DataType::Int32 => { let sb = col.as_any_mut().downcast_mut::().unwrap(); - sb.append_value(val.trim().parse().unwrap())?; + sb.append_value(val.trim().parse().unwrap()); } DataType::Int64 => { let sb = col.as_any_mut().downcast_mut::().unwrap(); - sb.append_value(val.trim().parse().unwrap())?; + sb.append_value(val.trim().parse().unwrap()); } DataType::Float64 => { let sb = col.as_any_mut().downcast_mut::().unwrap(); - sb.append_value(val.trim().parse().unwrap())?; + sb.append_value(val.trim().parse().unwrap()); } _ => Err(DataFusionError::Plan(format!( "Not implemented: {}", @@ -827,7 +827,7 @@ pub fn table_with_decimal() -> Arc { } fn make_decimal() -> RecordBatch { - let mut decimal_builder = DecimalBuilder::new(20, 10, 3); + let mut decimal_builder = Decimal128Builder::new(20, 10, 3); for i in 110000..110010 { decimal_builder.append_value(i as i128).unwrap(); } diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs index 9119e21e8bf..5056fa02f77 100644 --- a/datafusion/core/tests/sql/select.rs +++ b/datafusion/core/tests/sql/select.rs @@ -517,9 +517,9 @@ async fn query_get_indexed_field() -> Result<()> { for int_vec in vec![vec![0, 1, 2], vec![4, 5, 6], vec![7, 8, 9]] { let builder = lb.values(); for int in int_vec { - builder.append_value(int).unwrap(); + builder.append_value(int); } - lb.append(true).unwrap(); + lb.append(true); } let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(lb.finish())])?; @@ -568,11 +568,11 @@ async fn query_nested_get_indexed_field() -> Result<()> { for int_vec in int_vec_vec { let builder = nested_builder.values(); for int in int_vec { - builder.append_value(int).unwrap(); + builder.append_value(int); } - nested_builder.append(true).unwrap(); + nested_builder.append(true); } - lb.append(true).unwrap(); + lb.append(true); } let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(lb.finish())])?; @@ -628,9 +628,9 @@ async fn query_nested_get_indexed_field_on_struct() -> Result<()> { for int_vec in vec![vec![0, 1, 2, 3], vec![4, 5, 6, 7], vec![8, 9, 10, 11]] { let lb = sb.field_builder::>(0).unwrap(); for int in int_vec { - lb.values().append_value(int).unwrap(); + lb.values().append_value(int); } - lb.append(true).unwrap(); + lb.append(true); } let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(sb.finish())])?; let table = MemTable::try_new(schema, vec![vec![data]])?; diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index ce65717d2ff..b91e9c470c6 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -36,6 +36,6 @@ path = "src/lib.rs" [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "18.0.0", features = ["prettyprint"] } +arrow = { version = "19.0.0", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "10.0.0" } sqlparser = "0.18" diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml index eb539cbfcb1..e30895a4d95 100644 --- a/datafusion/jit/Cargo.toml +++ b/datafusion/jit/Cargo.toml @@ -36,7 +36,7 @@ path = "src/lib.rs" jit = [] [dependencies] -arrow = { version = "18.0.0" } +arrow = { version = "19.0.0" } cranelift = "0.86.1" cranelift-jit = "0.86.1" cranelift-module = "0.86.1" diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 24d2f1812d5..58c6b2d0d8e 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow = { version = "18.0.0", features = ["prettyprint"] } +arrow = { version = "19.0.0", features = ["prettyprint"] } async-trait = "0.1.41" chrono = { version = "0.4", default-features = false } datafusion-common = { path = "../common", version = "10.0.0" } diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index e5abf12af79..e32422f98f6 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"] [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "18.0.0", features = ["prettyprint"] } +arrow = { version = "19.0.0", features = ["prettyprint"] } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false } diff --git a/datafusion/physical-expr/src/aggregate/average.rs b/datafusion/physical-expr/src/aggregate/average.rs index 3eee84bb5f5..1b1d995257b 100644 --- a/datafusion/physical-expr/src/aggregate/average.rs +++ b/datafusion/physical-expr/src/aggregate/average.rs @@ -292,7 +292,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..7) .map(Some) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); @@ -310,7 +310,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(|i| if i == 2 { None } else { Some(i) }) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); generic_test_op!( @@ -328,7 +328,7 @@ mod tests { let array: ArrayRef = Arc::new( std::iter::repeat(None) .take(6) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); generic_test_op!( diff --git a/datafusion/physical-expr/src/aggregate/count_distinct.rs b/datafusion/physical-expr/src/aggregate/count_distinct.rs index 83c50b4cd5f..744d9b90d9b 100644 --- a/datafusion/physical-expr/src/aggregate/count_distinct.rs +++ b/datafusion/physical-expr/src/aggregate/count_distinct.rs @@ -267,22 +267,20 @@ mod tests { Some(values) => { for value in values.iter() { match value { - Some(v) => builder.values().append_value((*v).into())?, - None => builder.values().append_null()?, + Some(v) => builder.values().append_value((*v).into()), + None => builder.values().append_null(), } } - builder.append(true)?; + builder.append(true); } None => { - builder.append(false)?; + builder.append(false); } } } - let array = Arc::new(builder.finish()) as ArrayRef; - - Ok(array) as Result + Arc::new(builder.finish()) as ArrayRef }}; } @@ -696,7 +694,7 @@ mod tests { Some(vec![Some(-2_i32), Some(-3_i32)]), ], Int32Builder - )?; + ); let state_in2 = build_list!( vec![ @@ -704,7 +702,7 @@ mod tests { Some(vec![Some(5_u64), Some(7_u64)]), ], UInt64Builder - )?; + ); let (states, result) = run_merge_batch(&[state_in1, state_in2])?; diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs b/datafusion/physical-expr/src/aggregate/min_max.rs index 811aa473c17..bd56973b167 100644 --- a/datafusion/physical-expr/src/aggregate/min_max.rs +++ b/datafusion/physical-expr/src/aggregate/min_max.rs @@ -41,7 +41,7 @@ use datafusion_expr::Accumulator; use crate::aggregate::row_accumulator::RowAccumulator; use crate::expressions::format_state_name; use arrow::array::Array; -use arrow::array::DecimalArray; +use arrow::array::Decimal128Array; use datafusion_row::accessor::RowAccessor; // Min/max aggregation can take Dictionary encode input but always produces unpacked @@ -176,7 +176,7 @@ macro_rules! typed_min_max_batch_decimal128 { if null_count == $VALUES.len() { ScalarValue::Decimal128(None, *$PRECISION, *$SCALE) } else { - let array = $VALUES.as_any().downcast_ref::().unwrap(); + let array = $VALUES.as_any().downcast_ref::().unwrap(); if null_count == 0 { // there is no null value let mut result = array.value(0); @@ -777,7 +777,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(Some) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); @@ -788,7 +788,7 @@ mod tests { let array: ArrayRef = Arc::new( std::iter::repeat(None) .take(0) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); let result = min_batch(&array)?; @@ -798,7 +798,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(Some) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); generic_test_op!( @@ -816,7 +816,7 @@ mod tests { let array: ArrayRef = Arc::new( std::iter::repeat(None) .take(6) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); generic_test_op!( @@ -834,7 +834,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(|i| if i == 2 { None } else { Some(i) }) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); @@ -867,7 +867,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(Some) - .collect::() + .collect::() .with_precision_and_scale(10, 5)?, ); let result = max_batch(&array)?; @@ -877,7 +877,7 @@ mod tests { let array: ArrayRef = Arc::new( std::iter::repeat(None) .take(0) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); let result = max_batch(&array)?; @@ -887,7 +887,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(Some) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); generic_test_op!( @@ -904,7 +904,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(|i| if i == 2 { None } else { Some(i) }) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); generic_test_op!( @@ -921,7 +921,7 @@ mod tests { let array: ArrayRef = Arc::new( std::iter::repeat(None) .take(6) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); generic_test_op!( diff --git a/datafusion/physical-expr/src/aggregate/sum.rs b/datafusion/physical-expr/src/aggregate/sum.rs index ee8b224fed7..866e90f1eac 100644 --- a/datafusion/physical-expr/src/aggregate/sum.rs +++ b/datafusion/physical-expr/src/aggregate/sum.rs @@ -37,7 +37,7 @@ use datafusion_expr::Accumulator; use crate::aggregate::row_accumulator::RowAccumulator; use crate::expressions::format_state_name; use arrow::array::Array; -use arrow::array::DecimalArray; +use arrow::array::Decimal128Array; use arrow::compute::cast; use datafusion_row::accessor::RowAccessor; @@ -157,7 +157,7 @@ fn sum_decimal_batch( precision: &usize, scale: &usize, ) -> Result { - let array = values.as_any().downcast_ref::().unwrap(); + let array = values.as_any().downcast_ref::().unwrap(); if array.null_count() == array.len() { return Ok(ScalarValue::Decimal128(None, *precision, *scale)); @@ -541,7 +541,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(Some) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); let result = sum_batch(&array, &DataType::Decimal(10, 0))?; @@ -551,7 +551,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(Some) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); @@ -576,7 +576,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(|i| if i == 2 { None } else { Some(i) }) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); let result = sum_batch(&array, &DataType::Decimal(10, 0))?; @@ -586,7 +586,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(|i| if i == 2 { None } else { Some(i) }) - .collect::() + .collect::() .with_precision_and_scale(35, 0)?, ); generic_test_op!( @@ -610,7 +610,7 @@ mod tests { let array: ArrayRef = Arc::new( std::iter::repeat(None) .take(6) - .collect::() + .collect::() .with_precision_and_scale(10, 0)?, ); let result = sum_batch(&array, &DataType::Decimal(10, 0))?; diff --git a/datafusion/physical-expr/src/aggregate/sum_distinct.rs b/datafusion/physical-expr/src/aggregate/sum_distinct.rs index ddca3577598..a64b4b497c1 100644 --- a/datafusion/physical-expr/src/aggregate/sum_distinct.rs +++ b/datafusion/physical-expr/src/aggregate/sum_distinct.rs @@ -283,7 +283,7 @@ mod tests { let array: ArrayRef = Arc::new( (1..6) .map(|i| if i == 2 { None } else { Some(i % 2) }) - .collect::() + .collect::() .with_precision_and_scale(35, 0)?, ); generic_test_sum_distinct!( diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index 45ec881ba68..84e6732e399 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -48,14 +48,14 @@ macro_rules! array { for index in 0..args[0].len() { for arg in &args { if arg.is_null(index) { - builder.values().append_null()?; + builder.values().append_null(); } else { - builder.values().append_value(arg.value(index))?; + builder.values().append_value(arg.value(index)); } } - builder.append(true)?; + builder.append(true); } - Ok(Arc::new(builder.finish())) + Arc::new(builder.finish()) }}; } @@ -67,7 +67,7 @@ fn array_array(args: &[ArrayRef]) -> Result { )); } - match args[0].data_type() { + let res = match args[0].data_type() { DataType::Utf8 => array!(args, StringArray, StringBuilder), DataType::LargeUtf8 => array!(args, LargeStringArray, LargeStringBuilder), DataType::Boolean => array!(args, BooleanArray, BooleanBuilder), @@ -81,11 +81,14 @@ fn array_array(args: &[ArrayRef]) -> Result { DataType::UInt16 => array!(args, UInt16Array, UInt16Builder), DataType::UInt32 => array!(args, UInt32Array, UInt32Builder), DataType::UInt64 => array!(args, UInt64Array, UInt64Builder), - data_type => Err(DataFusionError::NotImplemented(format!( - "Array is not implemented for type '{:?}'.", - data_type - ))), - } + data_type => { + return Err(DataFusionError::NotImplemented(format!( + "Array is not implemented for type '{:?}'.", + data_type + ))) + } + }; + Ok(res) } /// put values in an array. diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs index 1e1cfd07f3b..eba3eca0376 100644 --- a/datafusion/physical-expr/src/datetime_expressions.rs +++ b/datafusion/physical-expr/src/datetime_expressions.rs @@ -387,11 +387,11 @@ mod tests { let mut string_builder = StringBuilder::new(2); let mut ts_builder = TimestampNanosecondArray::builder(2); - string_builder.append_value("2020-09-08T13:42:29.190855Z")?; - ts_builder.append_value(1599572549190855000)?; + string_builder.append_value("2020-09-08T13:42:29.190855Z"); + ts_builder.append_value(1599572549190855000); - string_builder.append_null()?; - ts_builder.append_null()?; + string_builder.append_null(); + ts_builder.append_null(); let expected_timestamps = &ts_builder.finish() as &dyn Array; let string_array = @@ -508,7 +508,7 @@ mod tests { // that we get an error. let mut builder = Int64Array::builder(1); - builder.append_value(1)?; + builder.append_value(1); let int64array = ColumnarValue::Array(Arc::new(builder.finish())); let expected_err = diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 93bcfae2ec7..b7492445b05 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -123,7 +123,7 @@ fn is_not_distinct_from_bool( /// Creates an BooleanArray the same size as `left`, /// applying `op` to all non-null elements of left fn compare_decimal_scalar( - left: &DecimalArray, + left: &Decimal128Array, right: i128, op: F, ) -> Result @@ -139,8 +139,8 @@ where /// Creates an BooleanArray the same size as `left`, /// by applying `op` to all non-null elements of left and right fn compare_decimal( - left: &DecimalArray, - right: &DecimalArray, + left: &Decimal128Array, + right: &Decimal128Array, op: F, ) -> Result where @@ -160,62 +160,68 @@ where } pub(super) fn eq_decimal_scalar( - left: &DecimalArray, + left: &Decimal128Array, right: i128, ) -> Result { compare_decimal_scalar(left, right, |left, right| left == right) } pub(super) fn eq_decimal( - left: &DecimalArray, - right: &DecimalArray, + left: &Decimal128Array, + right: &Decimal128Array, ) -> Result { compare_decimal(left, right, |left, right| left == right) } -fn neq_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn neq_decimal_scalar(left: &Decimal128Array, right: i128) -> Result { compare_decimal_scalar(left, right, |left, right| left != right) } -fn neq_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn neq_decimal(left: &Decimal128Array, right: &Decimal128Array) -> Result { compare_decimal(left, right, |left, right| left != right) } -fn lt_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn lt_decimal_scalar(left: &Decimal128Array, right: i128) -> Result { compare_decimal_scalar(left, right, |left, right| left < right) } -fn lt_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn lt_decimal(left: &Decimal128Array, right: &Decimal128Array) -> Result { compare_decimal(left, right, |left, right| left < right) } -fn lt_eq_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn lt_eq_decimal_scalar(left: &Decimal128Array, right: i128) -> Result { compare_decimal_scalar(left, right, |left, right| left <= right) } -fn lt_eq_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn lt_eq_decimal( + left: &Decimal128Array, + right: &Decimal128Array, +) -> Result { compare_decimal(left, right, |left, right| left <= right) } -fn gt_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn gt_decimal_scalar(left: &Decimal128Array, right: i128) -> Result { compare_decimal_scalar(left, right, |left, right| left > right) } -fn gt_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn gt_decimal(left: &Decimal128Array, right: &Decimal128Array) -> Result { compare_decimal(left, right, |left, right| left > right) } -fn gt_eq_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn gt_eq_decimal_scalar(left: &Decimal128Array, right: i128) -> Result { compare_decimal_scalar(left, right, |left, right| left >= right) } -fn gt_eq_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn gt_eq_decimal( + left: &Decimal128Array, + right: &Decimal128Array, +) -> Result { compare_decimal(left, right, |left, right| left >= right) } fn is_distinct_from_decimal( - left: &DecimalArray, - right: &DecimalArray, + left: &Decimal128Array, + right: &Decimal128Array, ) -> Result { Ok(left .iter() @@ -229,8 +235,8 @@ fn is_distinct_from_decimal( } fn is_not_distinct_from_decimal( - left: &DecimalArray, - right: &DecimalArray, + left: &Decimal128Array, + right: &Decimal128Array, ) -> Result { Ok(left .iter() @@ -243,13 +249,13 @@ fn is_not_distinct_from_decimal( .collect()) } -/// Creates an DecimalArray the same size as `left`, +/// Creates an Decimal128Array the same size as `left`, /// by applying `op` to all non-null elements of left and right fn arith_decimal( - left: &DecimalArray, - right: &DecimalArray, + left: &Decimal128Array, + right: &Decimal128Array, op: F, -) -> Result +) -> Result where F: Fn(i128, i128) -> Result, { @@ -266,10 +272,10 @@ where } fn arith_decimal_scalar( - left: &DecimalArray, + left: &Decimal128Array, right: i128, op: F, -) -> Result +) -> Result where F: Fn(i128, i128) -> Result, { @@ -284,38 +290,53 @@ where .collect() } -fn add_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn add_decimal( + left: &Decimal128Array, + right: &Decimal128Array, +) -> Result { let array = arith_decimal(left, right, |left, right| Ok(left + right))? .with_precision_and_scale(left.precision(), left.scale())?; Ok(array) } -fn add_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn add_decimal_scalar(left: &Decimal128Array, right: i128) -> Result { let array = arith_decimal_scalar(left, right, |left, right| Ok(left + right))? .with_precision_and_scale(left.precision(), left.scale())?; Ok(array) } -fn subtract_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn subtract_decimal( + left: &Decimal128Array, + right: &Decimal128Array, +) -> Result { let array = arith_decimal(left, right, |left, right| Ok(left - right))? .with_precision_and_scale(left.precision(), left.scale())?; Ok(array) } -fn subtract_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn subtract_decimal_scalar( + left: &Decimal128Array, + right: i128, +) -> Result { let array = arith_decimal_scalar(left, right, |left, right| Ok(left - right))? .with_precision_and_scale(left.precision(), left.scale())?; Ok(array) } -fn multiply_decimal(left: &DecimalArray, right: &DecimalArray) -> Result { +fn multiply_decimal( + left: &Decimal128Array, + right: &Decimal128Array, +) -> Result { let divide = 10_i128.pow(left.scale() as u32); let array = arith_decimal(left, right, |left, right| Ok(left * right / divide))? .with_precision_and_scale(left.precision(), left.scale())?; Ok(array) } -fn multiply_decimal_scalar(left: &DecimalArray, right: i128) -> Result { +fn multiply_decimal_scalar( + left: &Decimal128Array, + right: i128, +) -> Result { let divide = 10_i128.pow(left.scale() as u32); let array = arith_decimal_scalar(left, right, |left, right| Ok(left * right / divide))? @@ -323,7 +344,10 @@ fn multiply_decimal_scalar(left: &DecimalArray, right: i128) -> Result Result { +fn divide_decimal( + left: &Decimal128Array, + right: &Decimal128Array, +) -> Result { let mul = 10_f64.powi(left.scale() as i32); let array = arith_decimal(left, right, |left, right| { let l_value = left as f64; @@ -335,7 +359,7 @@ fn divide_decimal(left: &DecimalArray, right: &DecimalArray) -> Result Result { +fn divide_decimal_scalar(left: &Decimal128Array, right: i128) -> Result { let mul = 10_f64.powi(left.scale() as i32); let array = arith_decimal_scalar(left, right, |left, right| { let l_value = left as f64; @@ -347,7 +371,10 @@ fn divide_decimal_scalar(left: &DecimalArray, right: i128) -> Result Result { +fn modulus_decimal( + left: &Decimal128Array, + right: &Decimal128Array, +) -> Result { let array = arith_decimal(left, right, |left, right| { if right == 0 { Err(DataFusionError::ArrowError(DivideByZero)) @@ -359,7 +386,10 @@ fn modulus_decimal(left: &DecimalArray, right: &DecimalArray) -> Result Result { +fn modulus_decimal_scalar( + left: &Decimal128Array, + right: i128, +) -> Result { if right == 0 { return Err(DataFusionError::ArrowError(DivideByZero)); } @@ -794,7 +824,7 @@ macro_rules! binary_primitive_array_op { match $LEFT.data_type() { // TODO support decimal type // which is not the primitive type - DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, DecimalArray), + DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, Decimal128Array), DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array), DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array), DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array), @@ -819,7 +849,7 @@ macro_rules! binary_primitive_array_op { macro_rules! binary_primitive_array_op_scalar { ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{ let result: Result> = match $LEFT.data_type() { - DataType::Decimal(_,_) => compute_decimal_op_scalar!($LEFT, $RIGHT, $OP, DecimalArray), + DataType::Decimal(_,_) => compute_decimal_op_scalar!($LEFT, $RIGHT, $OP, Decimal128Array), DataType::Int8 => compute_op_scalar!($LEFT, $RIGHT, $OP, Int8Array), DataType::Int16 => compute_op_scalar!($LEFT, $RIGHT, $OP, Int16Array), DataType::Int32 => compute_op_scalar!($LEFT, $RIGHT, $OP, Int32Array), @@ -846,7 +876,7 @@ macro_rules! binary_array_op { ($LEFT:expr, $RIGHT:expr, $OP:ident) => {{ match $LEFT.data_type() { DataType::Null => compute_null_op!($LEFT, $RIGHT, $OP, NullArray), - DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, DecimalArray), + DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, Decimal128Array), DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array), DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array), DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array), @@ -1079,7 +1109,7 @@ macro_rules! binary_array_op_dyn_scalar { let result: Result> = match right { ScalarValue::Boolean(b) => compute_bool_op_dyn_scalar!($LEFT, b, $OP, $OP_TYPE), - ScalarValue::Decimal128(..) => compute_decimal_op_scalar!($LEFT, right, $OP, DecimalArray), + ScalarValue::Decimal128(..) => compute_decimal_op_scalar!($LEFT, right, $OP, Decimal128Array), ScalarValue::Utf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE), ScalarValue::LargeUtf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE), ScalarValue::Int8(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE), @@ -1809,7 +1839,7 @@ mod tests { let mut dict_builder = StringDictionaryBuilder::new(keys_builder, values_builder); dict_builder.append("one")?; - dict_builder.append_null()?; + dict_builder.append_null(); dict_builder.append("three")?; dict_builder.append("four")?; let dict_array = dict_builder.finish(); @@ -2629,19 +2659,19 @@ mod tests { array: &[Option], precision: usize, scale: usize, - ) -> Result { - let mut decimal_builder = DecimalBuilder::new(array.len(), precision, scale); + ) -> Decimal128Array { + let mut decimal_builder = Decimal128Builder::new(array.len(), precision, scale); for value in array { match value { None => { - decimal_builder.append_null()?; + decimal_builder.append_null(); } Some(v) => { - decimal_builder.append_value(*v)?; + decimal_builder.append_value(*v).expect("valid value"); } } } - Ok(decimal_builder.finish()) + decimal_builder.finish() } #[test] @@ -2656,7 +2686,7 @@ mod tests { ], 25, 3, - )?; + ); // eq: array = i128 let result = eq_decimal_scalar(&decimal_array, value_i128)?; assert_eq!( @@ -2704,7 +2734,7 @@ mod tests { ], 25, 3, - )?; + ); // eq: left == right let result = eq_decimal(&left_decimal_array, &right_decimal_array)?; assert_eq!( @@ -2860,7 +2890,7 @@ mod tests { ], 10, 0, - )?) as ArrayRef; + )) as ArrayRef; let int64_array = Arc::new(Int64Array::from(vec![ Some(value), @@ -2903,7 +2933,7 @@ mod tests { ], 10, 2, - )?) as ArrayRef; + )) as ArrayRef; let float64_array = Arc::new(Float64Array::from(vec![ Some(1.23), Some(1.22), @@ -2982,7 +3012,7 @@ mod tests { ], 25, 3, - )?; + ); let right_decimal_array = create_decimal_array( &[ Some(value_i128), @@ -2992,59 +3022,59 @@ mod tests { ], 25, 3, - )?; + ); // add let result = add_decimal(&left_decimal_array, &right_decimal_array)?; let expect = - create_decimal_array(&[Some(246), None, Some(245), Some(247)], 25, 3)?; + create_decimal_array(&[Some(246), None, Some(245), Some(247)], 25, 3); assert_eq!(expect, result); let result = add_decimal_scalar(&left_decimal_array, 10)?; let expect = - create_decimal_array(&[Some(133), None, Some(132), Some(134)], 25, 3)?; + create_decimal_array(&[Some(133), None, Some(132), Some(134)], 25, 3); assert_eq!(expect, result); // subtract let result = subtract_decimal(&left_decimal_array, &right_decimal_array)?; - let expect = create_decimal_array(&[Some(0), None, Some(-1), Some(1)], 25, 3)?; + let expect = create_decimal_array(&[Some(0), None, Some(-1), Some(1)], 25, 3); assert_eq!(expect, result); let result = subtract_decimal_scalar(&left_decimal_array, 10)?; let expect = - create_decimal_array(&[Some(113), None, Some(112), Some(114)], 25, 3)?; + create_decimal_array(&[Some(113), None, Some(112), Some(114)], 25, 3); assert_eq!(expect, result); // multiply let result = multiply_decimal(&left_decimal_array, &right_decimal_array)?; - let expect = create_decimal_array(&[Some(15), None, Some(15), Some(15)], 25, 3)?; + let expect = create_decimal_array(&[Some(15), None, Some(15), Some(15)], 25, 3); assert_eq!(expect, result); let result = multiply_decimal_scalar(&left_decimal_array, 10)?; - let expect = create_decimal_array(&[Some(1), None, Some(1), Some(1)], 25, 3)?; + let expect = create_decimal_array(&[Some(1), None, Some(1), Some(1)], 25, 3); assert_eq!(expect, result); // divide let left_decimal_array = create_decimal_array( &[Some(1234567), None, Some(1234567), Some(1234567)], 25, 3, - )?; + ); let right_decimal_array = - create_decimal_array(&[Some(10), Some(100), Some(55), Some(-123)], 25, 3)?; + create_decimal_array(&[Some(10), Some(100), Some(55), Some(-123)], 25, 3); let result = divide_decimal(&left_decimal_array, &right_decimal_array)?; let expect = create_decimal_array( &[Some(123456700), None, Some(22446672), Some(-10037130)], 25, 3, - )?; + ); assert_eq!(expect, result); let result = divide_decimal_scalar(&left_decimal_array, 10)?; let expect = create_decimal_array( &[Some(123456700), None, Some(123456700), Some(123456700)], 25, 3, - )?; + ); assert_eq!(expect, result); // modulus let result = modulus_decimal(&left_decimal_array, &right_decimal_array)?; - let expect = create_decimal_array(&[Some(7), None, Some(37), Some(16)], 25, 3)?; + let expect = create_decimal_array(&[Some(7), None, Some(37), Some(16)], 25, 3); assert_eq!(expect, result); let result = modulus_decimal_scalar(&left_decimal_array, 10)?; - let expect = create_decimal_array(&[Some(7), None, Some(7), Some(7)], 25, 3)?; + let expect = create_decimal_array(&[Some(7), None, Some(7), Some(7)], 25, 3); assert_eq!(expect, result); Ok(()) @@ -3083,7 +3113,7 @@ mod tests { ], 10, 2, - )?) as ArrayRef; + )) as ArrayRef; let int32_array = Arc::new(Int32Array::from(vec![ Some(123), Some(122), @@ -3096,7 +3126,7 @@ mod tests { &[Some(12423), None, Some(12422), Some(12524)], 13, 2, - )?) as ArrayRef; + )) as ArrayRef; apply_arithmetic_op( &schema, &int32_array, @@ -3115,7 +3145,7 @@ mod tests { &[Some(-12177), None, Some(-12178), Some(-12276)], 13, 2, - )?) as ArrayRef; + )) as ArrayRef; apply_arithmetic_op( &schema, &int32_array, @@ -3130,7 +3160,7 @@ mod tests { &[Some(15129), None, Some(15006), Some(15376)], 21, 2, - )?) as ArrayRef; + )) as ArrayRef; apply_arithmetic_op( &schema, &int32_array, @@ -3153,7 +3183,7 @@ mod tests { ], 23, 11, - )?) as ArrayRef; + )) as ArrayRef; apply_arithmetic_op( &schema, &int32_array, @@ -3171,7 +3201,7 @@ mod tests { &[Some(000), None, Some(100), Some(000)], 10, 2, - )?) as ArrayRef; + )) as ArrayRef; apply_arithmetic_op( &schema, &int32_array, diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index de1909676e3..72503bbdb44 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -161,7 +161,7 @@ mod tests { use crate::expressions::col; use arrow::{ array::{ - Array, BasicDecimalArray, DecimalArray, Float32Array, Float64Array, + Array, BasicDecimalArray, Decimal128Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, StringArray, Time64NanosecondArray, TimestampNanosecondArray, UInt32Array, }, @@ -271,12 +271,18 @@ mod tests { #[test] fn test_cast_decimal_to_decimal() -> Result<()> { - let array = vec![1234, 2222, 3, 4000, 5000]; + let array = vec![ + Some(1234), + Some(2222), + Some(3), + Some(4000), + Some(5000), + None, + ]; let decimal_array = array .iter() - .map(|v| Some(*v)) - .collect::() + .collect::() .with_precision_and_scale(10, 3)?; // closure that converts to i128 @@ -285,7 +291,7 @@ mod tests { generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 3), - DecimalArray, + Decimal128Array, DataType::Decimal(20, 6), vec![ Some(convert(1_234_000)), @@ -300,15 +306,14 @@ mod tests { let decimal_array = array .iter() - .map(|v| Some(*v)) - .collect::() + .collect::() .with_precision_and_scale(10, 3)?; let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes()); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 3), - DecimalArray, + Decimal128Array, DataType::Decimal(10, 2), vec![ Some(convert(123)), @@ -330,7 +335,7 @@ mod tests { // decimal to i8 let decimal_array = array .iter() - .collect::() + .collect::() .with_precision_and_scale(10, 0)?; generic_decimal_to_other_test_cast!( decimal_array, @@ -351,7 +356,7 @@ mod tests { // decimal to i16 let decimal_array = array .iter() - .collect::() + .collect::() .with_precision_and_scale(10, 0)?; generic_decimal_to_other_test_cast!( decimal_array, @@ -372,7 +377,7 @@ mod tests { // decimal to i32 let decimal_array = array .iter() - .collect::() + .collect::() .with_precision_and_scale(10, 0)?; generic_decimal_to_other_test_cast!( decimal_array, @@ -393,7 +398,7 @@ mod tests { // decimal to i64 let decimal_array = array .iter() - .collect::() + .collect::() .with_precision_and_scale(10, 0)?; generic_decimal_to_other_test_cast!( decimal_array, @@ -422,7 +427,7 @@ mod tests { ]; let decimal_array = array .iter() - .collect::() + .collect::() .with_precision_and_scale(10, 3)?; generic_decimal_to_other_test_cast!( decimal_array, @@ -443,7 +448,7 @@ mod tests { // decimal to float64 let decimal_array = array .into_iter() - .collect::() + .collect::() .with_precision_and_scale(20, 6)?; generic_decimal_to_other_test_cast!( decimal_array, @@ -471,7 +476,7 @@ mod tests { Int8Array, DataType::Int8, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(3, 0), vec![ Some(convert(1)), @@ -489,7 +494,7 @@ mod tests { Int16Array, DataType::Int16, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(5, 0), vec![ Some(convert(1)), @@ -507,7 +512,7 @@ mod tests { Int32Array, DataType::Int32, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(10, 0), vec![ Some(convert(1)), @@ -525,7 +530,7 @@ mod tests { Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(20, 0), vec![ Some(convert(1)), @@ -543,7 +548,7 @@ mod tests { Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(20, 2), vec![ Some(convert(100)), @@ -561,7 +566,7 @@ mod tests { Float32Array, DataType::Float32, vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], - DecimalArray, + Decimal128Array, DataType::Decimal(10, 2), vec![ Some(convert(150)), @@ -579,7 +584,7 @@ mod tests { Float64Array, DataType::Float64, vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], - DecimalArray, + Decimal128Array, DataType::Decimal(20, 4), vec![ Some(convert(15000)), diff --git a/datafusion/physical-expr/src/expressions/datetime.rs b/datafusion/physical-expr/src/expressions/datetime.rs index d4486a3ffac..3c59a90b0ea 100644 --- a/datafusion/physical-expr/src/expressions/datetime.rs +++ b/datafusion/physical-expr/src/expressions/datetime.rs @@ -406,7 +406,7 @@ mod tests { fn exercise(dt: &Expr, op: Operator, interval: &Expr) -> Result { let mut builder = Date32Builder::new(1); - builder.append_value(0).unwrap(); + builder.append_value(0); let a: ArrayRef = Arc::new(builder.finish()); let schema = Schema::new(vec![Field::new("a", DataType::Date32, false)]); let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a])?; diff --git a/datafusion/physical-expr/src/expressions/get_indexed_field.rs b/datafusion/physical-expr/src/expressions/get_indexed_field.rs index bb43229d141..40216b45562 100644 --- a/datafusion/physical-expr/src/expressions/get_indexed_field.rs +++ b/datafusion/physical-expr/src/expressions/get_indexed_field.rs @@ -155,9 +155,8 @@ mod tests { None => builder.append_null(), Some(v) => builder.append_value(v), } - .unwrap() } - lb.append(true).unwrap(); + lb.append(true); } lb.finish() @@ -271,9 +270,8 @@ mod tests { match int_value { None => fb.append_null(), Some(v) => fb.append_value(v), - } - .unwrap(); - builder.append(true).unwrap(); + }; + builder.append(true); let lb = builder .field_builder::>(1) .unwrap(); @@ -281,10 +279,9 @@ mod tests { match str_value { None => lb.values().append_null(), Some(v) => lb.values().append_value(v), - } - .unwrap(); + }; } - lb.append(true).unwrap(); + lb.append(true); } builder.finish() } diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 40a253c33fd..899a20835ba 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -292,7 +292,7 @@ fn try_cast_static_filter_to_set( } fn make_list_contains_decimal( - array: &DecimalArray, + array: &Decimal128Array, list: Vec, negated: bool, ) -> ColumnarValue { @@ -319,7 +319,7 @@ fn make_list_contains_decimal( } fn make_set_contains_decimal( - array: &DecimalArray, + array: &Decimal128Array, set: &HashSet, negated: bool, ) -> ColumnarValue { @@ -632,7 +632,7 @@ impl PhysicalExpr for InListExpr { Ok(set_contains_utf8(array, set, self.negated)) } DataType::Decimal(_, _) => { - let array = array.as_any().downcast_ref::().unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); Ok(make_set_contains_decimal(array, set, self.negated)) } datatype => Result::Err(DataFusionError::NotImplemented(format!( @@ -762,7 +762,7 @@ impl PhysicalExpr for InListExpr { } DataType::Decimal(_, _) => { let decimal_array = - array.as_any().downcast_ref::().unwrap(); + array.as_any().downcast_ref::().unwrap(); Ok(make_list_contains_decimal( decimal_array, list_values, @@ -1035,7 +1035,7 @@ mod tests { let schema = Schema::new(vec![Field::new("a", DataType::Decimal(13, 4), true)]); let array = vec![Some(100_0000_i128), None, Some(200_5000_i128)] .into_iter() - .collect::(); + .collect::(); let array = array.with_precision_and_scale(13, 4).unwrap(); let col_a = col("a", &schema)?; let batch = @@ -1281,7 +1281,7 @@ mod tests { let schema = Schema::new(vec![Field::new("a", DataType::Decimal(13, 4), true)]); let array = vec![Some(100_0000_i128), Some(200_5000_i128), None] .into_iter() - .collect::(); + .collect::(); let array = array.with_precision_and_scale(13, 4).unwrap(); let col_a = col("a", &schema)?; let batch = diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs index 2509c1d6b13..5e8cc30feed 100644 --- a/datafusion/physical-expr/src/expressions/try_cast.rs +++ b/datafusion/physical-expr/src/expressions/try_cast.rs @@ -119,7 +119,7 @@ mod tests { use super::*; use crate::expressions::col; use arrow::array::{ - BasicDecimalArray, DecimalArray, DecimalBuilder, StringArray, + BasicDecimalArray, Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray, }; use arrow::util::decimal::{BasicDecimal, Decimal128}; @@ -233,12 +233,12 @@ mod tests { fn test_try_cast_decimal_to_decimal() -> Result<()> { // try cast one decimal data type to another decimal data type let array: Vec = vec![1234, 2222, 3, 4000, 5000]; - let decimal_array = create_decimal_array(&array, 10, 3)?; + let decimal_array = create_decimal_array(&array, 10, 3); let convert = |v: i128| Decimal128::new(20, 6, &v.to_le_bytes()); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 3), - DecimalArray, + Decimal128Array, DataType::Decimal(20, 6), vec![ Some(convert(1_234_000)), @@ -250,12 +250,12 @@ mod tests { ] ); - let decimal_array = create_decimal_array(&array, 10, 3)?; + let decimal_array = create_decimal_array(&array, 10, 3); let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes()); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 3), - DecimalArray, + Decimal128Array, DataType::Decimal(10, 2), vec![ Some(convert(123)), @@ -272,10 +272,10 @@ mod tests { #[test] fn test_try_cast_decimal_to_numeric() -> Result<()> { - // TODO we should add function to create DecimalArray with value and metadata + // TODO we should add function to create Decimal128Array with value and metadata // https://github.com/apache/arrow-rs/issues/1009 let array: Vec = vec![1, 2, 3, 4, 5]; - let decimal_array = create_decimal_array(&array, 10, 0)?; + let decimal_array = create_decimal_array(&array, 10, 0); // decimal to i8 generic_decimal_to_other_test_cast!( decimal_array, @@ -293,7 +293,7 @@ mod tests { ); // decimal to i16 - let decimal_array = create_decimal_array(&array, 10, 0)?; + let decimal_array = create_decimal_array(&array, 10, 0); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 0), @@ -310,7 +310,7 @@ mod tests { ); // decimal to i32 - let decimal_array = create_decimal_array(&array, 10, 0)?; + let decimal_array = create_decimal_array(&array, 10, 0); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 0), @@ -327,7 +327,7 @@ mod tests { ); // decimal to i64 - let decimal_array = create_decimal_array(&array, 10, 0)?; + let decimal_array = create_decimal_array(&array, 10, 0); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 0), @@ -345,7 +345,7 @@ mod tests { // decimal to float32 let array: Vec = vec![1234, 2222, 3, 4000, 5000]; - let decimal_array = create_decimal_array(&array, 10, 3)?; + let decimal_array = create_decimal_array(&array, 10, 3); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(10, 3), @@ -361,7 +361,7 @@ mod tests { ] ); // decimal to float64 - let decimal_array = create_decimal_array(&array, 20, 6)?; + let decimal_array = create_decimal_array(&array, 20, 6); generic_decimal_to_other_test_cast!( decimal_array, DataType::Decimal(20, 6), @@ -388,7 +388,7 @@ mod tests { Int8Array, DataType::Int8, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(3, 0), vec![ Some(convert(1)), @@ -405,7 +405,7 @@ mod tests { Int16Array, DataType::Int16, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(5, 0), vec![ Some(convert(1)), @@ -422,7 +422,7 @@ mod tests { Int32Array, DataType::Int32, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(10, 0), vec![ Some(convert(1)), @@ -439,7 +439,7 @@ mod tests { Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(20, 0), vec![ Some(convert(1)), @@ -456,7 +456,7 @@ mod tests { Int64Array, DataType::Int64, vec![1, 2, 3, 4, 5], - DecimalArray, + Decimal128Array, DataType::Decimal(20, 2), vec![ Some(convert(100)), @@ -473,7 +473,7 @@ mod tests { Float32Array, DataType::Float32, vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], - DecimalArray, + Decimal128Array, DataType::Decimal(10, 2), vec![ Some(convert(150)), @@ -490,7 +490,7 @@ mod tests { Float64Array, DataType::Float64, vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], - DecimalArray, + Decimal128Array, DataType::Decimal(20, 4), vec![ Some(convert(15000)), @@ -581,12 +581,12 @@ mod tests { array: &[i128], precision: usize, scale: usize, - ) -> Result { - let mut decimal_builder = DecimalBuilder::new(array.len(), precision, scale); + ) -> Decimal128Array { + let mut decimal_builder = Decimal128Builder::new(array.len(), precision, scale); for value in array { - decimal_builder.append_value(*value)? + decimal_builder.append_value(*value).expect("valid value"); } - decimal_builder.append_null()?; - Ok(decimal_builder.finish()) + decimal_builder.append_null(); + decimal_builder.finish() } } diff --git a/datafusion/physical-expr/src/regex_expressions.rs b/datafusion/physical-expr/src/regex_expressions.rs index c53c9461553..2705d7ffb58 100644 --- a/datafusion/physical-expr/src/regex_expressions.rs +++ b/datafusion/physical-expr/src/regex_expressions.rs @@ -191,13 +191,13 @@ mod tests { let elem_builder: GenericStringBuilder = GenericStringBuilder::new(0); let mut expected_builder = ListBuilder::new(elem_builder); - expected_builder.values().append_value("a").unwrap(); - expected_builder.append(true).unwrap(); - expected_builder.append(false).unwrap(); - expected_builder.values().append_value("b").unwrap(); - expected_builder.append(true).unwrap(); - expected_builder.append(false).unwrap(); - expected_builder.append(false).unwrap(); + expected_builder.values().append_value("a"); + expected_builder.append(true); + expected_builder.append(false); + expected_builder.values().append_value("b"); + expected_builder.append(true); + expected_builder.append(false); + expected_builder.append(false); let expected = expected_builder.finish(); let re = regexp_match::(&[Arc::new(values), Arc::new(patterns)]).unwrap(); @@ -214,15 +214,15 @@ mod tests { let elem_builder: GenericStringBuilder = GenericStringBuilder::new(0); let mut expected_builder = ListBuilder::new(elem_builder); - expected_builder.values().append_value("a").unwrap(); - expected_builder.append(true).unwrap(); - expected_builder.values().append_value("a").unwrap(); - expected_builder.append(true).unwrap(); - expected_builder.values().append_value("b").unwrap(); - expected_builder.append(true).unwrap(); - expected_builder.values().append_value("b").unwrap(); - expected_builder.append(true).unwrap(); - expected_builder.append(false).unwrap(); + expected_builder.values().append_value("a"); + expected_builder.append(true); + expected_builder.values().append_value("a"); + expected_builder.append(true); + expected_builder.values().append_value("b"); + expected_builder.append(true); + expected_builder.values().append_value("b"); + expected_builder.append(true); + expected_builder.append(false); let expected = expected_builder.finish(); let re = diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index 5c302637493..f63aae7289c 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -37,7 +37,7 @@ default = [] json = ["pbjson", "pbjson-build", "serde", "serde_json"] [dependencies] -arrow = { version = "18.0.0" } +arrow = { version = "19.0.0" } datafusion = { path = "../core", version = "10.0.0" } datafusion-common = { path = "../common", version = "10.0.0" } datafusion-expr = { path = "../expr", version = "10.0.0" } diff --git a/datafusion/proto/src/to_proto.rs b/datafusion/proto/src/to_proto.rs index fd5276ca8bc..60f4079da2b 100644 --- a/datafusion/proto/src/to_proto.rs +++ b/datafusion/proto/src/to_proto.rs @@ -223,6 +223,9 @@ impl From<&DataType> for protobuf::arrow_type::ArrowTypeEnum { whole: *whole as u64, fractional: *fractional as u64, }), + DataType::Decimal256(_, _) => { + unimplemented!("The Decimal256 data type is not yet supported") + } DataType::Map(_, _) => { unimplemented!("The Map data type is not yet supported") } @@ -1237,7 +1240,8 @@ impl TryFrom<&DataType> for protobuf::scalar_type::Datatype { | DataType::Union(_, _, _) | DataType::Dictionary(_, _) | DataType::Map(_, _) - | DataType::Decimal(_, _) => { + | DataType::Decimal(_, _) + | DataType::Decimal256(_, _) => { return Err(Error::invalid_scalar_type(val)); } }; diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml index f2a524d31e7..9309699485d 100644 --- a/datafusion/row/Cargo.toml +++ b/datafusion/row/Cargo.toml @@ -37,7 +37,7 @@ path = "src/lib.rs" jit = ["datafusion-jit"] [dependencies] -arrow = { version = "18.0.0" } +arrow = { version = "19.0.0" } datafusion-common = { path = "../common", version = "10.0.0" } datafusion-jit = { path = "../jit", version = "10.0.0", optional = true } paste = "^1.0" diff --git a/datafusion/row/src/reader.rs b/datafusion/row/src/reader.rs index 7982ca12073..634b814ad35 100644 --- a/datafusion/row/src/reader.rs +++ b/datafusion/row/src/reader.rs @@ -300,9 +300,7 @@ macro_rules! fn_read_field { .as_any_mut() .downcast_mut::<$ARRAY>() .unwrap(); - to.append_option(row.[](col_idx)) - .map_err(DataFusionError::ArrowError) - .unwrap(); + to.append_option(row.[](col_idx)); } pub(crate) fn [](to: &mut Box, col_idx: usize, row: &RowReader) { @@ -310,9 +308,7 @@ macro_rules! fn_read_field { .as_any_mut() .downcast_mut::<$ARRAY>() .unwrap(); - to.append_value(row.[](col_idx)) - .map_err(DataFusionError::ArrowError) - .unwrap(); + to.append_value(row.[](col_idx)); } } }; @@ -340,9 +336,9 @@ pub(crate) fn read_field_binary( ) { let to = to.as_any_mut().downcast_mut::().unwrap(); if row.is_valid_at(col_idx) { - to.append_value(row.get_binary(col_idx)).unwrap(); + to.append_value(row.get_binary(col_idx)); } else { - to.append_null().unwrap(); + to.append_null(); } } @@ -352,9 +348,7 @@ pub(crate) fn read_field_binary_null_free( row: &RowReader, ) { let to = to.as_any_mut().downcast_mut::().unwrap(); - to.append_value(row.get_binary(col_idx)) - .map_err(DataFusionError::ArrowError) - .unwrap(); + to.append_value(row.get_binary(col_idx)); } fn read_field( diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index c3854172f78..db8720d3e47 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -38,7 +38,7 @@ unicode_expressions = [] [dependencies] ahash = { version = "0.7", default-features = false } -arrow = { version = "18.0.0", features = ["prettyprint"] } +arrow = { version = "19.0.0", features = ["prettyprint"] } datafusion-common = { path = "../common", version = "10.0.0" } datafusion-expr = { path = "../expr", version = "10.0.0" } hashbrown = "0.12"