From 3e091f3ed682acd39bff65c116bbdaf0b6627818 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 18 Jul 2022 12:02:57 -0700 Subject: [PATCH] Define Decimal128Builder and Decimal128Array --- arrow/src/array/array.rs | 2 +- arrow/src/array/array_decimal.rs | 79 ++++++++++--------- arrow/src/array/builder/decimal_builder.rs | 34 ++++---- arrow/src/array/builder/mod.rs | 2 +- arrow/src/array/builder/struct_builder.rs | 3 +- arrow/src/array/cast.rs | 6 +- arrow/src/array/data.rs | 6 +- arrow/src/array/equal/mod.rs | 11 +-- arrow/src/array/equal_json.rs | 12 +-- arrow/src/array/iterator.rs | 8 +- arrow/src/array/mod.rs | 12 ++- arrow/src/array/ord.rs | 6 +- arrow/src/array/transform/mod.rs | 12 +-- arrow/src/compute/kernels/cast.rs | 26 +++--- arrow/src/compute/kernels/sort.rs | 6 +- arrow/src/compute/kernels/take.rs | 15 ++-- arrow/src/csv/reader.rs | 7 +- arrow/src/ffi.rs | 14 ++-- arrow/src/util/display.rs | 2 +- arrow/src/util/integration_util.rs | 2 +- arrow/src/util/pretty.rs | 6 +- integration-testing/src/lib.rs | 2 +- .../src/arrow/array_reader/primitive_array.rs | 6 +- parquet/src/arrow/arrow_reader.rs | 4 +- parquet/src/arrow/arrow_writer/mod.rs | 6 +- parquet/src/arrow/buffer/converter.rs | 14 ++-- 26 files changed, 159 insertions(+), 144 deletions(-) diff --git a/arrow/src/array/array.rs b/arrow/src/array/array.rs index f01fa5cc91d..d29cf839d9e 100644 --- a/arrow/src/array/array.rs +++ b/arrow/src/array/array.rs @@ -403,7 +403,7 @@ pub fn make_array(data: ArrayData) -> ArrayRef { dt => panic!("Unexpected dictionary key type {:?}", dt), }, DataType::Null => Arc::new(NullArray::from(data)) as ArrayRef, - DataType::Decimal(_, _) => Arc::new(DecimalArray::from(data)) as ArrayRef, + DataType::Decimal(_, _) => Arc::new(Decimal128Array::from(data)) as ArrayRef, dt => panic!("Unexpected data type {:?}", dt), } } diff --git a/arrow/src/array/array_decimal.rs b/arrow/src/array/array_decimal.rs index 64b2c3b3b19..261d811f9d7 100644 --- a/arrow/src/array/array_decimal.rs +++ b/arrow/src/array/array_decimal.rs @@ -34,17 +34,17 @@ use crate::datatypes::{ use crate::error::{ArrowError, Result}; use crate::util::decimal::{BasicDecimal, Decimal128, Decimal256}; -/// `DecimalArray` stores fixed width decimal numbers, +/// `Decimal128Array` stores fixed width decimal numbers, /// with a fixed precision and scale. /// /// # Examples /// /// ``` -/// use arrow::array::{Array, BasicDecimalArray, DecimalArray}; +/// use arrow::array::{Array, BasicDecimalArray, Decimal128Array}; /// use arrow::datatypes::DataType; /// /// // Create a DecimalArray with the default precision and scale -/// let decimal_array: DecimalArray = vec![ +/// let decimal_array: Decimal128Array = vec![ /// Some(8_887_000_000), /// None, /// Some(-8_887_000_000), @@ -68,7 +68,7 @@ use crate::util::decimal::{BasicDecimal, Decimal128, Decimal256}; /// assert_eq!(6, decimal_array.scale()); /// ``` /// -pub struct DecimalArray { +pub struct Decimal128Array { data: ArrayData, value_data: RawPtrBox, precision: usize, @@ -186,13 +186,13 @@ pub trait BasicDecimalArray>: assert_eq!( child_data.child_data().len(), 0, - "DecimalArray can only be created from list array of u8 values \ + "Decimal128Array can only be created from list array of u8 values \ (i.e. FixedSizeList>)." ); assert_eq!( child_data.data_type(), &DataType::UInt8, - "DecimalArray can only be created from FixedSizeList arrays, mismatched data types." + "Decimal128Array can only be created from FixedSizeList arrays, mismatched data types." ); let list_offset = v.offset(); @@ -208,7 +208,7 @@ pub trait BasicDecimalArray>: } } -impl BasicDecimalArray for DecimalArray { +impl BasicDecimalArray for Decimal128Array { const VALUE_LENGTH: i32 = 16; fn data(&self) -> &ArrayData { @@ -240,8 +240,8 @@ impl BasicDecimalArray for Decimal256Array { } } -impl DecimalArray { - /// Creates a [DecimalArray] with default precision and scale, +impl Decimal128Array { + /// Creates a [Decimal128Array] with default precision and scale, /// based on an iterator of `i128` values without nulls pub fn from_iter_values>(iter: I) -> Self { let val_buf: Buffer = iter.into_iter().collect(); @@ -256,10 +256,10 @@ impl DecimalArray { vec![], ) }; - DecimalArray::from(data) + Decimal128Array::from(data) } - /// Returns a DecimalArray with the same data as self, with the + /// Returns a Decimal128Array with the same data as self, with the /// specified precision. /// /// Returns an Error if: @@ -319,12 +319,12 @@ impl DecimalArray { } } -impl From for DecimalArray { +impl From for Decimal128Array { fn from(data: ArrayData) -> Self { assert_eq!( data.buffers().len(), 1, - "DecimalArray data should contain 1 buffer only (values)" + "Decimal128Array data should contain 1 buffer only (values)" ); let values = data.buffers()[0].as_ptr(); let (precision, scale) = match data.data_type() { @@ -345,7 +345,7 @@ impl From for Decimal256Array { assert_eq!( data.buffers().len(), 1, - "DecimalArray data should contain 1 buffer only (values)" + "Decimal128Array data should contain 1 buffer only (values)" ); let values = data.buffers()[0].as_ptr(); let (precision, scale) = match data.data_type() { @@ -361,7 +361,7 @@ impl From for Decimal256Array { } } -impl<'a> IntoIterator for &'a DecimalArray { +impl<'a> IntoIterator for &'a Decimal128Array { type Item = Option; type IntoIter = DecimalIter<'a>; @@ -370,14 +370,14 @@ impl<'a> IntoIterator for &'a DecimalArray { } } -impl<'a> DecimalArray { +impl<'a> Decimal128Array { /// constructs a new iterator pub fn iter(&'a self) -> DecimalIter<'a> { DecimalIter::new(self) } } -impl>> FromIterator for DecimalArray { +impl>> FromIterator for Decimal128Array { fn from_iter>(iter: I) -> Self { let iter = iter.into_iter(); let (lower, upper) = iter.size_hint(); @@ -409,7 +409,7 @@ impl>> FromIterator for DecimalArray { vec![], ) }; - DecimalArray::from(data) + Decimal128Array::from(data) } } @@ -459,12 +459,12 @@ macro_rules! def_decimal_array { }; } -def_decimal_array!(DecimalArray, "DecimalArray"); +def_decimal_array!(Decimal128Array, "Decimal128Array"); def_decimal_array!(Decimal256Array, "Decimal256Array"); #[cfg(test)] mod tests { - use crate::{array::DecimalBuilder, datatypes::Field}; + use crate::{array::Decimal128Builder, datatypes::Field}; use super::*; @@ -481,7 +481,7 @@ mod tests { .add_buffer(Buffer::from(&values[..])) .build() .unwrap(); - let decimal_array = DecimalArray::from(array_data); + let decimal_array = Decimal128Array::from(array_data); assert_eq!(8_887_000_000_i128, decimal_array.value(0).into()); assert_eq!(-8_887_000_000_i128, decimal_array.value(1).into()); assert_eq!(16, decimal_array.value_length()); @@ -490,7 +490,7 @@ mod tests { #[test] #[cfg(not(feature = "force_validate"))] fn test_decimal_append_error_value() { - let mut decimal_builder = DecimalBuilder::new(10, 5, 3); + let mut decimal_builder = Decimal128Builder::new(10, 5, 3); let mut result = decimal_builder.append_value(123456); let mut error = result.unwrap_err(); assert_eq!( @@ -507,7 +507,7 @@ mod tests { let arr = decimal_builder.finish(); assert_eq!("12.345", arr.value_as_string(1)); - decimal_builder = DecimalBuilder::new(10, 2, 1); + decimal_builder = Decimal128Builder::new(10, 2, 1); result = decimal_builder.append_value(100); error = result.unwrap_err(); assert_eq!( @@ -531,7 +531,7 @@ mod tests { #[test] fn test_decimal_from_iter_values() { - let array = DecimalArray::from_iter_values(vec![-100, 0, 101].into_iter()); + let array = Decimal128Array::from_iter_values(vec![-100, 0, 101].into_iter()); assert_eq!(array.len(), 3); assert_eq!(array.data_type(), &DataType::Decimal(38, 10)); assert_eq!(-100_i128, array.value(0).into()); @@ -544,7 +544,8 @@ mod tests { #[test] fn test_decimal_from_iter() { - let array: DecimalArray = vec![Some(-100), None, Some(101)].into_iter().collect(); + let array: Decimal128Array = + vec![Some(-100), None, Some(101)].into_iter().collect(); assert_eq!(array.len(), 3); assert_eq!(array.data_type(), &DataType::Decimal(38, 10)); assert_eq!(-100_i128, array.value(0).into()); @@ -557,7 +558,7 @@ mod tests { #[test] fn test_decimal_iter() { let data = vec![Some(-100), None, Some(101)]; - let array: DecimalArray = data.clone().into_iter().collect(); + let array: Decimal128Array = data.clone().into_iter().collect(); let collected: Vec<_> = array.iter().collect(); assert_eq!(data, collected); @@ -566,7 +567,7 @@ mod tests { #[test] fn test_decimal_into_iter() { let data = vec![Some(-100), None, Some(101)]; - let array: DecimalArray = data.clone().into_iter().collect(); + let array: Decimal128Array = data.clone().into_iter().collect(); let collected: Vec<_> = array.into_iter().collect(); assert_eq!(data, collected); @@ -575,7 +576,7 @@ mod tests { #[test] fn test_decimal_iter_sized() { let data = vec![Some(-100), None, Some(101)]; - let array: DecimalArray = data.into_iter().collect(); + let array: Decimal128Array = data.into_iter().collect(); let mut iter = array.into_iter(); // is exact sized @@ -597,7 +598,7 @@ mod tests { let arr = [123450, -123450, 100, -100, 10, -10, 0] .into_iter() .map(Some) - .collect::() + .collect::() .with_precision_and_scale(6, 3) .unwrap(); @@ -612,7 +613,7 @@ mod tests { #[test] fn test_decimal_array_with_precision_and_scale() { - let arr = DecimalArray::from_iter_values([12345, 456, 7890, -123223423432432]) + let arr = Decimal128Array::from_iter_values([12345, 456, 7890, -123223423432432]) .with_precision_and_scale(20, 2) .unwrap(); @@ -631,7 +632,7 @@ mod tests { expected = "-123223423432432 is too small to store in a Decimal of precision 5. Min is -99999" )] fn test_decimal_array_with_precision_and_scale_out_of_range() { - DecimalArray::from_iter_values([12345, 456, 7890, -123223423432432]) + Decimal128Array::from_iter_values([12345, 456, 7890, -123223423432432]) // precision is too small to hold value .with_precision_and_scale(5, 2) .unwrap(); @@ -640,7 +641,7 @@ mod tests { #[test] #[should_panic(expected = "precision 40 is greater than max 38")] fn test_decimal_array_with_precision_and_scale_invalid_precision() { - DecimalArray::from_iter_values([12345, 456]) + Decimal128Array::from_iter_values([12345, 456]) .with_precision_and_scale(40, 2) .unwrap(); } @@ -648,7 +649,7 @@ mod tests { #[test] #[should_panic(expected = "scale 40 is greater than max 38")] fn test_decimal_array_with_precision_and_scale_invalid_scale() { - DecimalArray::from_iter_values([12345, 456]) + Decimal128Array::from_iter_values([12345, 456]) .with_precision_and_scale(20, 40) .unwrap(); } @@ -656,7 +657,7 @@ mod tests { #[test] #[should_panic(expected = "scale 10 is greater than precision 4")] fn test_decimal_array_with_precision_and_scale_invalid_precision_and_scale() { - DecimalArray::from_iter_values([12345, 456]) + Decimal128Array::from_iter_values([12345, 456]) .with_precision_and_scale(4, 10) .unwrap(); } @@ -665,12 +666,12 @@ mod tests { fn test_decimal_array_fmt_debug() { let arr = [Some(8887000000), Some(-8887000000), None] .iter() - .collect::() + .collect::() .with_precision_and_scale(23, 6) .unwrap(); assert_eq!( - "DecimalArray<23, 6>\n[\n 8887.000000,\n -8887.000000,\n null,\n]", + "Decimal128Array<23, 6>\n[\n 8887.000000,\n -8887.000000,\n null,\n]", format!("{:?}", arr) ); } @@ -686,7 +687,7 @@ mod tests { .unwrap(); let binary_array = FixedSizeBinaryArray::from(value_data); - let decimal = DecimalArray::from_fixed_size_binary_array(binary_array, 38, 1); + let decimal = Decimal128Array::from_fixed_size_binary_array(binary_array, 38, 1); assert_eq!(decimal.len(), 3); assert_eq!(decimal.value_as_string(0), "0.2".to_string()); @@ -708,7 +709,7 @@ mod tests { .unwrap(); let binary_array = FixedSizeBinaryArray::from(value_data); - let _ = DecimalArray::from_fixed_size_binary_array(binary_array, 38, 1); + let _ = Decimal128Array::from_fixed_size_binary_array(binary_array, 38, 1); } #[test] @@ -736,7 +737,7 @@ mod tests { .build() .unwrap(); let list_array = FixedSizeListArray::from(list_data); - let decimal = DecimalArray::from_fixed_size_list_array(list_array, 38, 0); + let decimal = Decimal128Array::from_fixed_size_list_array(list_array, 38, 0); assert_eq!(decimal.len(), 2); assert!(decimal.is_null(0)); diff --git a/arrow/src/array/builder/decimal_builder.rs b/arrow/src/array/builder/decimal_builder.rs index 20dce7679d7..e5dfa32f029 100644 --- a/arrow/src/array/builder/decimal_builder.rs +++ b/arrow/src/array/builder/decimal_builder.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use crate::array::array_decimal::{BasicDecimalArray, Decimal256Array}; use crate::array::ArrayRef; -use crate::array::DecimalArray; +use crate::array::Decimal128Array; use crate::array::{ArrayBuilder, FixedSizeBinaryBuilder}; use crate::error::{ArrowError, Result}; @@ -28,12 +28,12 @@ use crate::error::{ArrowError, Result}; use crate::datatypes::validate_decimal_precision; use crate::util::decimal::{BasicDecimal, Decimal256}; -/// Array Builder for [`DecimalArray`] +/// Array Builder for [`Decimal128Array`] /// -/// See [`DecimalArray`] for example. +/// See [`Decimal128Array`] for example. /// #[derive(Debug)] -pub struct DecimalBuilder { +pub struct Decimal128Builder { builder: FixedSizeBinaryBuilder, precision: usize, scale: usize, @@ -53,9 +53,9 @@ pub struct Decimal256Builder { scale: usize, } -impl DecimalBuilder { +impl Decimal128Builder { const BYTE_LENGTH: i32 = 16; - /// Creates a new [`DecimalBuilder`], `capacity` is the number of bytes in the values + /// Creates a new [`Decimal128Builder`], `capacity` is the number of bytes in the values /// array pub fn new(capacity: usize, precision: usize, scale: usize) -> Self { Self { @@ -89,7 +89,7 @@ impl DecimalBuilder { Self::from_i128_to_fixed_size_bytes(value, Self::BYTE_LENGTH as usize)?; if Self::BYTE_LENGTH != value_as_bytes.len() as i32 { return Err(ArrowError::InvalidArgumentError( - "Byte slice does not have the same length as DecimalBuilder value lengths".to_string() + "Byte slice does not have the same length as Decimal128Builder value lengths".to_string() )); } self.builder.append_value(value_as_bytes.as_slice()) @@ -98,7 +98,7 @@ impl DecimalBuilder { pub(crate) fn from_i128_to_fixed_size_bytes(v: i128, size: usize) -> Result> { if size > 16 { return Err(ArrowError::InvalidArgumentError( - "DecimalBuilder only supports values up to 16 bytes.".to_string(), + "Decimal128Builder only supports values up to 16 bytes.".to_string(), )); } let res = v.to_le_bytes(); @@ -112,9 +112,9 @@ impl DecimalBuilder { self.builder.append_null() } - /// Builds the `DecimalArray` and reset this builder. - pub fn finish(&mut self) -> DecimalArray { - DecimalArray::from_fixed_size_binary_array( + /// Builds the `Decimal128Array` and reset this builder. + pub fn finish(&mut self) -> Decimal128Array { + Decimal128Array::from_fixed_size_binary_array( self.builder.finish(), self.precision, self.scale, @@ -122,7 +122,7 @@ impl DecimalBuilder { } } -impl ArrayBuilder for DecimalBuilder { +impl ArrayBuilder for Decimal128Builder { /// Returns the builder as a non-mutable `Any` reference. fn as_any(&self) -> &dyn Any { self @@ -205,19 +205,19 @@ impl Decimal256Builder { mod tests { use super::*; - use crate::array::array_decimal::BasicDecimalArray; + use crate::array::array_decimal::{BasicDecimalArray, Decimal128Array}; use crate::array::Array; use crate::datatypes::DataType; use crate::util::decimal::Decimal128; #[test] fn test_decimal_builder() { - let mut builder = DecimalBuilder::new(30, 38, 6); + let mut builder = Decimal128Builder::new(30, 38, 6); builder.append_value(8_887_000_000_i128).unwrap(); builder.append_null().unwrap(); builder.append_value(-8_887_000_000_i128).unwrap(); - let decimal_array: DecimalArray = builder.finish(); + let decimal_array: Decimal128Array = builder.finish(); assert_eq!(&DataType::Decimal(38, 6), decimal_array.data_type()); assert_eq!(3, decimal_array.len()); @@ -228,7 +228,7 @@ mod tests { #[test] fn test_decimal_builder_with_decimal128() { - let mut builder = DecimalBuilder::new(30, 38, 6); + let mut builder = Decimal128Builder::new(30, 38, 6); builder .append_value(Decimal128::new_from_i128(30, 38, 8_887_000_000_i128)) @@ -237,7 +237,7 @@ mod tests { builder .append_value(Decimal128::new_from_i128(30, 38, -8_887_000_000_i128)) .unwrap(); - let decimal_array: DecimalArray = builder.finish(); + let decimal_array: Decimal128Array = builder.finish(); assert_eq!(&DataType::Decimal(38, 6), decimal_array.data_type()); assert_eq!(3, decimal_array.len()); diff --git a/arrow/src/array/builder/mod.rs b/arrow/src/array/builder/mod.rs index 045a11648d5..251d476a004 100644 --- a/arrow/src/array/builder/mod.rs +++ b/arrow/src/array/builder/mod.rs @@ -45,8 +45,8 @@ use super::ArrayRef; pub use boolean_buffer_builder::BooleanBufferBuilder; pub use boolean_builder::BooleanBuilder; pub use buffer_builder::BufferBuilder; +pub use decimal_builder::Decimal128Builder; pub use decimal_builder::Decimal256Builder; -pub use decimal_builder::DecimalBuilder; pub use fixed_size_binary_builder::FixedSizeBinaryBuilder; pub use fixed_size_list_builder::FixedSizeListBuilder; pub use generic_binary_builder::GenericBinaryBuilder; diff --git a/arrow/src/array/builder/struct_builder.rs b/arrow/src/array/builder/struct_builder.rs index 206eb17c242..0e70bf54a42 100644 --- a/arrow/src/array/builder/struct_builder.rs +++ b/arrow/src/array/builder/struct_builder.rs @@ -19,6 +19,7 @@ use std::any::Any; use std::fmt; use std::sync::Arc; +use crate::array::builder::decimal_builder::Decimal128Builder; use crate::array::*; use crate::datatypes::DataType; use crate::datatypes::Field; @@ -111,7 +112,7 @@ pub fn make_builder(datatype: &DataType, capacity: usize) -> Box { - Box::new(DecimalBuilder::new(capacity, *precision, *scale)) + Box::new(Decimal128Builder::new(capacity, *precision, *scale)) } DataType::Utf8 => Box::new(StringBuilder::new(capacity)), DataType::Date32 => Box::new(Date32Builder::new(capacity)), diff --git a/arrow/src/array/cast.rs b/arrow/src/array/cast.rs index d0b77a0d27b..dd6c7135710 100644 --- a/arrow/src/array/cast.rs +++ b/arrow/src/array/cast.rs @@ -96,7 +96,7 @@ array_downcast_fn!(as_null_array, NullArray); array_downcast_fn!(as_struct_array, StructArray); array_downcast_fn!(as_union_array, UnionArray); array_downcast_fn!(as_map_array, MapArray); -array_downcast_fn!(as_decimal_array, DecimalArray); +array_downcast_fn!(as_decimal_array, Decimal128Array); #[cfg(test)] mod tests { @@ -106,9 +106,9 @@ mod tests { #[test] fn test_as_decimal_array_ref() { - let array: DecimalArray = vec![Some(123), None, Some(1111)] + let array: Decimal128Array = vec![Some(123), None, Some(1111)] .into_iter() - .collect::() + .collect::() .with_precision_and_scale(10, 2) .unwrap(); assert!(!as_decimal_array(&array).is_empty()); diff --git a/arrow/src/array/data.rs b/arrow/src/array/data.rs index eba496cbf09..5c7bd69d817 100644 --- a/arrow/src/array/data.rs +++ b/arrow/src/array/data.rs @@ -1513,7 +1513,7 @@ mod tests { use std::ptr::NonNull; use crate::array::{ - make_array, Array, BooleanBuilder, DecimalBuilder, FixedSizeListBuilder, + make_array, Array, BooleanBuilder, Decimal128Builder, FixedSizeListBuilder, Int32Array, Int32Builder, Int64Array, StringArray, StructBuilder, UInt64Array, UInt8Builder, }; @@ -2769,7 +2769,7 @@ mod tests { let byte_width = 16; let mut fixed_size_builder = FixedSizeListBuilder::new(values_builder, byte_width); - let value_as_bytes = DecimalBuilder::from_i128_to_fixed_size_bytes( + let value_as_bytes = Decimal128Builder::from_i128_to_fixed_size_bytes( 123456, fixed_size_builder.value_length() as usize, ) @@ -2796,7 +2796,7 @@ mod tests { #[test] fn test_decimal_validation() { - let mut builder = DecimalBuilder::new(4, 10, 4); + let mut builder = Decimal128Builder::new(4, 10, 4); builder.append_value(10000).unwrap(); builder.append_value(20000).unwrap(); let array = builder.finish(); diff --git a/arrow/src/array/equal/mod.rs b/arrow/src/array/equal/mod.rs index 74599c2ed6a..b8a7bc1bc5e 100644 --- a/arrow/src/array/equal/mod.rs +++ b/arrow/src/array/equal/mod.rs @@ -20,9 +20,10 @@ //! depend on dynamic casting of `Array`. use super::{ - Array, ArrayData, BooleanArray, DecimalArray, DictionaryArray, FixedSizeBinaryArray, - FixedSizeListArray, GenericBinaryArray, GenericListArray, GenericStringArray, - MapArray, NullArray, OffsetSizeTrait, PrimitiveArray, StructArray, + Array, ArrayData, BooleanArray, Decimal128Array, DictionaryArray, + FixedSizeBinaryArray, FixedSizeListArray, GenericBinaryArray, GenericListArray, + GenericStringArray, MapArray, NullArray, OffsetSizeTrait, PrimitiveArray, + StructArray, }; use crate::datatypes::{ArrowPrimitiveType, DataType, IntervalUnit}; use half::f16; @@ -109,7 +110,7 @@ impl PartialEq for FixedSizeBinaryArray { } } -impl PartialEq for DecimalArray { +impl PartialEq for Decimal128Array { fn eq(&self, other: &Self) -> bool { equal(self.data(), other.data()) } @@ -840,7 +841,7 @@ mod tests { fn create_decimal_array(data: &[Option]) -> ArrayData { data.iter() - .collect::() + .collect::() .with_precision_and_scale(23, 6) .unwrap() .into() diff --git a/arrow/src/array/equal_json.rs b/arrow/src/array/equal_json.rs index 3fc84a7e3ab..95a2f0a43f6 100644 --- a/arrow/src/array/equal_json.rs +++ b/arrow/src/array/equal_json.rs @@ -361,7 +361,7 @@ impl PartialEq for Value { } } -impl JsonEqual for DecimalArray { +impl JsonEqual for Decimal128Array { fn equals_json(&self, json: &[&Value]) -> bool { if self.len() != json.len() { return false; @@ -394,7 +394,7 @@ impl JsonEqual for Decimal256Array { } } -impl PartialEq for DecimalArray { +impl PartialEq for Decimal128Array { fn eq(&self, json: &Value) -> bool { match json { Value::Array(json_array) => self.equals_json_values(json_array), @@ -403,8 +403,8 @@ impl PartialEq for DecimalArray { } } -impl PartialEq for Value { - fn eq(&self, arrow: &DecimalArray) -> bool { +impl PartialEq for Value { + fn eq(&self, arrow: &Decimal128Array) -> bool { match self { Value::Array(json_array) => arrow.equals_json_values(json_array), _ => false, @@ -947,7 +947,7 @@ mod tests { // Test the equal case let arrow_array = [Some(1_000), None, Some(-250)] .iter() - .collect::() + .collect::() .with_precision_and_scale(23, 6) .unwrap(); let json_array: Value = serde_json::from_str( @@ -966,7 +966,7 @@ mod tests { // Test unequal case let arrow_array = [Some(1_000), None, Some(55)] .iter() - .collect::() + .collect::() .with_precision_and_scale(23, 6) .unwrap(); let json_array: Value = serde_json::from_str( diff --git a/arrow/src/array/iterator.rs b/arrow/src/array/iterator.rs index 9ac2d0642d4..8e45de28636 100644 --- a/arrow/src/array/iterator.rs +++ b/arrow/src/array/iterator.rs @@ -19,7 +19,7 @@ use crate::array::BasicDecimalArray; use crate::datatypes::ArrowPrimitiveType; use super::{ - Array, ArrayRef, BooleanArray, DecimalArray, GenericBinaryArray, GenericListArray, + Array, ArrayRef, BooleanArray, Decimal128Array, GenericBinaryArray, GenericListArray, GenericStringArray, OffsetSizeTrait, PrimitiveArray, }; @@ -394,16 +394,16 @@ impl<'a, S: OffsetSizeTrait> std::iter::ExactSizeIterator } /// an iterator that returns `Some(i128)` or `None`, that can be used on a -/// [`DecimalArray`] +/// [`Decimal128Array`] #[derive(Debug)] pub struct DecimalIter<'a> { - array: &'a DecimalArray, + array: &'a Decimal128Array, current: usize, current_end: usize, } impl<'a> DecimalIter<'a> { - pub fn new(array: &'a DecimalArray) -> Self { + pub fn new(array: &'a Decimal128Array) -> Self { Self { array, current: 0, diff --git a/arrow/src/array/mod.rs b/arrow/src/array/mod.rs index 2f025f11c0f..c5d90a42cc9 100644 --- a/arrow/src/array/mod.rs +++ b/arrow/src/array/mod.rs @@ -183,8 +183,12 @@ pub use self::array_binary::FixedSizeBinaryArray; pub use self::array_binary::LargeBinaryArray; pub use self::array_boolean::BooleanArray; pub use self::array_decimal::BasicDecimalArray; +pub use self::array_decimal::Decimal128Array; pub use self::array_decimal::Decimal256Array; -pub use self::array_decimal::DecimalArray; + +#[deprecated(note = "Please use `Decimal128Array` instead")] +pub type DecimalArray = Decimal128Array; + pub use self::array_dictionary::DictionaryArray; pub use self::array_list::FixedSizeListArray; pub use self::array_list::LargeListArray; @@ -471,8 +475,12 @@ pub use self::builder::BinaryBuilder; pub use self::builder::BooleanBufferBuilder; pub use self::builder::BooleanBuilder; pub use self::builder::BufferBuilder; +pub use self::builder::Decimal128Builder; pub use self::builder::Decimal256Builder; -pub use self::builder::DecimalBuilder; + +#[deprecated(note = "Please use `Decimal128Builder` instead")] +pub type DecimalBuilder = Decimal128Builder; + pub use self::builder::FixedSizeBinaryBuilder; pub use self::builder::FixedSizeListBuilder; pub use self::builder::GenericListBuilder; diff --git a/arrow/src/array/ord.rs b/arrow/src/array/ord.rs index 019b1163b50..888c31c5d95 100644 --- a/arrow/src/array/ord.rs +++ b/arrow/src/array/ord.rs @@ -227,8 +227,8 @@ pub fn build_compare(left: &dyn Array, right: &dyn Array) -> Result { - let left: DecimalArray = DecimalArray::from(left.data().clone()); - let right: DecimalArray = DecimalArray::from(right.data().clone()); + let left: Decimal128Array = Decimal128Array::from(left.data().clone()); + let right: Decimal128Array = Decimal128Array::from(right.data().clone()); Box::new(move |i, j| left.value(i).cmp(&right.value(j))) } (lhs, _) => { @@ -303,7 +303,7 @@ pub mod tests { fn test_decimal() -> Result<()> { let array = vec![Some(5), Some(2), Some(3)] .iter() - .collect::() + .collect::() .with_precision_and_scale(23, 6) .unwrap(); diff --git a/arrow/src/array/transform/mod.rs b/arrow/src/array/transform/mod.rs index a103c35e506..5c15503a9db 100644 --- a/arrow/src/array/transform/mod.rs +++ b/arrow/src/array/transform/mod.rs @@ -670,7 +670,7 @@ mod tests { use super::*; - use crate::array::DecimalArray; + use crate::array::Decimal128Array; use crate::{ array::{ Array, ArrayData, ArrayRef, BooleanArray, DictionaryArray, @@ -690,10 +690,10 @@ mod tests { array: &[Option], precision: usize, scale: usize, - ) -> DecimalArray { + ) -> Decimal128Array { array .iter() - .collect::() + .collect::() .with_precision_and_scale(precision, scale) .unwrap() } @@ -708,7 +708,7 @@ mod tests { a.extend(0, 0, 3); a.extend(0, 2, 3); let result = a.freeze(); - let array = DecimalArray::from(result); + let array = Decimal128Array::from(result); let expected = create_decimal_array(&[Some(1), Some(2), None, None], 10, 3); assert_eq!(array, expected); } @@ -722,7 +722,7 @@ mod tests { let mut a = MutableArrayData::new(arrays, true, 2); a.extend(0, 0, 2); // 2, null let result = a.freeze(); - let array = DecimalArray::from(result); + let array = Decimal128Array::from(result); let expected = create_decimal_array(&[Some(2), None], 10, 3); assert_eq!(array, expected); } @@ -739,7 +739,7 @@ mod tests { a.extend_nulls(3); // 2, null, null, null, null a.extend(0, 1, 3); //2, null, null, null, null, null, 3 let result = a.freeze(); - let array = DecimalArray::from(result); + let array = Decimal128Array::from(result); let expected = create_decimal_array( &[Some(2), None, None, None, None, None, Some(3)], 10, diff --git a/arrow/src/compute/kernels/cast.rs b/arrow/src/compute/kernels/cast.rs index 3dd2ad69264..9abb9ec4266 100644 --- a/arrow/src/compute/kernels/cast.rs +++ b/arrow/src/compute/kernels/cast.rs @@ -284,7 +284,7 @@ macro_rules! cast_integer_to_decimal { mul * v }) }) - .collect::() + .collect::() .with_precision_and_scale(*$PRECISION, *$SCALE)?; Ok(Arc::new(decimal_array)) }}; @@ -304,7 +304,7 @@ macro_rules! cast_floating_point_to_decimal { ((v as f64) * mul) as i128 }) }) - .collect::() + .collect::() .with_precision_and_scale(*$PRECISION, *$SCALE)?; Ok(Arc::new(decimal_array)) }}; @@ -313,7 +313,7 @@ macro_rules! cast_floating_point_to_decimal { // cast the decimal array to integer array macro_rules! cast_decimal_to_integer { ($ARRAY:expr, $SCALE : ident, $VALUE_BUILDER: ident, $NATIVE_TYPE : ident, $DATA_TYPE : expr) => {{ - let array = $ARRAY.as_any().downcast_ref::().unwrap(); + let array = $ARRAY.as_any().downcast_ref::().unwrap(); let mut value_builder = $VALUE_BUILDER::new(array.len()); let div: i128 = 10_i128.pow(*$SCALE as u32); let min_bound = ($NATIVE_TYPE::MIN) as i128; @@ -343,7 +343,7 @@ macro_rules! cast_decimal_to_integer { // cast the decimal array to floating-point array macro_rules! cast_decimal_to_float { ($ARRAY:expr, $SCALE : ident, $VALUE_BUILDER: ident, $NATIVE_TYPE : ty) => {{ - let array = $ARRAY.as_any().downcast_ref::().unwrap(); + let array = $ARRAY.as_any().downcast_ref::().unwrap(); let div = 10_f64.powi(*$SCALE as i32); let mut value_builder = $VALUE_BUILDER::new(array.len()); for i in 0..array.len() { @@ -1196,7 +1196,7 @@ fn cast_decimal_to_decimal( output_precision: &usize, output_scale: &usize, ) -> Result { - let array = array.as_any().downcast_ref::().unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); let output_array = if input_scale > output_scale { // For example, input_scale is 4 and output_scale is 3; @@ -1205,7 +1205,7 @@ fn cast_decimal_to_decimal( array .iter() .map(|v| v.map(|v| v / div)) - .collect::() + .collect::() } else { // For example, input_scale is 3 and output_scale is 4; // Original value is 1123_i128, and will be cast to 11230_i128. @@ -1213,7 +1213,7 @@ fn cast_decimal_to_decimal( array .iter() .map(|v| v.map(|v| v * mul)) - .collect::() + .collect::() } .with_precision_and_scale(*output_precision, *output_scale)?; @@ -2168,10 +2168,10 @@ mod tests { array: &[Option], precision: usize, scale: usize, - ) -> Result { + ) -> Result { array .iter() - .collect::() + .collect::() .with_precision_and_scale(precision, scale) } @@ -2185,7 +2185,7 @@ mod tests { let array = Arc::new(input_decimal_array) as ArrayRef; generate_cast_test_case!( &array, - DecimalArray, + Decimal128Array, &output_type, vec![ Some(Decimal128::new_from_i128(20, 4, 11234560_i128)), @@ -2364,7 +2364,7 @@ mod tests { for array in input_datas { generate_cast_test_case!( &array, - DecimalArray, + Decimal128Array, &decimal_type, vec![ Some(Decimal128::new_from_i128(38, 6, 1000000_i128)), @@ -2396,7 +2396,7 @@ mod tests { let array = Arc::new(array) as ArrayRef; generate_cast_test_case!( &array, - DecimalArray, + Decimal128Array, &decimal_type, vec![ Some(Decimal128::new_from_i128(38, 6, 1100000_i128)), @@ -2421,7 +2421,7 @@ mod tests { let array = Arc::new(array) as ArrayRef; generate_cast_test_case!( &array, - DecimalArray, + Decimal128Array, &decimal_type, vec![ Some(Decimal128::new_from_i128(38, 6, 1100000_i128)), diff --git a/arrow/src/compute/kernels/sort.rs b/arrow/src/compute/kernels/sort.rs index 8e0831c6140..d79f2249d3a 100644 --- a/arrow/src/compute/kernels/sort.rs +++ b/arrow/src/compute/kernels/sort.rs @@ -500,7 +500,7 @@ where // downcast to decimal array let decimal_array = decimal_values .as_any() - .downcast_ref::() + .downcast_ref::() .expect("Unable to downcast to decimal array"); let valids = value_indices .into_iter() @@ -1079,9 +1079,9 @@ mod tests { use std::convert::TryFrom; use std::sync::Arc; - fn create_decimal_array(data: &[Option]) -> DecimalArray { + fn create_decimal_array(data: &[Option]) -> Decimal128Array { data.iter() - .collect::() + .collect::() .with_precision_and_scale(23, 6) .unwrap() } diff --git a/arrow/src/compute/kernels/take.rs b/arrow/src/compute/kernels/take.rs index 4ff7d84d6df..1ff8c00a2ed 100644 --- a/arrow/src/compute/kernels/take.rs +++ b/arrow/src/compute/kernels/take.rs @@ -149,7 +149,8 @@ where Ok(Arc::new(take_boolean(values, indices)?)) } DataType::Decimal(_, _) => { - let decimal_values = values.as_any().downcast_ref::().unwrap(); + let decimal_values = + values.as_any().downcast_ref::().unwrap(); Ok(Arc::new(take_decimal128(decimal_values, indices)?)) } DataType::Int8 => downcast_take!(Int8Type, values, indices), @@ -506,9 +507,9 @@ where /// `take` implementation for decimal arrays fn take_decimal128( - decimal_values: &DecimalArray, + decimal_values: &Decimal128Array, indices: &PrimitiveArray, -) -> Result +) -> Result where IndexType: ArrowNumericType, IndexType::Native: ToPrimitive, @@ -533,9 +534,9 @@ where let t: Result> = t.map(|t| t.flatten()); t }) - .collect::>()? + .collect::>()? // PERF: we could avoid re-validating that the data in - // DecimalArray was in range as we know it came from a valid DecimalArray + // Decimal128Array was in range as we know it came from a valid Decimal128Array .with_precision_and_scale(decimal_values.precision(), decimal_values.scale()) } @@ -976,13 +977,13 @@ mod tests { ) -> Result<()> { let output = data .into_iter() - .collect::() + .collect::() .with_precision_and_scale(*precision, *scale) .unwrap(); let expected = expected_data .into_iter() - .collect::() + .collect::() .with_precision_and_scale(*precision, *scale) .unwrap(); diff --git a/arrow/src/csv/reader.rs b/arrow/src/csv/reader.rs index 7250f943e48..95b23378e48 100644 --- a/arrow/src/csv/reader.rs +++ b/arrow/src/csv/reader.rs @@ -50,7 +50,8 @@ use std::io::{Read, Seek, SeekFrom}; use std::sync::Arc; use crate::array::{ - ArrayRef, BooleanArray, DecimalBuilder, DictionaryArray, PrimitiveArray, StringArray, + ArrayRef, BooleanArray, Decimal128Builder, DictionaryArray, PrimitiveArray, + StringArray, }; use crate::datatypes::*; use crate::error::{ArrowError, Result}; @@ -698,7 +699,7 @@ fn build_decimal_array( precision: usize, scale: usize, ) -> Result { - let mut decimal_builder = DecimalBuilder::new(rows.len(), precision, scale); + let mut decimal_builder = Decimal128Builder::new(rows.len(), precision, scale); for row in rows { let col_s = row.get(col_idx); match col_s { @@ -1218,7 +1219,7 @@ mod tests { let lat = batch .column(1) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); assert_eq!("57.653484", lat.value_as_string(0)); diff --git a/arrow/src/ffi.rs b/arrow/src/ffi.rs index 97cbe76c84c..2d95b4ea639 100644 --- a/arrow/src/ffi.rs +++ b/arrow/src/ffi.rs @@ -909,11 +909,11 @@ impl<'a> ArrowArrayChild<'a> { mod tests { use super::*; use crate::array::{ - export_array_into_raw, make_array, Array, ArrayData, BooleanArray, DecimalArray, - DictionaryArray, DurationSecondArray, FixedSizeBinaryArray, FixedSizeListArray, - GenericBinaryArray, GenericListArray, GenericStringArray, Int32Array, MapArray, - NullArray, OffsetSizeTrait, Time32MillisecondArray, TimestampMillisecondArray, - UInt32Array, + export_array_into_raw, make_array, Array, ArrayData, BooleanArray, + Decimal128Array, DictionaryArray, DurationSecondArray, FixedSizeBinaryArray, + FixedSizeListArray, GenericBinaryArray, GenericListArray, GenericStringArray, + Int32Array, MapArray, NullArray, OffsetSizeTrait, Time32MillisecondArray, + TimestampMillisecondArray, UInt32Array, }; use crate::compute::kernels; use crate::datatypes::{Field, Int8Type}; @@ -948,7 +948,7 @@ mod tests { // create an array natively let original_array = [Some(12345_i128), Some(-12345_i128), None] .into_iter() - .collect::() + .collect::() .with_precision_and_scale(6, 2) .unwrap(); @@ -960,7 +960,7 @@ mod tests { let array = make_array(data); // perform some operation - let array = array.as_any().downcast_ref::().unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); // verify assert_eq!(array, &original_array); diff --git a/arrow/src/util/display.rs b/arrow/src/util/display.rs index 7a7da8ccb0f..9c8afb77590 100644 --- a/arrow/src/util/display.rs +++ b/arrow/src/util/display.rs @@ -256,7 +256,7 @@ macro_rules! make_string_from_fixed_size_list { pub fn make_string_from_decimal(column: &Arc, row: usize) -> Result { let array = column .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); let formatted_decimal = array.value_as_string(row); diff --git a/arrow/src/util/integration_util.rs b/arrow/src/util/integration_util.rs index ee32f0c3990..aadf0327734 100644 --- a/arrow/src/util/integration_util.rs +++ b/arrow/src/util/integration_util.rs @@ -361,7 +361,7 @@ impl ArrowJsonBatch { arr.equals_json(&json_array.iter().collect::>()[..]) } DataType::Decimal(_, _) => { - let arr = arr.as_any().downcast_ref::().unwrap(); + let arr = arr.as_any().downcast_ref::().unwrap(); arr.equals_json(&json_array.iter().collect::>()[..]) } DataType::Dictionary(ref key_type, _) => match key_type.as_ref() { diff --git a/arrow/src/util/pretty.rs b/arrow/src/util/pretty.rs index 124de6127dd..6622593df09 100644 --- a/arrow/src/util/pretty.rs +++ b/arrow/src/util/pretty.rs @@ -120,7 +120,7 @@ mod tests { }; use super::*; - use crate::array::{DecimalArray, FixedSizeListBuilder}; + use crate::array::{Decimal128Array, FixedSizeListBuilder}; use std::fmt::Write; use std::sync::Arc; @@ -523,7 +523,7 @@ mod tests { let array = [Some(101), None, Some(200), Some(3040)] .into_iter() - .collect::() + .collect::() .with_precision_and_scale(precision, scale) .unwrap(); @@ -563,7 +563,7 @@ mod tests { let array = [Some(101), None, Some(200), Some(3040)] .into_iter() - .collect::() + .collect::() .with_precision_and_scale(precision, scale) .unwrap(); diff --git a/integration-testing/src/lib.rs b/integration-testing/src/lib.rs index e4cc872ffd1..32ea6339e59 100644 --- a/integration-testing/src/lib.rs +++ b/integration-testing/src/lib.rs @@ -592,7 +592,7 @@ fn array_from_json( } } DataType::Decimal(precision, scale) => { - let mut b = DecimalBuilder::new(json_col.count, *precision, *scale); + let mut b = Decimal128Builder::new(json_col.count, *precision, *scale); // C++ interop tests involve incompatible decimal values unsafe { b.disable_value_validation(); diff --git a/parquet/src/arrow/array_reader/primitive_array.rs b/parquet/src/arrow/array_reader/primitive_array.rs index cb41d1fba9c..6f6644b5d5d 100644 --- a/parquet/src/arrow/array_reader/primitive_array.rs +++ b/parquet/src/arrow/array_reader/primitive_array.rs @@ -25,7 +25,7 @@ use crate::data_type::DataType; use crate::errors::{ParquetError, Result}; use crate::schema::types::ColumnDescPtr; use arrow::array::{ - ArrayDataBuilder, ArrayRef, BooleanArray, BooleanBufferBuilder, DecimalArray, + ArrayDataBuilder, ArrayRef, BooleanArray, BooleanBufferBuilder, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, }; use arrow::buffer::Buffer; @@ -203,7 +203,7 @@ where .unwrap() .iter() .map(|v| v.map(|v| v.into())) - .collect::(), + .collect::(), ArrowType::Int64 => array .as_any() @@ -211,7 +211,7 @@ where .unwrap() .iter() .map(|v| v.map(|v| v.into())) - .collect::(), + .collect::(), _ => { return Err(arrow_err!( "Cannot convert {:?} to decimal", diff --git a/parquet/src/arrow/arrow_reader.rs b/parquet/src/arrow/arrow_reader.rs index ebbb864d630..fb4489defd1 100644 --- a/parquet/src/arrow/arrow_reader.rs +++ b/parquet/src/arrow/arrow_reader.rs @@ -717,7 +717,7 @@ mod tests { #[test] fn test_read_decimal_file() { - use arrow::array::DecimalArray; + use arrow::array::Decimal128Array; let testdata = arrow::util::test_util::parquet_test_data(); let file_variants = vec![("fixed_length", 25), ("int32", 4), ("int64", 10)]; for (prefix, target_precision) in file_variants { @@ -732,7 +732,7 @@ mod tests { let col = batch .column(0) .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); let expected = 1..25; diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index 73f46f971f9..70ddf60f4aa 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -572,7 +572,7 @@ fn write_leaf( ArrowDataType::Decimal(_, _) => { let array = column .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); get_decimal_array_slice(array, indices) } @@ -689,7 +689,7 @@ fn get_interval_dt_array_slice( } fn get_decimal_array_slice( - array: &arrow_array::DecimalArray, + array: &arrow_array::Decimal128Array, indices: &[usize], ) -> Vec { let mut values = Vec::with_capacity(indices.len()); @@ -932,7 +932,7 @@ mod tests { let decimal_values = vec![10_000, 50_000, 0, -100] .into_iter() .map(Some) - .collect::() + .collect::() .with_precision_and_scale(5, 2) .unwrap(); diff --git a/parquet/src/arrow/buffer/converter.rs b/parquet/src/arrow/buffer/converter.rs index 51e1d8290ee..14063d71647 100644 --- a/parquet/src/arrow/buffer/converter.rs +++ b/parquet/src/arrow/buffer/converter.rs @@ -17,7 +17,7 @@ use crate::data_type::{ByteArray, FixedLenByteArray, Int96}; use arrow::array::{ - Array, ArrayRef, BinaryArray, BinaryBuilder, DecimalArray, FixedSizeBinaryArray, + Array, ArrayRef, BinaryArray, BinaryBuilder, Decimal128Array, FixedSizeBinaryArray, FixedSizeBinaryBuilder, IntervalDayTimeArray, IntervalDayTimeBuilder, IntervalYearMonthArray, IntervalYearMonthBuilder, LargeBinaryArray, LargeBinaryBuilder, LargeStringArray, LargeStringBuilder, StringArray, StringBuilder, @@ -78,7 +78,7 @@ impl DecimalArrayConverter { } fn from_bytes_to_i128(b: &[u8]) -> i128 { - assert!(b.len() <= 16, "DecimalArray supports only up to size 16"); + assert!(b.len() <= 16, "Decimal128Array supports only up to size 16"); let first_bit = b[0] & 128u8 == 128u8; let mut result = if first_bit { [255u8; 16] } else { [0u8; 16] }; for (i, v) in b.iter().enumerate() { @@ -88,12 +88,14 @@ impl DecimalArrayConverter { } } -impl Converter>, DecimalArray> for DecimalArrayConverter { - fn convert(&self, source: Vec>) -> Result { +impl Converter>, Decimal128Array> + for DecimalArrayConverter +{ + fn convert(&self, source: Vec>) -> Result { let array = source .into_iter() .map(|array| array.map(|array| Self::from_bytes_to_i128(array.data()))) - .collect::() + .collect::() .with_precision_and_scale(self.precision as usize, self.scale as usize)?; Ok(array) @@ -272,7 +274,7 @@ pub type IntervalDayTimeConverter = ArrayRefConverter< pub type DecimalConverter = ArrayRefConverter< Vec>, - DecimalArray, + Decimal128Array, DecimalArrayConverter, >;