Skip to content

Commit

Permalink
Define Decimal128Builder and Decimal128Array
Browse files Browse the repository at this point in the history
  • Loading branch information
@viirya
viirya committed Jul 18, 2022
1 parent 6a37d9f commit 00d66c8
Show file tree
Hide file tree
Showing 26 changed files with 197 additions and 141 deletions.
2 changes: 1 addition & 1 deletion arrow/src/array/array.rs
View file
Expand Up @@ -403,7 +403,7 @@ pub fn make_array(data: ArrayData) -> ArrayRef {
dt => panic!("Unexpected dictionary key type {:?}", dt),
},
DataType::Null => Arc::new(NullArray::from(data)) as ArrayRef,
DataType::Decimal(_, _) => Arc::new(DecimalArray::from(data)) as ArrayRef,
DataType::Decimal(_, _) => Arc::new(Decimal128Array::from(data)) as ArrayRef,
dt => panic!("Unexpected data type {:?}", dt),
}
}
Expand Down
116 changes: 77 additions & 39 deletions arrow/src/array/array_decimal.rs
View file
Expand Up @@ -34,17 +34,17 @@ use crate::datatypes::{
use crate::error::{ArrowError, Result};
use crate::util::decimal::{BasicDecimal, Decimal128, Decimal256};

/// `DecimalArray` stores fixed width decimal numbers,
/// `Decimal128Array` stores fixed width decimal numbers,
/// with a fixed precision and scale.
///
/// # Examples
///
/// ```
/// use arrow::array::{Array, BasicDecimalArray, DecimalArray};
/// use arrow::array::{Array, BasicDecimalArray, Decimal128Array};
/// use arrow::datatypes::DataType;
///
/// // Create a DecimalArray with the default precision and scale
/// let decimal_array: DecimalArray = vec![
/// let decimal_array: Decimal128Array = vec![
/// Some(8_887_000_000),
/// None,
/// Some(-8_887_000_000),
Expand All @@ -68,13 +68,50 @@ use crate::util::decimal::{BasicDecimal, Decimal128, Decimal256};
/// assert_eq!(6, decimal_array.scale());
/// ```
///
pub struct DecimalArray {
pub struct Decimal128Array {
data: ArrayData,
value_data: RawPtrBox<u8>,
precision: usize,
scale: usize,
}

/// `DecimalArray` stores fixed width decimal numbers,
/// with a fixed precision and scale.
///
/// # Examples
///
/// ```
/// use arrow::array::{Array, BasicDecimalArray, DecimalArray};
/// use arrow::datatypes::DataType;
///
/// // Create a DecimalArray with the default precision and scale
/// let decimal_array: DecimalArray = vec![
/// Some(8_887_000_000),
/// None,
/// Some(-8_887_000_000),
/// ]
/// .into_iter().collect();
///
/// // set precision and scale so values are interpreted
/// // as `8887.000000`, `Null`, and `-8887.000000`
/// let decimal_array = decimal_array
/// .with_precision_and_scale(23, 6)
/// .unwrap();
///
/// assert_eq!(&DataType::Decimal(23, 6), decimal_array.data_type());
/// assert_eq!(8_887_000_000_i128, decimal_array.value(0).as_i128());
/// assert_eq!("8887.000000", decimal_array.value_as_string(0));
/// assert_eq!(3, decimal_array.len());
/// assert_eq!(1, decimal_array.null_count());
/// assert_eq!(32, decimal_array.value_offset(2));
/// assert_eq!(16, decimal_array.value_length());
/// assert_eq!(23, decimal_array.precision());
/// assert_eq!(6, decimal_array.scale());
/// ```
///
#[deprecated(note = "Please use `Decimal128Array` instead")]
pub type DecimalArray = Decimal128Array;

pub struct Decimal256Array {
data: ArrayData,
value_data: RawPtrBox<u8>,
Expand Down Expand Up @@ -186,13 +223,13 @@ pub trait BasicDecimalArray<T: BasicDecimal, U: From<ArrayData>>:
assert_eq!(
child_data.child_data().len(),
0,
"DecimalArray can only be created from list array of u8 values \
"Decimal128Array can only be created from list array of u8 values \
(i.e. FixedSizeList<PrimitiveArray<u8>>)."
);
assert_eq!(
child_data.data_type(),
&DataType::UInt8,
"DecimalArray can only be created from FixedSizeList<u8> arrays, mismatched data types."
"Decimal128Array can only be created from FixedSizeList<u8> arrays, mismatched data types."
);

let list_offset = v.offset();
Expand All @@ -208,7 +245,7 @@ pub trait BasicDecimalArray<T: BasicDecimal, U: From<ArrayData>>:
}
}

impl BasicDecimalArray<Decimal128, DecimalArray> for DecimalArray {
impl BasicDecimalArray<Decimal128, Decimal128Array> for Decimal128Array {
const VALUE_LENGTH: i32 = 16;

fn data(&self) -> &ArrayData {
Expand Down Expand Up @@ -240,8 +277,8 @@ impl BasicDecimalArray<Decimal256, Decimal256Array> for Decimal256Array {
}
}

impl DecimalArray {
/// Creates a [DecimalArray] with default precision and scale,
impl Decimal128Array {
/// Creates a [Decimal128Array] with default precision and scale,
/// based on an iterator of `i128` values without nulls
pub fn from_iter_values<I: IntoIterator<Item = i128>>(iter: I) -> Self {
let val_buf: Buffer = iter.into_iter().collect();
Expand All @@ -256,10 +293,10 @@ impl DecimalArray {
vec![],
)
};
DecimalArray::from(data)
Decimal128Array::from(data)
}

/// Returns a DecimalArray with the same data as self, with the
/// Returns a Decimal128Array with the same data as self, with the
/// specified precision.
///
/// Returns an Error if:
Expand Down Expand Up @@ -319,12 +356,12 @@ impl DecimalArray {
}
}

impl From<ArrayData> for DecimalArray {
impl From<ArrayData> for Decimal128Array {
fn from(data: ArrayData) -> Self {
assert_eq!(
data.buffers().len(),
1,
"DecimalArray data should contain 1 buffer only (values)"
"Decimal128Array data should contain 1 buffer only (values)"
);
let values = data.buffers()[0].as_ptr();
let (precision, scale) = match data.data_type() {
Expand All @@ -345,7 +382,7 @@ impl From<ArrayData> for Decimal256Array {
assert_eq!(
data.buffers().len(),
1,
"DecimalArray data should contain 1 buffer only (values)"
"Decimal128Array data should contain 1 buffer only (values)"
);
let values = data.buffers()[0].as_ptr();
let (precision, scale) = match data.data_type() {
Expand All @@ -361,7 +398,7 @@ impl From<ArrayData> for Decimal256Array {
}
}

impl<'a> IntoIterator for &'a DecimalArray {
impl<'a> IntoIterator for &'a Decimal128Array {
type Item = Option<i128>;
type IntoIter = DecimalIter<'a>;

Expand All @@ -370,14 +407,14 @@ impl<'a> IntoIterator for &'a DecimalArray {
}
}

impl<'a> DecimalArray {
impl<'a> Decimal128Array {
/// constructs a new iterator
pub fn iter(&'a self) -> DecimalIter<'a> {
DecimalIter::new(self)
}
}

impl<Ptr: Borrow<Option<i128>>> FromIterator<Ptr> for DecimalArray {
impl<Ptr: Borrow<Option<i128>>> FromIterator<Ptr> for Decimal128Array {
fn from_iter<I: IntoIterator<Item = Ptr>>(iter: I) -> Self {
let iter = iter.into_iter();
let (lower, upper) = iter.size_hint();
Expand Down Expand Up @@ -409,7 +446,7 @@ impl<Ptr: Borrow<Option<i128>>> FromIterator<Ptr> for DecimalArray {
vec![],
)
};
DecimalArray::from(data)
Decimal128Array::from(data)
}
}

Expand Down Expand Up @@ -459,12 +496,12 @@ macro_rules! def_decimal_array {
};
}

def_decimal_array!(DecimalArray, "DecimalArray");
def_decimal_array!(Decimal128Array, "Decimal128Array");
def_decimal_array!(Decimal256Array, "Decimal256Array");

#[cfg(test)]
mod tests {
use crate::{array::DecimalBuilder, datatypes::Field};
use crate::{array::Decimal128Builder, datatypes::Field};

use super::*;

Expand All @@ -481,7 +518,7 @@ mod tests {
.add_buffer(Buffer::from(&values[..]))
.build()
.unwrap();
let decimal_array = DecimalArray::from(array_data);
let decimal_array = Decimal128Array::from(array_data);
assert_eq!(8_887_000_000_i128, decimal_array.value(0).into());
assert_eq!(-8_887_000_000_i128, decimal_array.value(1).into());
assert_eq!(16, decimal_array.value_length());
Expand All @@ -490,7 +527,7 @@ mod tests {
#[test]
#[cfg(not(feature = "force_validate"))]
fn test_decimal_append_error_value() {
let mut decimal_builder = DecimalBuilder::new(10, 5, 3);
let mut decimal_builder = Decimal128Builder::new(10, 5, 3);
let mut result = decimal_builder.append_value(123456);
let mut error = result.unwrap_err();
assert_eq!(
Expand All @@ -507,7 +544,7 @@ mod tests {
let arr = decimal_builder.finish();
assert_eq!("12.345", arr.value_as_string(1));

decimal_builder = DecimalBuilder::new(10, 2, 1);
decimal_builder = Decimal128Builder::new(10, 2, 1);
result = decimal_builder.append_value(100);
error = result.unwrap_err();
assert_eq!(
Expand All @@ -531,7 +568,7 @@ mod tests {

#[test]
fn test_decimal_from_iter_values() {
let array = DecimalArray::from_iter_values(vec![-100, 0, 101].into_iter());
let array = Decimal128Array::from_iter_values(vec![-100, 0, 101].into_iter());
assert_eq!(array.len(), 3);
assert_eq!(array.data_type(), &DataType::Decimal(38, 10));
assert_eq!(-100_i128, array.value(0).into());
Expand All @@ -544,7 +581,8 @@ mod tests {

#[test]
fn test_decimal_from_iter() {
let array: DecimalArray = vec![Some(-100), None, Some(101)].into_iter().collect();
let array: Decimal128Array =
vec![Some(-100), None, Some(101)].into_iter().collect();
assert_eq!(array.len(), 3);
assert_eq!(array.data_type(), &DataType::Decimal(38, 10));
assert_eq!(-100_i128, array.value(0).into());
Expand All @@ -557,7 +595,7 @@ mod tests {
#[test]
fn test_decimal_iter() {
let data = vec![Some(-100), None, Some(101)];
let array: DecimalArray = data.clone().into_iter().collect();
let array: Decimal128Array = data.clone().into_iter().collect();

let collected: Vec<_> = array.iter().collect();
assert_eq!(data, collected);
Expand All @@ -566,7 +604,7 @@ mod tests {
#[test]
fn test_decimal_into_iter() {
let data = vec![Some(-100), None, Some(101)];
let array: DecimalArray = data.clone().into_iter().collect();
let array: Decimal128Array = data.clone().into_iter().collect();

let collected: Vec<_> = array.into_iter().collect();
assert_eq!(data, collected);
Expand All @@ -575,7 +613,7 @@ mod tests {
#[test]
fn test_decimal_iter_sized() {
let data = vec![Some(-100), None, Some(101)];
let array: DecimalArray = data.into_iter().collect();
let array: Decimal128Array = data.into_iter().collect();
let mut iter = array.into_iter();

// is exact sized
Expand All @@ -597,7 +635,7 @@ mod tests {
let arr = [123450, -123450, 100, -100, 10, -10, 0]
.into_iter()
.map(Some)
.collect::<DecimalArray>()
.collect::<Decimal128Array>()
.with_precision_and_scale(6, 3)
.unwrap();

Expand All @@ -612,7 +650,7 @@ mod tests {

#[test]
fn test_decimal_array_with_precision_and_scale() {
let arr = DecimalArray::from_iter_values([12345, 456, 7890, -123223423432432])
let arr = Decimal128Array::from_iter_values([12345, 456, 7890, -123223423432432])
.with_precision_and_scale(20, 2)
.unwrap();

Expand All @@ -631,7 +669,7 @@ mod tests {
expected = "-123223423432432 is too small to store in a Decimal of precision 5. Min is -99999"
)]
fn test_decimal_array_with_precision_and_scale_out_of_range() {
DecimalArray::from_iter_values([12345, 456, 7890, -123223423432432])
Decimal128Array::from_iter_values([12345, 456, 7890, -123223423432432])
// precision is too small to hold value
.with_precision_and_scale(5, 2)
.unwrap();
Expand All @@ -640,23 +678,23 @@ mod tests {
#[test]
#[should_panic(expected = "precision 40 is greater than max 38")]
fn test_decimal_array_with_precision_and_scale_invalid_precision() {
DecimalArray::from_iter_values([12345, 456])
Decimal128Array::from_iter_values([12345, 456])
.with_precision_and_scale(40, 2)
.unwrap();
}

#[test]
#[should_panic(expected = "scale 40 is greater than max 38")]
fn test_decimal_array_with_precision_and_scale_invalid_scale() {
DecimalArray::from_iter_values([12345, 456])
Decimal128Array::from_iter_values([12345, 456])
.with_precision_and_scale(20, 40)
.unwrap();
}

#[test]
#[should_panic(expected = "scale 10 is greater than precision 4")]
fn test_decimal_array_with_precision_and_scale_invalid_precision_and_scale() {
DecimalArray::from_iter_values([12345, 456])
Decimal128Array::from_iter_values([12345, 456])
.with_precision_and_scale(4, 10)
.unwrap();
}
Expand All @@ -665,12 +703,12 @@ mod tests {
fn test_decimal_array_fmt_debug() {
let arr = [Some(8887000000), Some(-8887000000), None]
.iter()
.collect::<DecimalArray>()
.collect::<Decimal128Array>()
.with_precision_and_scale(23, 6)
.unwrap();

assert_eq!(
"DecimalArray<23, 6>\n[\n 8887.000000,\n -8887.000000,\n null,\n]",
"Decimal128Array<23, 6>\n[\n 8887.000000,\n -8887.000000,\n null,\n]",
format!("{:?}", arr)
);
}
Expand All @@ -686,7 +724,7 @@ mod tests {
.unwrap();

let binary_array = FixedSizeBinaryArray::from(value_data);
let decimal = DecimalArray::from_fixed_size_binary_array(binary_array, 38, 1);
let decimal = Decimal128Array::from_fixed_size_binary_array(binary_array, 38, 1);

assert_eq!(decimal.len(), 3);
assert_eq!(decimal.value_as_string(0), "0.2".to_string());
Expand All @@ -708,7 +746,7 @@ mod tests {
.unwrap();

let binary_array = FixedSizeBinaryArray::from(value_data);
let _ = DecimalArray::from_fixed_size_binary_array(binary_array, 38, 1);
let _ = Decimal128Array::from_fixed_size_binary_array(binary_array, 38, 1);
}

#[test]
Expand Down Expand Up @@ -736,7 +774,7 @@ mod tests {
.build()
.unwrap();
let list_array = FixedSizeListArray::from(list_data);
let decimal = DecimalArray::from_fixed_size_list_array(list_array, 38, 0);
let decimal = Decimal128Array::from_fixed_size_list_array(list_array, 38, 0);

assert_eq!(decimal.len(), 2);
assert!(decimal.is_null(0));
Expand Down

0 comments on commit 00d66c8

Please sign in to comment.