From 32810bc334091517b719faeb9380a4a839d3257e Mon Sep 17 00:00:00 2001 From: remzi <13716567376yh@gmail.com> Date: Mon, 11 Jul 2022 11:11:37 +0800 Subject: [PATCH 1/4] Add support of converting FixedSizeBinaryArray to Decimal Signed-off-by: remzi <13716567376yh@gmail.com> --- arrow/src/array/array_decimal.rs | 64 +++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/arrow/src/array/array_decimal.rs b/arrow/src/array/array_decimal.rs index 15863cfff71..8dbe7dc4f01 100644 --- a/arrow/src/array/array_decimal.rs +++ b/arrow/src/array/array_decimal.rs @@ -20,10 +20,10 @@ use std::convert::From; use std::fmt; use std::{any::Any, iter::FromIterator}; -use super::BooleanBufferBuilder; use super::{ array::print_long_array, raw_pointer::RawPtrBox, Array, ArrayData, FixedSizeListArray, }; +use super::{BooleanBufferBuilder, FixedSizeBinaryArray}; pub use crate::array::DecimalIter; use crate::buffer::Buffer; use crate::datatypes::DataType; @@ -148,6 +148,32 @@ pub trait BasicDecimalArray>: self.value(row).to_string() } + /// Build a decimal array from [`FixedSizeBinaryArray`]. + /// + /// This function does not check the validation of each + /// value for performance reason. + fn from_fixed_size_binary_array( + v: FixedSizeBinaryArray, + precision: usize, + scale: usize, + ) -> U { + assert!( + v.value_length() == Self::VALUE_LENGTH, + "Value length of the array ({}) must equal to the byte width of the decimal ({})", + v.value_length(), + Self::VALUE_LENGTH, + ); + + let builder = ArrayData::builder(DataType::Decimal(precision, scale)) + .len(v.len()) + .add_buffer(v.value_data()) + .null_bit_buffer(v.data_ref().null_buffer().cloned()) + .offset(v.offset()); + + let array_data = unsafe { builder.build_unchecked() }; + U::from(array_data) + } + fn from_fixed_size_list_array( v: FixedSizeListArray, precision: usize, @@ -646,6 +672,42 @@ mod tests { ); } + #[test] + fn test_decimal_array_from_fixed_size_binary() { + let value_data = ArrayData::builder(DataType::FixedSizeBinary(16)) + .offset(1) + .len(3) + .add_buffer(Buffer::from_slice_ref(&[99999_i128, 2, 34, 560])) + .null_bit_buffer(Some(Buffer::from_slice_ref(&[0b1010]))) + .build() + .unwrap(); + + let binary_array = FixedSizeBinaryArray::from(value_data); + let decimal = DecimalArray::from_fixed_size_binary_array(binary_array, 38, 1); + + assert_eq!(decimal.len(), 3); + assert_eq!(decimal.value_as_string(0), "0.2".to_string()); + assert!(decimal.is_null(1)); + assert_eq!(decimal.value_as_string(2), "56.0".to_string()); + } + + #[test] + #[should_panic( + expected = "Value length of the array (8) must equal to the byte width of the decimal (16)" + )] + fn test_decimal_array_from_fixed_size_binary_wrong_length() { + let value_data = ArrayData::builder(DataType::FixedSizeBinary(8)) + .offset(1) + .len(3) + .add_buffer(Buffer::from_slice_ref(&[99999_i64, 2, 34, 560])) + .null_bit_buffer(Some(Buffer::from_slice_ref(&[0b1010]))) + .build() + .unwrap(); + + let binary_array = FixedSizeBinaryArray::from(value_data); + let _ = DecimalArray::from_fixed_size_binary_array(binary_array, 38, 1); + } + #[test] fn test_decimal_array_from_fixed_size_list() { let value_data = ArrayData::builder(DataType::UInt8) From 775c6c32deb5235a19fcd70b550de047deaa846d Mon Sep 17 00:00:00 2001 From: Remzi Yang <59198230+HaoYang670@users.noreply.github.com> Date: Wed, 13 Jul 2022 09:45:06 +0800 Subject: [PATCH 2/4] Update arrow/src/array/array_decimal.rs Co-authored-by: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> --- arrow/src/array/array_decimal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arrow/src/array/array_decimal.rs b/arrow/src/array/array_decimal.rs index 8dbe7dc4f01..aca3d784c78 100644 --- a/arrow/src/array/array_decimal.rs +++ b/arrow/src/array/array_decimal.rs @@ -150,8 +150,8 @@ pub trait BasicDecimalArray>: /// Build a decimal array from [`FixedSizeBinaryArray`]. /// - /// This function does not check the validation of each - /// value for performance reason. + /// NB: This function does not validate that each value is in the permissible + /// range for a decimal fn from_fixed_size_binary_array( v: FixedSizeBinaryArray, precision: usize, From d31f2e41e0c64066b1955030e0cc5ed342d84130 Mon Sep 17 00:00:00 2001 From: remzi <13716567376yh@gmail.com> Date: Wed, 13 Jul 2022 10:00:12 +0800 Subject: [PATCH 3/4] use better builder API Signed-off-by: remzi <13716567376yh@gmail.com> --- arrow/src/array/array_decimal.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/arrow/src/array/array_decimal.rs b/arrow/src/array/array_decimal.rs index aca3d784c78..ccb1fe05284 100644 --- a/arrow/src/array/array_decimal.rs +++ b/arrow/src/array/array_decimal.rs @@ -163,12 +163,10 @@ pub trait BasicDecimalArray>: v.value_length(), Self::VALUE_LENGTH, ); - - let builder = ArrayData::builder(DataType::Decimal(precision, scale)) - .len(v.len()) - .add_buffer(v.value_data()) - .null_bit_buffer(v.data_ref().null_buffer().cloned()) - .offset(v.offset()); + let builder = v + .into_data() + .into_builder() + .data_type(DataType::Decimal(precision, scale)); let array_data = unsafe { builder.build_unchecked() }; U::from(array_data) From 7c12db4a4dd4d1c0925f3bb32c1eac27192e42dd Mon Sep 17 00:00:00 2001 From: remzi <13716567376yh@gmail.com> Date: Fri, 15 Jul 2022 07:04:48 +0800 Subject: [PATCH 4/4] trigger CI Signed-off-by: remzi <13716567376yh@gmail.com>