From e725bd147960b27a119800d88be1a6baa3cb49ec Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 25 Jul 2022 15:13:11 -0700 Subject: [PATCH 1/3] Use ArrayAccessor --- arrow/src/compute/kernels/cast.rs | 140 ++++++++++++++++++------------ 1 file changed, 84 insertions(+), 56 deletions(-) diff --git a/arrow/src/compute/kernels/cast.rs b/arrow/src/compute/kernels/cast.rs index 781f199a691..be0a00b5050 100644 --- a/arrow/src/compute/kernels/cast.rs +++ b/arrow/src/compute/kernels/cast.rs @@ -38,7 +38,6 @@ use std::str; use std::sync::Arc; -use crate::array::BasicDecimalArray; use crate::buffer::MutableBuffer; use crate::compute::divide_scalar; use crate::compute::kernels::arithmetic::{divide, multiply}; @@ -270,45 +269,62 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result { cast_with_options(array, to_type, &DEFAULT_CAST_OPTIONS) } -// cast the integer array to defined decimal data type array -macro_rules! cast_integer_to_decimal { - ($ARRAY: expr, $ARRAY_TYPE: ident, $PRECISION : ident, $SCALE : ident) => {{ - let array = $ARRAY.as_any().downcast_ref::<$ARRAY_TYPE>().unwrap(); - let mul: i128 = 10_i128.pow(*$SCALE as u32); - let decimal_array = array - .iter() - .map(|v| { - v.map(|v| { - let v = v as i128; - // with_precision_and_scale validates the - // value is within range for the output precision - mul * v - }) - }) - .collect::() - .with_precision_and_scale(*$PRECISION, *$SCALE)?; - Ok(Arc::new(decimal_array)) - }}; +/// Cast the primitive array to defined decimal data type array +fn cast_primitive_to_decimal( + array: T, + op: F, + precision: usize, + scale: usize, +) -> Result> +where + F: Fn(T::Item) -> i128, +{ + #[allow(clippy::redundant_closure)] + let decimal_array = ArrayIter::new(array) + .map(|v| v.map(|v| op(v))) + .collect::() + .with_precision_and_scale(precision, scale)?; + + Ok(Arc::new(decimal_array)) } -// cast the floating-point array to defined decimal data type array -macro_rules! cast_floating_point_to_decimal { - ($ARRAY: expr, $ARRAY_TYPE: ident, $PRECISION : ident, $SCALE : ident) => {{ - let array = $ARRAY.as_any().downcast_ref::<$ARRAY_TYPE>().unwrap(); - let mul = 10_f64.powi(*$SCALE as i32); - let decimal_array = array - .iter() - .map(|v| { - v.map(|v| { - // with_precision_and_scale validates the - // value is within range for the output precision - ((v as f64) * mul) as i128 - }) - }) - .collect::() - .with_precision_and_scale(*$PRECISION, *$SCALE)?; - Ok(Arc::new(decimal_array)) - }}; +fn cast_integer_to_decimal( + array: &PrimitiveArray, + precision: usize, + scale: usize, +) -> Result> { + let mul: i128 = 10_i128.pow(scale as u32); + + // with_precision_and_scale validates the + // value is within range for the output precision + cast_primitive_to_decimal( + array, + |v| v.to_isize().unwrap() as i128 * mul, + precision, + scale, + ) +} + +fn cast_floating_point_to_decimal( + array: &PrimitiveArray, + precision: usize, + scale: usize, +) -> Result> +where + ::Native: ToPrimitive, +{ + let mul = 10_f64.powi(scale as i32); + + cast_primitive_to_decimal( + array, + |v| { + // with_precision_and_scale validates the + // value is within range for the output precision + ((num::ToPrimitive::to_f64(&v).unwrap()) * mul) as i128 + }, + precision, + scale, + ) } // cast the decimal array to integer array @@ -428,24 +444,36 @@ pub fn cast_with_options( // cast data to decimal match from_type { // TODO now just support signed numeric to decimal, support decimal to numeric later - Int8 => { - cast_integer_to_decimal!(array, Int8Array, precision, scale) - } - Int16 => { - cast_integer_to_decimal!(array, Int16Array, precision, scale) - } - Int32 => { - cast_integer_to_decimal!(array, Int32Array, precision, scale) - } - Int64 => { - cast_integer_to_decimal!(array, Int64Array, precision, scale) - } - Float32 => { - cast_floating_point_to_decimal!(array, Float32Array, precision, scale) - } - Float64 => { - cast_floating_point_to_decimal!(array, Float64Array, precision, scale) - } + Int8 => cast_integer_to_decimal( + as_primitive_array::(array), + *precision, + *scale, + ), + Int16 => cast_integer_to_decimal( + as_primitive_array::(array), + *precision, + *scale, + ), + Int32 => cast_integer_to_decimal( + as_primitive_array::(array), + *precision, + *scale, + ), + Int64 => cast_integer_to_decimal( + as_primitive_array::(array), + *precision, + *scale, + ), + Float32 => cast_floating_point_to_decimal( + as_primitive_array::(array), + *precision, + *scale, + ), + Float64 => cast_floating_point_to_decimal( + as_primitive_array::(array), + *precision, + *scale, + ), Null => Ok(new_null_array(to_type, array.len())), _ => Err(ArrowError::CastError(format!( "Casting from {:?} to {:?} not supported", From c775d608600ad90a4eaf01d84467b0babbafc62d Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Tue, 26 Jul 2022 16:11:37 -0700 Subject: [PATCH 2/3] Add to_i128 and to_f64 to ArrowNumericType --- arrow/src/compute/kernels/cast.rs | 9 +--- arrow/src/datatypes/native.rs | 71 ++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/arrow/src/compute/kernels/cast.rs b/arrow/src/compute/kernels/cast.rs index be0a00b5050..2b7abafd75b 100644 --- a/arrow/src/compute/kernels/cast.rs +++ b/arrow/src/compute/kernels/cast.rs @@ -297,12 +297,7 @@ fn cast_integer_to_decimal( // with_precision_and_scale validates the // value is within range for the output precision - cast_primitive_to_decimal( - array, - |v| v.to_isize().unwrap() as i128 * mul, - precision, - scale, - ) + cast_primitive_to_decimal(array, |v| v.to_i128().unwrap() * mul, precision, scale) } fn cast_floating_point_to_decimal( @@ -320,7 +315,7 @@ where |v| { // with_precision_and_scale validates the // value is within range for the output precision - ((num::ToPrimitive::to_f64(&v).unwrap()) * mul) as i128 + (ArrowNativeType::to_f64(&v).unwrap() * mul) as i128 }, precision, scale, diff --git a/arrow/src/datatypes/native.rs b/arrow/src/datatypes/native.rs index d9a3f667d8e..576759e2c50 100644 --- a/arrow/src/datatypes/native.rs +++ b/arrow/src/datatypes/native.rs @@ -79,6 +79,18 @@ pub trait ArrowNativeType: None } + /// Convert native type to i128. + #[inline] + fn to_i128(&self) -> Option { + None + } + + /// Convert native type to f64. + #[inline] + fn to_f64(&self) -> Option { + None + } + /// Convert native type from i32. #[inline] fn from_i32(_: i32) -> Option { @@ -148,6 +160,11 @@ impl ArrowNativeType for i8 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for i16 { @@ -172,6 +189,11 @@ impl ArrowNativeType for i16 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for i32 { @@ -202,6 +224,11 @@ impl ArrowNativeType for i32 { fn from_i32(val: i32) -> Option { Some(val) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for i64 { @@ -232,6 +259,11 @@ impl ArrowNativeType for i64 { fn from_i64(val: i64) -> Option { Some(val) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for i128 { @@ -266,6 +298,11 @@ impl ArrowNativeType for i128 { fn from_i128(val: i128) -> Option { Some(val) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self) + } } impl JsonSerializable for u8 { @@ -290,6 +327,11 @@ impl ArrowNativeType for u8 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for u16 { @@ -314,6 +356,11 @@ impl ArrowNativeType for u16 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for u32 { @@ -338,6 +385,11 @@ impl ArrowNativeType for u32 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for u64 { @@ -362,6 +414,11 @@ impl ArrowNativeType for u64 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } + + #[inline] + fn to_i128(&self) -> Option { + Some(*self as i128) + } } impl JsonSerializable for f16 { @@ -384,9 +441,19 @@ impl JsonSerializable for f64 { impl ArrowNativeType for f16 {} impl private::Sealed for f16 {} -impl ArrowNativeType for f32 {} +impl ArrowNativeType for f32 { + #[inline] + fn to_f64(&self) -> Option { + Some(*self as f64) + } +} impl private::Sealed for f32 {} -impl ArrowNativeType for f64 {} +impl ArrowNativeType for f64 { + #[inline] + fn to_f64(&self) -> Option { + Some(*self) + } +} impl private::Sealed for f64 {} /// Allows conversion from supported Arrow types to a byte slice. From 3ca49111635df55b2c4bd8e97588c43347357821 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 27 Jul 2022 11:49:19 -0700 Subject: [PATCH 3/3] Use AsPrimitive --- arrow/src/compute/kernels/cast.rs | 12 ++++-- arrow/src/datatypes/native.rs | 71 +------------------------------ 2 files changed, 10 insertions(+), 73 deletions(-) diff --git a/arrow/src/compute/kernels/cast.rs b/arrow/src/compute/kernels/cast.rs index 2b7abafd75b..25aa525b452 100644 --- a/arrow/src/compute/kernels/cast.rs +++ b/arrow/src/compute/kernels/cast.rs @@ -47,6 +47,7 @@ use crate::datatypes::*; use crate::error::{ArrowError, Result}; use crate::{array::*, compute::take}; use crate::{buffer::Buffer, util::serialization::lexical_to_string}; +use num::cast::AsPrimitive; use num::{NumCast, ToPrimitive}; /// CastOptions provides a way to override the default cast behaviors @@ -292,12 +293,15 @@ fn cast_integer_to_decimal( array: &PrimitiveArray, precision: usize, scale: usize, -) -> Result> { +) -> Result> +where + ::Native: AsPrimitive, +{ let mul: i128 = 10_i128.pow(scale as u32); // with_precision_and_scale validates the // value is within range for the output precision - cast_primitive_to_decimal(array, |v| v.to_i128().unwrap() * mul, precision, scale) + cast_primitive_to_decimal(array, |v| v.as_() * mul, precision, scale) } fn cast_floating_point_to_decimal( @@ -306,7 +310,7 @@ fn cast_floating_point_to_decimal( scale: usize, ) -> Result> where - ::Native: ToPrimitive, + ::Native: AsPrimitive, { let mul = 10_f64.powi(scale as i32); @@ -315,7 +319,7 @@ where |v| { // with_precision_and_scale validates the // value is within range for the output precision - (ArrowNativeType::to_f64(&v).unwrap() * mul) as i128 + (v.as_() * mul) as i128 }, precision, scale, diff --git a/arrow/src/datatypes/native.rs b/arrow/src/datatypes/native.rs index 576759e2c50..d9a3f667d8e 100644 --- a/arrow/src/datatypes/native.rs +++ b/arrow/src/datatypes/native.rs @@ -79,18 +79,6 @@ pub trait ArrowNativeType: None } - /// Convert native type to i128. - #[inline] - fn to_i128(&self) -> Option { - None - } - - /// Convert native type to f64. - #[inline] - fn to_f64(&self) -> Option { - None - } - /// Convert native type from i32. #[inline] fn from_i32(_: i32) -> Option { @@ -160,11 +148,6 @@ impl ArrowNativeType for i8 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for i16 { @@ -189,11 +172,6 @@ impl ArrowNativeType for i16 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for i32 { @@ -224,11 +202,6 @@ impl ArrowNativeType for i32 { fn from_i32(val: i32) -> Option { Some(val) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for i64 { @@ -259,11 +232,6 @@ impl ArrowNativeType for i64 { fn from_i64(val: i64) -> Option { Some(val) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for i128 { @@ -298,11 +266,6 @@ impl ArrowNativeType for i128 { fn from_i128(val: i128) -> Option { Some(val) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self) - } } impl JsonSerializable for u8 { @@ -327,11 +290,6 @@ impl ArrowNativeType for u8 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for u16 { @@ -356,11 +314,6 @@ impl ArrowNativeType for u16 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for u32 { @@ -385,11 +338,6 @@ impl ArrowNativeType for u32 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for u64 { @@ -414,11 +362,6 @@ impl ArrowNativeType for u64 { fn to_isize(&self) -> Option { num::ToPrimitive::to_isize(self) } - - #[inline] - fn to_i128(&self) -> Option { - Some(*self as i128) - } } impl JsonSerializable for f16 { @@ -441,19 +384,9 @@ impl JsonSerializable for f64 { impl ArrowNativeType for f16 {} impl private::Sealed for f16 {} -impl ArrowNativeType for f32 { - #[inline] - fn to_f64(&self) -> Option { - Some(*self as f64) - } -} +impl ArrowNativeType for f32 {} impl private::Sealed for f32 {} -impl ArrowNativeType for f64 { - #[inline] - fn to_f64(&self) -> Option { - Some(*self) - } -} +impl ArrowNativeType for f64 {} impl private::Sealed for f64 {} /// Allows conversion from supported Arrow types to a byte slice.