From a4a49e7ffc2b040d1034edeb37edff49a6fb4c86 Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Mon, 28 Nov 2022 17:34:50 +0800 Subject: [PATCH 1/4] support cast decimal for round when the option is false --- arrow-cast/src/cast.rs | 107 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/arrow-cast/src/cast.rs b/arrow-cast/src/cast.rs index 07c7d6a3ac5..e7a002bdda5 100644 --- a/arrow-cast/src/cast.rs +++ b/arrow-cast/src/cast.rs @@ -2128,9 +2128,10 @@ fn cast_decimal_to_decimal( if input_scale > output_scale { // For example, input_scale is 4 and output_scale is 3; // Original value is 11234_i128, and will be cast to 1123_i128. - let array = array.as_any().downcast_ref::().unwrap(); if BYTE_WIDTH1 == 16 { + let array = array.as_any().downcast_ref::().unwrap(); if BYTE_WIDTH2 == 16 { + // the div must be greater or equal than 10 let div = 10_i128 .pow_checked((input_scale - output_scale) as u32) .map_err(|_| { @@ -2139,10 +2140,23 @@ fn cast_decimal_to_decimal( *output_scale, )) })?; + let half = div / 2; + let neg_half = -half; array .try_unary::<_, Decimal128Type, _>(|v| { - v.checked_div(div).ok_or_else(|| { + // cast to smaller scale, need to round the result + // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation + let d = v / div; + let r = v % div; + if v >= 0 && r >= half { + d.checked_add(1) + } else if v < 0 && r <= neg_half { + d.checked_sub(1) + } else { + Some(d) + } + .ok_or_else(|| { ArrowError::CastError(format!( "Cannot cast to {:?}({}, {}). Overflowing on {:?}", Decimal128Type::PREFIX, @@ -2166,9 +2180,23 @@ fn cast_decimal_to_decimal( )) })?; + let half = div / i256::from_i128(2_i128); + let neg_half = -half; + array .try_unary::<_, Decimal256Type, _>(|v| { - i256::from_i128(v).checked_div(div).ok_or_else(|| { + // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation + let v = i256::from_i128(v); + let d = v / div; + let r = v % div; + if v >= i256::ZERO && r >= half { + d.checked_add(i256::ONE) + } else if v < i256::ZERO && r <= neg_half { + d.checked_sub(i256::ONE) + } else { + Some(d) + } + .ok_or_else(|| { ArrowError::CastError(format!( "Cannot cast to {:?}({}, {}). Overflowing on {:?}", Decimal256Type::PREFIX, @@ -2193,10 +2221,21 @@ fn cast_decimal_to_decimal( *output_scale, )) })?; + let half = div / i256::from_i128(2_i128); + let neg_half = -half; if BYTE_WIDTH2 == 16 { array .try_unary::<_, Decimal128Type, _>(|v| { - v.checked_div(div).ok_or_else(|| { + // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation + let d = v / div; + let r = v % div; + if v >= i256::ZERO && r >= half { + d.checked_add(i256::ONE) + } else if v < i256::ZERO && r <= neg_half { + d.checked_sub(i256::ONE) + } else { + Some(d) + }.ok_or_else(|| { ArrowError::CastError(format!( "Cannot cast to {:?}({}, {}). Overflowing on {:?}", Decimal128Type::PREFIX, @@ -2217,7 +2256,17 @@ fn cast_decimal_to_decimal( } else { array .try_unary::<_, Decimal256Type, _>(|v| { - v.checked_div(div).ok_or_else(|| { + // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation + let d = v / div; + let r = v % div; + if v >= i256::ZERO && r >= half { + d.checked_add(i256::ONE) + } else if v < i256::ZERO && r <= neg_half { + d.checked_sub(i256::ONE) + } else { + Some(d) + } + .ok_or_else(|| { ArrowError::CastError(format!( "Cannot cast to {:?}({}, {}). Overflowing on {:?}", Decimal256Type::PREFIX, @@ -3588,6 +3637,26 @@ mod tests { } } } + + let cast_option = CastOptions { safe: false }; + let casted_array_with_option = + cast_with_options($INPUT_ARRAY, $OUTPUT_TYPE, &cast_option).unwrap(); + let result_array = casted_array_with_option + .as_any() + .downcast_ref::<$OUTPUT_TYPE_ARRAY>() + .unwrap(); + assert_eq!($OUTPUT_TYPE, result_array.data_type()); + assert_eq!(result_array.len(), $OUTPUT_VALUES.len()); + for (i, x) in $OUTPUT_VALUES.iter().enumerate() { + match x { + Some(x) => { + assert_eq!(result_array.value(i), *x); + } + None => { + assert!(result_array.is_null(i)); + } + } + } }; } @@ -3795,6 +3864,34 @@ mod tests { result.unwrap_err().to_string()); } + #[test] + fn test_cast_decimal256_to_decimal128_overflow() { + let input_type = DataType::Decimal256(76, 5); + let output_type = DataType::Decimal128(38, 7); + assert!(can_cast_types(&input_type, &output_type)); + let array = vec![Some(i256::from_i128(i128::MAX))]; + let input_decimal_array = create_decimal256_array(array, 76, 5).unwrap(); + let array = Arc::new(input_decimal_array) as ArrayRef; + let result = + cast_with_options(&array, &output_type, &CastOptions { safe: false }); + assert_eq!("Invalid argument error: 17014118346046923173168730371588410572700 cannot be casted to 128-bit integer for Decimal128", + result.unwrap_err().to_string()); + } + + #[test] + fn test_cast_decimal256_to_decimal256_overflow() { + let input_type = DataType::Decimal256(76, 5); + let output_type = DataType::Decimal256(76, 55); + assert!(can_cast_types(&input_type, &output_type)); + let array = vec![Some(i256::from_i128(i128::MAX))]; + let input_decimal_array = create_decimal256_array(array, 76, 5).unwrap(); + let array = Arc::new(input_decimal_array) as ArrayRef; + let result = + cast_with_options(&array, &output_type, &CastOptions { safe: false }); + assert_eq!("Cast error: Cannot cast to \"Decimal256\"(76, 55). Overflowing on 170141183460469231731687303715884105727", + result.unwrap_err().to_string()); + } + #[test] fn test_cast_decimal128_to_decimal256() { let input_type = DataType::Decimal128(20, 3); From 01dd8bafbc46e4ab5c04c4c52f3a1d263179ef41 Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Tue, 29 Nov 2022 17:49:34 +0800 Subject: [PATCH 2/4] fix conflict after merge --- arrow-cast/src/cast.rs | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/arrow-cast/src/cast.rs b/arrow-cast/src/cast.rs index 1abb3598c56..3861e8b1638 100644 --- a/arrow-cast/src/cast.rs +++ b/arrow-cast/src/cast.rs @@ -3864,34 +3864,6 @@ mod tests { result.unwrap_err().to_string()); } - #[test] - fn test_cast_decimal256_to_decimal128_overflow() { - let input_type = DataType::Decimal256(76, 5); - let output_type = DataType::Decimal128(38, 7); - assert!(can_cast_types(&input_type, &output_type)); - let array = vec![Some(i256::from_i128(i128::MAX))]; - let input_decimal_array = create_decimal256_array(array, 76, 5).unwrap(); - let array = Arc::new(input_decimal_array) as ArrayRef; - let result = - cast_with_options(&array, &output_type, &CastOptions { safe: false }); - assert_eq!("Invalid argument error: 17014118346046923173168730371588410572700 cannot be casted to 128-bit integer for Decimal128", - result.unwrap_err().to_string()); - } - - #[test] - fn test_cast_decimal256_to_decimal256_overflow() { - let input_type = DataType::Decimal256(76, 5); - let output_type = DataType::Decimal256(76, 55); - assert!(can_cast_types(&input_type, &output_type)); - let array = vec![Some(i256::from_i128(i128::MAX))]; - let input_decimal_array = create_decimal256_array(array, 76, 5).unwrap(); - let array = Arc::new(input_decimal_array) as ArrayRef; - let result = - cast_with_options(&array, &output_type, &CastOptions { safe: false }); - assert_eq!("Cast error: Cannot cast to \"Decimal256\"(76, 55). Overflowing on 170141183460469231731687303715884105727", - result.unwrap_err().to_string()); - } - #[test] fn test_cast_decimal128_to_decimal256() { let input_type = DataType::Decimal128(20, 3); From d2fa48b174bd8a381505a9f221be22eea76e488a Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Wed, 30 Nov 2022 22:00:40 +0800 Subject: [PATCH 3/4] fix error case --- arrow-cast/src/cast.rs | 65 ++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/arrow-cast/src/cast.rs b/arrow-cast/src/cast.rs index 47776990a42..1480e03729d 100644 --- a/arrow-cast/src/cast.rs +++ b/arrow-cast/src/cast.rs @@ -3682,6 +3682,43 @@ mod tests { .with_precision_and_scale(precision, scale) } + #[test] + #[cfg(not(feature = "force_validate"))] + #[should_panic( + expected = "5789604461865809771178549250434395392663499233282028201972879200395656481997 cannot be casted to 128-bit integer for Decimal128" + )] + fn test_cast_decimal_to_decimal_round_with_error() { + // decimal256 to decimal128 overflow + let array = vec![ + Some(i256::from_i128(1123454)), + Some(i256::from_i128(2123456)), + Some(i256::from_i128(-3123453)), + Some(i256::from_i128(-3123456)), + None, + Some(i256::MAX), + Some(i256::MIN), + ]; + let input_decimal_array = create_decimal256_array(array, 76, 4).unwrap(); + let array = Arc::new(input_decimal_array) as ArrayRef; + let input_type = DataType::Decimal256(76, 4); + let output_type = DataType::Decimal128(20, 3); + assert!(can_cast_types(&input_type, &output_type)); + generate_cast_test_case!( + &array, + Decimal128Array, + &output_type, + vec![ + Some(112345_i128), + Some(212346_i128), + Some(-312345_i128), + Some(-312346_i128), + None, + None, + None, + ] + ); + } + #[test] #[cfg(not(feature = "force_validate"))] fn test_cast_decimal_to_decimal_round() { @@ -3771,34 +3808,6 @@ mod tests { None ] ); - - // decimal256 to decimal128 overflow - let array = vec![ - Some(i256::from_i128(1123454)), - Some(i256::from_i128(2123456)), - Some(i256::from_i128(-3123453)), - Some(i256::from_i128(-3123456)), - None, - Some(i256::MAX), - Some(i256::MIN), - ]; - let input_decimal_array = create_decimal256_array(array, 76, 4).unwrap(); - let array = Arc::new(input_decimal_array) as ArrayRef; - assert!(can_cast_types(&input_type, &output_type)); - generate_cast_test_case!( - &array, - Decimal128Array, - &output_type, - vec![ - Some(112345_i128), - Some(212346_i128), - Some(-312345_i128), - Some(-312346_i128), - None, - None, - None - ] - ); } #[test] From 477d1385fd3b8838a2f0e5bae5923d7743aa9abf Mon Sep 17 00:00:00 2001 From: liukun4515 Date: Thu, 1 Dec 2022 14:13:45 +0800 Subject: [PATCH 4/4] change to wrapping api --- arrow-cast/src/cast.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arrow-cast/src/cast.rs b/arrow-cast/src/cast.rs index 3f0b3d1be4e..30ae278ebe0 100644 --- a/arrow-cast/src/cast.rs +++ b/arrow-cast/src/cast.rs @@ -2149,8 +2149,8 @@ fn cast_decimal_to_decimal( .try_unary::<_, Decimal128Type, _>(|v| { // cast to smaller scale, need to round the result // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation - let d = v / div; - let r = v % div; + let d = v.wrapping_div(div); + let r = v.wrapping_rem(div); if v >= 0 && r >= half { d.checked_add(1) } else if v < 0 && r <= neg_half { @@ -2189,8 +2189,8 @@ fn cast_decimal_to_decimal( .try_unary::<_, Decimal256Type, _>(|v| { // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation let v = i256::from_i128(v); - let d = v / div; - let r = v % div; + let d = v.wrapping_div(div); + let r = v.wrapping_rem(div); if v >= i256::ZERO && r >= half { d.checked_add(i256::ONE) } else if v < i256::ZERO && r <= neg_half { @@ -2229,8 +2229,8 @@ fn cast_decimal_to_decimal( array .try_unary::<_, Decimal128Type, _>(|v| { // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation - let d = v / div; - let r = v % div; + let d = v.wrapping_div(div); + let r = v.wrapping_rem(div); if v >= i256::ZERO && r >= half { d.checked_add(i256::ONE) } else if v < i256::ZERO && r <= neg_half { @@ -2259,8 +2259,8 @@ fn cast_decimal_to_decimal( array .try_unary::<_, Decimal256Type, _>(|v| { // the div must be gt_eq 10, we don't need to check the overflow for the `div`/`mod` operation - let d = v / div; - let r = v % div; + let d = v.wrapping_div(div); + let r = v.wrapping_rem(div); if v >= i256::ZERO && r >= half { d.checked_add(i256::ONE) } else if v < i256::ZERO && r <= neg_half {