From c92945c8fb77911d53cd5e4daebfb6513ead6566 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 14 Oct 2022 20:10:41 +1300 Subject: [PATCH 1/2] Don't validate decimal precision in ArrayData (#2637) --- arrow-data/src/data.rs | 19 ------------------- arrow/tests/array_validation.rs | 9 ++++++--- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/arrow-data/src/data.rs b/arrow-data/src/data.rs index 37c059748fe..b53e9f0af4d 100644 --- a/arrow-data/src/data.rs +++ b/arrow-data/src/data.rs @@ -18,9 +18,6 @@ //! Contains `ArrayData`, a generic representation of Arrow array data which encapsulates //! common attributes and operations for Arrow array. -use crate::decimal::{ - validate_decimal256_precision_with_lt_bytes, validate_decimal_precision, -}; use crate::{bit_iterator::BitSliceIterator, bitmap::Bitmap}; use arrow_buffer::{bit_util, ArrowNativeType, Buffer, MutableBuffer}; use arrow_schema::{ArrowError, DataType, IntervalUnit, UnionMode}; @@ -1004,22 +1001,6 @@ impl ArrayData { pub fn validate_values(&self) -> Result<(), ArrowError> { match &self.data_type { - DataType::Decimal128(p, _) => { - let values_buffer: &[i128] = self.typed_buffer(0, self.len)?; - for value in values_buffer { - validate_decimal_precision(*value, *p)?; - } - Ok(()) - } - DataType::Decimal256(p, _) => { - let values = self.buffers()[0].as_slice(); - for pos in 0..self.len() { - let offset = pos * 32; - let raw_bytes = &values[offset..offset + 32]; - validate_decimal256_precision_with_lt_bytes(raw_bytes, *p)?; - } - Ok(()) - } DataType::Utf8 => self.validate_utf8::(), DataType::LargeUtf8 => self.validate_utf8::(), DataType::Binary => self.validate_offsets_full::(self.buffers[1].len()), diff --git a/arrow/tests/array_validation.rs b/arrow/tests/array_validation.rs index f4dcda2e8de..c4de65fcf5e 100644 --- a/arrow/tests/array_validation.rs +++ b/arrow/tests/array_validation.rs @@ -25,6 +25,7 @@ use arrow_data::ArrayData; use arrow_schema::{DataType, Field, UnionMode}; use std::ptr::NonNull; use std::sync::Arc; +use arrow_array::Decimal128Array; #[test] #[should_panic( @@ -1038,7 +1039,6 @@ fn test_string_data_from_foreign() { } #[test] -#[cfg(not(feature = "force_validate"))] fn test_decimal_full_validation() { let values_builder = UInt8Builder::with_capacity(10); let byte_width = 16; @@ -1055,8 +1055,11 @@ fn test_decimal_full_validation() { .len(fixed_size_array.len()) .add_buffer(fixed_size_array.data_ref().child_data()[0].buffers()[0].clone()); let array_data = unsafe { builder.build_unchecked() }; - let validation_result = array_data.validate_full(); - let error = validation_result.unwrap_err(); + array_data.validate_full().unwrap(); + + let array = Decimal128Array::from(array_data); + let error = array.validate_decimal_precision(array.precision()).unwrap_err(); + assert_eq!( "Invalid argument error: 123456 is too large to store in a Decimal128 of precision 5. Max is 99999", error.to_string() From 4e2687a812068757117c50d5ab356c010b2c4f73 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Sat, 15 Oct 2022 09:35:18 +1300 Subject: [PATCH 2/2] Format --- arrow/tests/array_validation.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arrow/tests/array_validation.rs b/arrow/tests/array_validation.rs index c4de65fcf5e..16f031a1eb1 100644 --- a/arrow/tests/array_validation.rs +++ b/arrow/tests/array_validation.rs @@ -20,12 +20,12 @@ use arrow::array::{ Int32Array, Int32Builder, Int64Array, StringArray, StructBuilder, UInt64Array, UInt8Builder, }; +use arrow_array::Decimal128Array; use arrow_buffer::{ArrowNativeType, Buffer}; use arrow_data::ArrayData; use arrow_schema::{DataType, Field, UnionMode}; use std::ptr::NonNull; use std::sync::Arc; -use arrow_array::Decimal128Array; #[test] #[should_panic( @@ -1058,7 +1058,9 @@ fn test_decimal_full_validation() { array_data.validate_full().unwrap(); let array = Decimal128Array::from(array_data); - let error = array.validate_decimal_precision(array.precision()).unwrap_err(); + let error = array + .validate_decimal_precision(array.precision()) + .unwrap_err(); assert_eq!( "Invalid argument error: 123456 is too large to store in a Decimal128 of precision 5. Max is 99999",