Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rename DataType::Decimal to DataType::Decimal128 #2229

Merged
merged 3 commits into from Jul 30, 2022
Merged

Rename DataType::Decimal to DataType::Decimal128 #2229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
32f3605
Rename DataType::Decimal to DataType::Decimal128
viirya Jul 29, 2022
c92d2ab
Update doc
viirya Jul 29, 2022
b6ec3a1
Merge remote-tracking branch 'upstream/master' into decimal128_type
viirya Jul 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions arrow/src/array/array.rs
View file
Open in desktop
Expand Up @@ -482,7 +482,7 @@ pub fn make_array(data: ArrayData) -> ArrayRef {
dt => panic!("Unexpected dictionary key type {:?}", dt),
},
DataType::Null => Arc::new(NullArray::from(data)) as ArrayRef,
DataType::Decimal(_, _) => Arc::new(Decimal128Array::from(data)) as ArrayRef,
DataType::Decimal128(_, _) => Arc::new(Decimal128Array::from(data)) as ArrayRef,
DataType::Decimal256(_, _) => Arc::new(Decimal256Array::from(data)) as ArrayRef,
dt => panic!("Unexpected data type {:?}", dt),
}
Expand Down Expand Up @@ -647,7 +647,7 @@ pub fn new_null_array(data_type: &DataType, length: usize) -> ArrayRef {
)
})
}
DataType::Decimal(_, _) => {
DataType::Decimal128(_, _) => {
new_null_sized_decimal(data_type, length, std::mem::size_of::<i128>())
}
DataType::Decimal256(_, _) => new_null_sized_decimal(data_type, length, 32),
Expand Down
22 changes: 11 additions & 11 deletions arrow/src/array/array_decimal.rs
View file
Open in desktop
Expand Up @@ -56,7 +56,7 @@ use crate::util::decimal::{BasicDecimal, Decimal128, Decimal256};
/// .with_precision_and_scale(23, 6)
/// .unwrap();
///
/// assert_eq!(&DataType::Decimal(23, 6), decimal_array.data_type());
/// assert_eq!(&DataType::Decimal128(23, 6), decimal_array.data_type());
/// assert_eq!(8_887_000_000_i128, decimal_array.value(0).as_i128());
/// assert_eq!("8887.000000", decimal_array.value_as_string(0));
/// assert_eq!(3, decimal_array.len());
Expand Down Expand Up @@ -170,7 +170,7 @@ pub trait BasicDecimalArray<T: BasicDecimal, U: From<ArrayData>>:
Self::VALUE_LENGTH,
);
let data_type = if Self::VALUE_LENGTH == 16 {
DataType::Decimal(precision, scale)
DataType::Decimal128(precision, scale)
} else {
DataType::Decimal256(precision, scale)
};
Expand Down Expand Up @@ -206,7 +206,7 @@ pub trait BasicDecimalArray<T: BasicDecimal, U: From<ArrayData>>:
let list_offset = v.offset();
let child_offset = child_data.offset();
let data_type = if Self::VALUE_LENGTH == 16 {
DataType::Decimal(precision, scale)
DataType::Decimal128(precision, scale)
} else {
DataType::Decimal256(precision, scale)
};
Expand Down Expand Up @@ -314,11 +314,11 @@ impl Decimal128Array {

assert_eq!(
self.data.data_type(),
&DataType::Decimal(self.precision, self.scale)
&DataType::Decimal128(self.precision, self.scale)
);

// safety: self.data is valid DataType::Decimal as checked above
let new_data_type = DataType::Decimal(precision, scale);
let new_data_type = DataType::Decimal128(precision, scale);
self.precision = precision;
self.scale = scale;
self.data = self.data.with_data_type(new_data_type);
Expand All @@ -328,7 +328,7 @@ impl Decimal128Array {
/// The default precision and scale used when not specified.
pub fn default_type() -> DataType {
// Keep maximum precision
DataType::Decimal(DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE)
DataType::Decimal128(DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE)
}
}

Expand All @@ -341,7 +341,7 @@ impl From<ArrayData> for Decimal128Array {
);
let values = data.buffers()[0].as_ptr();
let (precision, scale) = match data.data_type() {
DataType::Decimal(precision, scale) => (*precision, *scale),
DataType::Decimal128(precision, scale) => (*precision, *scale),
_ => panic!("Expected data type to be Decimal"),
};
Self {
Expand Down Expand Up @@ -523,7 +523,7 @@ mod tests {
192, 219, 180, 17, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 36, 75, 238, 253,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
];
let array_data = ArrayData::builder(DataType::Decimal(38, 6))
let array_data = ArrayData::builder(DataType::Decimal128(38, 6))
.len(2)
.add_buffer(Buffer::from(&values[..]))
.build()
Expand Down Expand Up @@ -580,7 +580,7 @@ mod tests {
fn test_decimal_from_iter_values() {
let array = Decimal128Array::from_iter_values(vec![-100, 0, 101].into_iter());
assert_eq!(array.len(), 3);
assert_eq!(array.data_type(), &DataType::Decimal(38, 10));
assert_eq!(array.data_type(), &DataType::Decimal128(38, 10));
assert_eq!(-100_i128, array.value(0).into());
assert!(!array.is_null(0));
assert_eq!(0_i128, array.value(1).into());
Expand All @@ -594,7 +594,7 @@ mod tests {
let array: Decimal128Array =
vec![Some(-100), None, Some(101)].into_iter().collect();
assert_eq!(array.len(), 3);
assert_eq!(array.data_type(), &DataType::Decimal(38, 10));
assert_eq!(array.data_type(), &DataType::Decimal128(38, 10));
assert_eq!(-100_i128, array.value(0).into());
assert!(!array.is_null(0));
assert!(array.is_null(1));
Expand Down Expand Up @@ -665,7 +665,7 @@ mod tests {
.with_precision_and_scale(20, 2)
.unwrap();

assert_eq!(arr.data_type(), &DataType::Decimal(20, 2));
assert_eq!(arr.data_type(), &DataType::Decimal128(20, 2));
assert_eq!(arr.precision(), 20);
assert_eq!(arr.scale(), 2);

Expand Down
4 changes: 2 additions & 2 deletions arrow/src/array/builder/decimal_builder.rs
View file
Open in desktop
Expand Up @@ -274,7 +274,7 @@ mod tests {
builder.append_option(Some(8_887_000_000_i128)).unwrap();
let decimal_array: Decimal128Array = builder.finish();

assert_eq!(&DataType::Decimal(38, 6), decimal_array.data_type());
assert_eq!(&DataType::Decimal128(38, 6), decimal_array.data_type());
assert_eq!(5, decimal_array.len());
assert_eq!(2, decimal_array.null_count());
assert_eq!(32, decimal_array.value_offset(2));
Expand All @@ -294,7 +294,7 @@ mod tests {
.unwrap();
let decimal_array: Decimal128Array = builder.finish();

assert_eq!(&DataType::Decimal(38, 6), decimal_array.data_type());
assert_eq!(&DataType::Decimal128(38, 6), decimal_array.data_type());
assert_eq!(3, decimal_array.len());
assert_eq!(1, decimal_array.null_count());
assert_eq!(32, decimal_array.value_offset(2));
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/array/builder/struct_builder.rs
View file
Open in desktop
Expand Up @@ -112,7 +112,7 @@ pub fn make_builder(datatype: &DataType, capacity: usize) -> Box<dyn ArrayBuilde
DataType::FixedSizeBinary(len) => {
Box::new(FixedSizeBinaryBuilder::new(capacity, *len))
}
DataType::Decimal(precision, scale) => {
DataType::Decimal128(precision, scale) => {
Box::new(Decimal128Builder::new(capacity, *precision, *scale))
}
DataType::Utf8 => Box::new(StringBuilder::new(capacity)),
Expand Down
16 changes: 8 additions & 8 deletions arrow/src/array/data.rs
View file
Open in desktop
Expand Up @@ -209,7 +209,7 @@ pub(crate) fn new_buffers(data_type: &DataType, capacity: usize) -> [MutableBuff
DataType::FixedSizeList(_, _) | DataType::Struct(_) => {
[empty_buffer, MutableBuffer::new(0)]
}
DataType::Decimal(_, _) | DataType::Decimal256(_, _) => [
DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => [
MutableBuffer::new(capacity * mem::size_of::<u8>()),
empty_buffer,
],
Expand Down Expand Up @@ -396,16 +396,16 @@ impl ArrayData {
/// panic's if the new DataType is not compatible with the
/// existing type.
///
/// Note: currently only changing a [DataType::Decimal]s precision
/// Note: currently only changing a [DataType::Decimal128]s precision
/// and scale are supported
#[inline]
pub(crate) fn with_data_type(mut self, new_data_type: DataType) -> Self {
assert!(
matches!(self.data_type, DataType::Decimal(_, _)),
matches!(self.data_type, DataType::Decimal128(_, _)),
"only DecimalType is supported for existing type"
);
assert!(
matches!(new_data_type, DataType::Decimal(_, _)),
matches!(new_data_type, DataType::Decimal128(_, _)),
"only DecimalType is supported for new datatype"
);
self.data_type = new_data_type;
Expand Down Expand Up @@ -598,7 +598,7 @@ impl ArrayData {
| DataType::LargeBinary
| DataType::Interval(_)
| DataType::FixedSizeBinary(_)
| DataType::Decimal(_, _)
| DataType::Decimal128(_, _)
| DataType::Decimal256(_, _) => vec![],
DataType::List(field) => {
vec![Self::new_empty(field.data_type())]
Expand Down Expand Up @@ -1031,7 +1031,7 @@ impl ArrayData {

pub fn validate_values(&self) -> Result<()> {
match &self.data_type {
DataType::Decimal(p, _) => {
DataType::Decimal128(p, _) => {
let values_buffer: &[i128] = self.typed_buffer(0, self.len)?;
for value in values_buffer {
validate_decimal_precision(*value, *p)?;
Expand Down Expand Up @@ -1361,7 +1361,7 @@ pub(crate) fn layout(data_type: &DataType) -> DataTypeLayout {
}
}
DataType::Dictionary(key_type, _value_type) => layout(key_type),
DataType::Decimal(_, _) => {
DataType::Decimal128(_, _) => {
// Decimals are always some fixed width; The rust implementation
// always uses 16 bytes / size of i128
DataTypeLayout::new_fixed_width(size_of::<i128>())
Expand Down Expand Up @@ -2834,7 +2834,7 @@ mod tests {
let fixed_size_array = fixed_size_builder.finish();

// Build ArrayData for Decimal
let builder = ArrayData::builder(DataType::Decimal(5, 3))
let builder = ArrayData::builder(DataType::Decimal128(5, 3))
.len(fixed_size_array.len())
.add_buffer(fixed_size_array.data_ref().child_data()[0].buffers()[0].clone());
let array_data = unsafe { builder.build_unchecked() };
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/array/equal/decimal.rs
View file
Open in desktop
Expand Up @@ -29,7 +29,7 @@ pub(super) fn decimal_equal(
len: usize,
) -> bool {
let size = match lhs.data_type() {
DataType::Decimal(_, _) => 16,
DataType::Decimal128(_, _) => 16,
DataType::Decimal256(_, _) => 32,
_ => unreachable!(),
};
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/array/equal/mod.rs
View file
Open in desktop
Expand Up @@ -187,7 +187,7 @@ fn equal_values(
DataType::FixedSizeBinary(_) => {
fixed_binary_equal(lhs, rhs, lhs_start, rhs_start, len)
}
DataType::Decimal(_, _) | DataType::Decimal256(_, _) => {
DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => {
decimal_equal(lhs, rhs, lhs_start, rhs_start, len)
}
DataType::List(_) => list_equal::<i32>(lhs, rhs, lhs_start, rhs_start, len),
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/array/ord.rs
View file
Open in desktop
Expand Up @@ -226,7 +226,7 @@ pub fn build_compare(left: &dyn Array, right: &dyn Array) -> Result<DynComparato
}
}
}
(Decimal(_, _), Decimal(_, _)) => {
(Decimal128(_, _), Decimal128(_, _)) => {
let left: Decimal128Array = Decimal128Array::from(left.data().clone());
let right: Decimal128Array = Decimal128Array::from(right.data().clone());
Box::new(move |i, j| left.value(i).cmp(&right.value(j)))
Expand Down
6 changes: 3 additions & 3 deletions arrow/src/array/transform/mod.rs
View file
Open in desktop
Expand Up @@ -205,7 +205,7 @@ fn build_extend_dictionary(
fn build_extend(array: &ArrayData) -> Extend {
use crate::datatypes::*;
match array.data_type() {
DataType::Decimal(_, _) => primitive::build_extend::<i128>(array),
DataType::Decimal128(_, _) => primitive::build_extend::<i128>(array),
DataType::Null => null::build_extend(array),
DataType::Boolean => boolean::build_extend(array),
DataType::UInt8 => primitive::build_extend::<u8>(array),
Expand Down Expand Up @@ -256,7 +256,7 @@ fn build_extend(array: &ArrayData) -> Extend {
fn build_extend_nulls(data_type: &DataType) -> ExtendNulls {
use crate::datatypes::*;
Box::new(match data_type {
DataType::Decimal(_, _) => primitive::extend_nulls::<i128>,
DataType::Decimal128(_, _) => primitive::extend_nulls::<i128>,
DataType::Null => null::extend_nulls,
DataType::Boolean => boolean::extend_nulls,
DataType::UInt8 => primitive::extend_nulls::<u8>,
Expand Down Expand Up @@ -410,7 +410,7 @@ impl<'a> MutableArrayData<'a> {
};

let child_data = match &data_type {
DataType::Decimal(_, _)
DataType::Decimal128(_, _)
| DataType::Decimal256(_, _)
| DataType::Null
| DataType::Boolean
Expand Down
34 changes: 18 additions & 16 deletions arrow/src/compute/kernels/cast.rs
View file
Open in desktop
Expand Up @@ -72,11 +72,11 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool {
match (from_type, to_type) {
// TODO UTF8/unsigned numeric to decimal
// cast one decimal type to another decimal type
(Decimal(_, _), Decimal(_, _)) => true,
(Decimal128(_, _), Decimal128(_, _)) => true,
// signed numeric to decimal
(Null | Int8 | Int16 | Int32 | Int64 | Float32 | Float64, Decimal(_, _)) |
(Null | Int8 | Int16 | Int32 | Int64 | Float32 | Float64, Decimal128(_, _)) |
// decimal to signed numeric
(Decimal(_, _), Null | Int8 | Int16 | Int32 | Int64 | Float32 | Float64)
(Decimal128(_, _), Null | Int8 | Int16 | Int32 | Int64 | Float32 | Float64)
| (
Null,
Boolean
Expand Down Expand Up @@ -109,8 +109,8 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool {
| Map(_, _)
| Dictionary(_, _)
) => true,
(Decimal(_, _), _) => false,
(_, Decimal(_, _)) => false,
(Decimal128(_, _), _) => false,
(_, Decimal128(_, _)) => false,
(Struct(_), _) => false,
(_, Struct(_)) => false,
(LargeList(list_from), LargeList(list_to)) => {
Expand Down Expand Up @@ -410,8 +410,10 @@ pub fn cast_with_options(
return Ok(array.clone());
}
match (from_type, to_type) {
(Decimal(_, s1), Decimal(p2, s2)) => cast_decimal_to_decimal(array, s1, p2, s2),
(Decimal(_, scale), _) => {
(Decimal128(_, s1), Decimal128(p2, s2)) => {
cast_decimal_to_decimal(array, s1, p2, s2)
}
(Decimal128(_, scale), _) => {
// cast decimal to other type
match to_type {
Int8 => {
Expand Down Expand Up @@ -439,7 +441,7 @@ pub fn cast_with_options(
))),
}
}
(_, Decimal(precision, scale)) => {
(_, Decimal128(precision, scale)) => {
// cast data to decimal
match from_type {
// TODO now just support signed numeric to decimal, support decimal to numeric later
Expand Down Expand Up @@ -2205,8 +2207,8 @@ mod tests {

#[test]
fn test_cast_decimal_to_decimal() {
let input_type = DataType::Decimal(20, 3);
let output_type = DataType::Decimal(20, 4);
let input_type = DataType::Decimal128(20, 3);
let output_type = DataType::Decimal128(20, 4);
assert!(can_cast_types(&input_type, &output_type));
let array = vec![Some(1123456), Some(2123456), Some(3123456), None];
let input_decimal_array = create_decimal_array(&array, 20, 3).unwrap();
Expand All @@ -2226,15 +2228,15 @@ mod tests {
let array = vec![Some(123456), None];
let input_decimal_array = create_decimal_array(&array, 10, 0).unwrap();
let array = Arc::new(input_decimal_array) as ArrayRef;
let result = cast(&array, &DataType::Decimal(2, 2));
let result = cast(&array, &DataType::Decimal128(2, 2));
assert!(result.is_err());
assert_eq!("Invalid argument error: 12345600 is too large to store in a Decimal of precision 2. Max is 99",
result.unwrap_err().to_string());
}

#[test]
fn test_cast_decimal_to_numeric() {
let decimal_type = DataType::Decimal(38, 2);
let decimal_type = DataType::Decimal128(38, 2);
// negative test
assert!(!can_cast_types(&decimal_type, &DataType::UInt8));
let value_array: Vec<Option<i128>> =
Expand Down Expand Up @@ -2355,7 +2357,7 @@ mod tests {
#[test]
fn test_cast_numeric_to_decimal() {
// test negative cast type
let decimal_type = DataType::Decimal(38, 6);
let decimal_type = DataType::Decimal128(38, 6);
assert!(!can_cast_types(&DataType::UInt64, &decimal_type));

// i8, i16, i32, i64
Expand Down Expand Up @@ -2408,7 +2410,7 @@ mod tests {
// the 100 will be converted to 1000_i128, but it is out of range for max value in the precision 3.
let array = Int8Array::from(vec![1, 2, 3, 4, 100]);
let array = Arc::new(array) as ArrayRef;
let casted_array = cast(&array, &DataType::Decimal(3, 1));
let casted_array = cast(&array, &DataType::Decimal128(3, 1));
assert!(casted_array.is_err());
assert_eq!("Invalid argument error: 1000 is too large to store in a Decimal of precision 3. Max is 999", casted_array.unwrap_err().to_string());

Expand Down Expand Up @@ -4282,7 +4284,7 @@ mod tests {

#[test]
fn test_cast_null_array_to_from_decimal_array() {
let data_type = DataType::Decimal(12, 4);
let data_type = DataType::Decimal128(12, 4);
let array = new_null_array(&DataType::Null, 4);
assert_eq!(array.data_type(), &DataType::Null);
let cast_array = cast(&array, &data_type).expect("cast failed");
Expand Down Expand Up @@ -4804,7 +4806,7 @@ mod tests {
Dictionary(Box::new(DataType::Int8), Box::new(DataType::Int32)),
Dictionary(Box::new(DataType::Int16), Box::new(DataType::Utf8)),
Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
Decimal(38, 0),
Decimal128(38, 0),
]
}

Expand Down
2 changes: 1 addition & 1 deletion arrow/src/compute/kernels/sort.rs
View file
Open in desktop
Expand Up @@ -145,7 +145,7 @@ pub fn sort_to_indices(
let (v, n) = partition_validity(values);

Ok(match values.data_type() {
DataType::Decimal(_, _) => sort_decimal(values, v, n, cmp, &options, limit),
DataType::Decimal128(_, _) => sort_decimal(values, v, n, cmp, &options, limit),
DataType::Boolean => sort_boolean(values, v, n, &options, limit),
DataType::Int8 => {
sort_primitive::<Int8Type, _>(values, v, n, cmp, &options, limit)
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/compute/kernels/take.rs
View file
Open in desktop
Expand Up @@ -148,7 +148,7 @@ where
let values = values.as_any().downcast_ref::<BooleanArray>().unwrap();
Ok(Arc::new(take_boolean(values, indices)?))
}
DataType::Decimal(_, _) => {
DataType::Decimal128(_, _) => {
let decimal_values =
values.as_any().downcast_ref::<Decimal128Array>().unwrap();
Ok(Arc::new(take_decimal128(decimal_values, indices)?))
Expand Down