Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor Binary Builder and String Builder Constructors #2592

Merged
merged 4 commits into from
Aug 27, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion arrow/benches/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ fn bench_string(c: &mut Criterion) {
));
group.bench_function("bench_string", |b| {
b.iter(|| {
let mut builder = StringBuilder::new(64);
let mut builder = StringBuilder::new();
for _ in 0..NUM_BATCHES * BATCH_SIZE {
builder.append_value(SAMPLE_STRING);
}
Expand Down
4 changes: 2 additions & 2 deletions arrow/benches/cast_kernels.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ fn build_utf8_date_array(size: usize, with_nulls: bool) -> ArrayRef {

// use random numbers to avoid spurious compiler optimizations wrt to branching
let mut rng = seedable_rng();
let mut builder = StringBuilder::new(size);
let mut builder = StringBuilder::new();
let range = Uniform::new(0, 737776);

for _ in 0..size {
Expand All @@ -66,7 +66,7 @@ fn build_utf8_date_time_array(size: usize, with_nulls: bool) -> ArrayRef {

// use random numbers to avoid spurious compiler optimizations wrt to branching
let mut rng = seedable_rng();
let mut builder = StringBuilder::new(size);
let mut builder = StringBuilder::new();
let range = Uniform::new(0, 1608071414123);

for _ in 0..size {
Expand Down
5 changes: 4 additions & 1 deletion arrow/benches/string_dictionary_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,10 @@ fn criterion_benchmark(c: &mut Criterion) {
let strings = build_strings(dict_size, total_size, key_len);
b.iter(|| {
let keys = Int32Builder::with_capacity(strings.len());
let values = StringBuilder::new((key_len + 1) * dict_size);
let values = StringBuilder::with_capacity(
key_len + 1,
(key_len + 1) * dict_size,
);
let mut builder = StringDictionaryBuilder::new(keys, values);

for val in &strings {
Expand Down
4 changes: 2 additions & 2 deletions arrow/src/array/array_dictionary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ impl<'a, T: ArrowDictionaryKeyType> FromIterator<Option<&'a str>> for Dictionary
let it = iter.into_iter();
let (lower, _) = it.size_hint();
let key_builder = PrimitiveBuilder::<T>::with_capacity(lower);
let value_builder = StringBuilder::new(256);
let value_builder = StringBuilder::with_capacity(256, 1024);
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
it.for_each(|i| {
if let Some(i) = i {
Expand Down Expand Up @@ -368,7 +368,7 @@ impl<'a, T: ArrowDictionaryKeyType> FromIterator<&'a str> for DictionaryArray<T>
let it = iter.into_iter();
let (lower, _) = it.size_hint();
let key_builder = PrimitiveBuilder::<T>::with_capacity(lower);
let value_builder = StringBuilder::new(256);
let value_builder = StringBuilder::with_capacity(256, 1024);
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
it.for_each(|i| {
builder
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/array/array_string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ mod tests {

#[test]
fn test_nested_string_array() {
let string_builder = StringBuilder::new(3);
let string_builder = StringBuilder::with_capacity(3, 10);
let mut list_of_string_builder = ListBuilder::new(string_builder);

list_of_string_builder.values().append_value("foo");
Expand Down
23 changes: 11 additions & 12 deletions arrow/src/array/builder/generic_binary_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,8 @@ pub struct GenericBinaryBuilder<OffsetSize: OffsetSizeTrait> {

impl<OffsetSize: OffsetSizeTrait> GenericBinaryBuilder<OffsetSize> {
/// Creates a new [`GenericBinaryBuilder`].
/// `capacity` is the number of bytes in the values array.
pub fn new(capacity: usize) -> Self {
let mut offsets_builder = BufferBuilder::<OffsetSize>::new(1024);
offsets_builder.append(OffsetSize::zero());
Self {
value_builder: UInt8BufferBuilder::new(capacity),
offsets_builder,
null_buffer_builder: NullBufferBuilder::new(1024),
}
pub fn new() -> Self {
Self::with_capacity(1024, 1024)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it a reasonable assumption that the array contains the same number of elements and bytes?

Copy link
Contributor Author

@psvri psvri Aug 27, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When we used new previously , it was using 1024 as no. of items, hence I have retained the same.

Copy link
Contributor

@tustvold tustvold Aug 27, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not a reasonable assumption no, but it is a decent guess to avoid worst case bump allocation, probably 😅

}

/// Creates a new [`GenericBinaryBuilder`],
Expand Down Expand Up @@ -100,6 +93,12 @@ impl<OffsetSize: OffsetSizeTrait> GenericBinaryBuilder<OffsetSize> {
}
}

impl<OffsetSize: OffsetSizeTrait> Default for GenericBinaryBuilder<OffsetSize> {
fn default() -> Self {
Self::new()
}
}

impl<OffsetSize: OffsetSizeTrait> ArrayBuilder for GenericBinaryBuilder<OffsetSize> {
/// Returns the builder as a non-mutable `Any` reference.
fn as_any(&self) -> &dyn Any {
Expand Down Expand Up @@ -138,7 +137,7 @@ mod tests {
use crate::array::{Array, OffsetSizeTrait};

fn _test_generic_binary_builder<O: OffsetSizeTrait>() {
let mut builder = GenericBinaryBuilder::<O>::new(20);
let mut builder = GenericBinaryBuilder::<O>::new();

builder.append_value(b"hello");
builder.append_value(b"");
Expand Down Expand Up @@ -168,7 +167,7 @@ mod tests {
}

fn _test_generic_binary_builder_all_nulls<O: OffsetSizeTrait>() {
let mut builder = GenericBinaryBuilder::<O>::new(10);
let mut builder = GenericBinaryBuilder::<O>::new();
builder.append_null();
builder.append_null();
builder.append_null();
Expand All @@ -194,7 +193,7 @@ mod tests {
}

fn _test_generic_binary_builder_reset<O: OffsetSizeTrait>() {
let mut builder = GenericBinaryBuilder::<O>::new(20);
let mut builder = GenericBinaryBuilder::<O>::new();

builder.append_value(b"hello");
builder.append_value(b"");
Expand Down
13 changes: 9 additions & 4 deletions arrow/src/array/builder/generic_string_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@ pub struct GenericStringBuilder<OffsetSize: OffsetSizeTrait> {

impl<OffsetSize: OffsetSizeTrait> GenericStringBuilder<OffsetSize> {
/// Creates a new [`GenericStringBuilder`],
/// `capacity` is the number of bytes of string data to pre-allocate space for in this builder
pub fn new(capacity: usize) -> Self {
pub fn new() -> Self {
Self {
builder: GenericBinaryBuilder::new(capacity),
builder: GenericBinaryBuilder::new(),
}
}

Expand Down Expand Up @@ -82,6 +81,12 @@ impl<OffsetSize: OffsetSizeTrait> GenericStringBuilder<OffsetSize> {
}
}

impl<OffsetSize: OffsetSizeTrait> Default for GenericStringBuilder<OffsetSize> {
fn default() -> Self {
Self::new()
}
}

impl<OffsetSize: OffsetSizeTrait> ArrayBuilder for GenericStringBuilder<OffsetSize> {
/// Returns the builder as a non-mutable `Any` reference.
fn as_any(&self) -> &dyn Any {
Expand Down Expand Up @@ -121,7 +126,7 @@ mod tests {
use crate::array::{Array, ArrayBuilder, OffsetSizeTrait};

fn _test_generic_string_array_builder<O: OffsetSizeTrait>() {
let mut builder = GenericStringBuilder::<O>::new(20);
let mut builder = GenericStringBuilder::<O>::new();
let owned = "arrow".to_owned();

builder.append_value("hello");
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/array/builder/map_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ mod tests {

#[test]
fn test_map_array_builder() {
let string_builder = StringBuilder::new(4);
let string_builder = StringBuilder::new();
let int_builder = Int32Builder::with_capacity(4);

let mut builder = MapBuilder::new(None, string_builder, int_builder);
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/array/builder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ pub use union_builder::UnionBuilder;
/// let mut data_builders: Vec<Box<dyn ArrayBuilder>> = vec![
/// Box::new(Float64Builder::new()),
/// Box::new(Int64Builder::new()),
/// Box::new(StringBuilder::new(1024)),
/// Box::new(StringBuilder::new()),
/// ];
///
/// // Fill
Expand Down
4 changes: 2 additions & 2 deletions arrow/src/array/builder/string_dictionary_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ use std::sync::Arc;
/// // It can thus hold up to 256 distinct string values.
///
/// let key_builder = PrimitiveBuilder::<Int8Type>::with_capacity(100);
/// let value_builder = StringBuilder::new(100);
/// let value_builder = StringBuilder::new();
/// let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
///
/// // The builder builds the dictionary value by value
Expand Down Expand Up @@ -291,7 +291,7 @@ mod tests {
#[test]
fn test_string_dictionary_builder() {
let key_builder = PrimitiveBuilder::<Int8Type>::with_capacity(5);
let value_builder = StringBuilder::new(2);
let value_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
builder.append("abc").unwrap();
builder.append_null();
Expand Down
6 changes: 3 additions & 3 deletions arrow/src/array/builder/struct_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,14 @@ pub fn make_builder(datatype: &DataType, capacity: usize) -> Box<dyn ArrayBuilde
DataType::UInt64 => Box::new(UInt64Builder::with_capacity(capacity)),
DataType::Float32 => Box::new(Float32Builder::with_capacity(capacity)),
DataType::Float64 => Box::new(Float64Builder::with_capacity(capacity)),
DataType::Binary => Box::new(BinaryBuilder::new(capacity)),
DataType::Binary => Box::new(BinaryBuilder::with_capacity(1024, capacity)),
DataType::FixedSizeBinary(len) => {
Box::new(FixedSizeBinaryBuilder::with_capacity(capacity, *len))
}
DataType::Decimal128(precision, scale) => Box::new(
Decimal128Builder::with_capacity(capacity, *precision, *scale),
),
DataType::Utf8 => Box::new(StringBuilder::new(capacity)),
DataType::Utf8 => Box::new(StringBuilder::with_capacity(1024, capacity)),
DataType::Date32 => Box::new(Date32Builder::with_capacity(capacity)),
DataType::Date64 => Box::new(Date64Builder::with_capacity(capacity)),
DataType::Time32(TimeUnit::Second) => {
Expand Down Expand Up @@ -255,7 +255,7 @@ mod tests {

#[test]
fn test_struct_array_builder() {
let string_builder = StringBuilder::new(4);
let string_builder = StringBuilder::new();
let int_builder = Int32Builder::new();

let mut fields = Vec::new();
Expand Down
4 changes: 2 additions & 2 deletions arrow/src/array/transform/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1507,15 +1507,15 @@ mod tests {
#[test]
fn test_list_of_strings_append() -> Result<()> {
// [["alpha", "beta", None]]
let mut builder = ListBuilder::new(StringBuilder::new(32));
let mut builder = ListBuilder::new(StringBuilder::new());
builder.values().append_value("Hello");
builder.values().append_value("Arrow");
builder.values().append_null();
builder.append(true);
let a = builder.finish();

// [["alpha", "beta"], [None], ["gamma", "delta", None]]
let mut builder = ListBuilder::new(StringBuilder::new(32));
let mut builder = ListBuilder::new(StringBuilder::new());
builder.values().append_value("alpha");
builder.values().append_value("beta");
builder.append(true);
Expand Down
8 changes: 4 additions & 4 deletions arrow/src/compute/kernels/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2355,7 +2355,7 @@ where
let values = cast_values.as_any().downcast_ref::<StringArray>().unwrap();

let keys_builder = PrimitiveBuilder::<K>::with_capacity(values.len());
let values_builder = StringBuilder::new(values.len());
let values_builder = StringBuilder::with_capacity(1024, values.len());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In this case I'd be tempted to just go with new, this estimate isn't obviously better

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I should have retained new.

let mut b = StringDictionaryBuilder::new(keys_builder, values_builder);

// copy each element one at a time
Expand Down Expand Up @@ -4753,7 +4753,7 @@ mod tests {
use DataType::*;

let keys_builder = PrimitiveBuilder::<Int8Type>::new();
let values_builder = StringBuilder::new(10);
let values_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);
builder.append("one").unwrap();
builder.append_null();
Expand Down Expand Up @@ -4845,7 +4845,7 @@ mod tests {
// string values (and encode the expected behavior here);

let keys_builder = PrimitiveBuilder::<Int32Type>::new();
let values_builder = StringBuilder::new(10);
let values_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(keys_builder, values_builder);

// add 200 distinct values (which can be stored by a
Expand Down Expand Up @@ -5388,7 +5388,7 @@ mod tests {
fn make_dictionary_utf8<K: ArrowDictionaryKeyType>() -> ArrayRef {
let keys_builder = PrimitiveBuilder::<K>::new();
// Pick Int32 arbitrarily for dictionary values
let values_builder = StringBuilder::new(2);
let values_builder = StringBuilder::new();
let mut b = StringDictionaryBuilder::new(keys_builder, values_builder);
b.append("foo").unwrap();
b.append("bar").unwrap();
Expand Down
14 changes: 7 additions & 7 deletions arrow/src/compute/kernels/comparison.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3739,7 +3739,7 @@ mod tests {
// contains(null, null) = false
#[test]
fn test_contains_utf8() {
let values_builder = StringBuilder::new(10);
let values_builder = StringBuilder::new();
let mut builder = ListBuilder::new(values_builder);

builder.values().append_value("Lorem");
Expand Down Expand Up @@ -4727,7 +4727,7 @@ mod tests {
#[test]
fn test_eq_dyn_utf8_scalar_with_dict() {
let key_builder = PrimitiveBuilder::<Int8Type>::new();
let value_builder = StringBuilder::new(100);
let value_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
builder.append("abc").unwrap();
builder.append_null();
Expand Down Expand Up @@ -4755,7 +4755,7 @@ mod tests {
#[test]
fn test_lt_dyn_utf8_scalar_with_dict() {
let key_builder = PrimitiveBuilder::<Int8Type>::new();
let value_builder = StringBuilder::new(100);
let value_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
builder.append("abc").unwrap();
builder.append_null();
Expand Down Expand Up @@ -4784,7 +4784,7 @@ mod tests {
#[test]
fn test_lt_eq_dyn_utf8_scalar_with_dict() {
let key_builder = PrimitiveBuilder::<Int8Type>::new();
let value_builder = StringBuilder::new(100);
let value_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
builder.append("abc").unwrap();
builder.append_null();
Expand Down Expand Up @@ -4813,7 +4813,7 @@ mod tests {
#[test]
fn test_gt_eq_dyn_utf8_scalar_with_dict() {
let key_builder = PrimitiveBuilder::<Int8Type>::new();
let value_builder = StringBuilder::new(100);
let value_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
builder.append("abc").unwrap();
builder.append_null();
Expand Down Expand Up @@ -4843,7 +4843,7 @@ mod tests {
#[test]
fn test_gt_dyn_utf8_scalar_with_dict() {
let key_builder = PrimitiveBuilder::<Int8Type>::new();
let value_builder = StringBuilder::new(100);
let value_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
builder.append("abc").unwrap();
builder.append_null();
Expand Down Expand Up @@ -4872,7 +4872,7 @@ mod tests {
#[test]
fn test_neq_dyn_utf8_scalar_with_dict() {
let key_builder = PrimitiveBuilder::<Int8Type>::new();
let value_builder = StringBuilder::new(100);
let value_builder = StringBuilder::new();
let mut builder = StringDictionaryBuilder::new(key_builder, value_builder);
builder.append("abc").unwrap();
builder.append_null();
Expand Down
4 changes: 2 additions & 2 deletions arrow/src/compute/kernels/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1416,7 +1416,7 @@ mod tests {
#[test]
fn test_filter_map() {
let mut builder =
MapBuilder::new(None, StringBuilder::new(16), Int64Builder::with_capacity(4));
MapBuilder::new(None, StringBuilder::new(), Int64Builder::with_capacity(4));
// [{"key1": 1}, {"key2": 2, "key3": 3}, null, {"key1": 1}
builder.keys().append_value("key1");
builder.values().append_value(1);
Expand All @@ -1438,7 +1438,7 @@ mod tests {
let got = filter(&maparray, &indices).unwrap();

let mut builder =
MapBuilder::new(None, StringBuilder::new(8), Int64Builder::with_capacity(2));
MapBuilder::new(None, StringBuilder::new(), Int64Builder::with_capacity(2));
builder.keys().append_value("key1");
builder.values().append_value(1);
builder.append(true).unwrap();
Expand Down
8 changes: 5 additions & 3 deletions arrow/src/compute/kernels/regexp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ pub fn regexp_match<OffsetSize: OffsetSizeTrait>(
flags_array: Option<&GenericStringArray<OffsetSize>>,
) -> Result<ArrayRef> {
let mut patterns: HashMap<String, Regex> = HashMap::new();
let builder: GenericStringBuilder<OffsetSize> = GenericStringBuilder::new(0);
let builder: GenericStringBuilder<OffsetSize> =
GenericStringBuilder::with_capacity(0, 0);
let mut list_builder = ListBuilder::new(builder);

let complete_pattern = match flags_array {
Expand Down Expand Up @@ -118,7 +119,7 @@ mod tests {
pattern_values.push("");
let pattern = StringArray::from(pattern_values);
let actual = regexp_match(&array, &pattern, None).unwrap();
let elem_builder: GenericStringBuilder<i32> = GenericStringBuilder::new(0);
let elem_builder: GenericStringBuilder<i32> = GenericStringBuilder::new();
let mut expected_builder = ListBuilder::new(elem_builder);
expected_builder.values().append_value("005");
expected_builder.append(true);
Expand All @@ -141,7 +142,8 @@ mod tests {
let pattern = StringArray::from(vec![r"x.*-(\d*)-.*"; 4]);
let flags = StringArray::from(vec!["i"; 4]);
let actual = regexp_match(&array, &pattern, Some(&flags)).unwrap();
let elem_builder: GenericStringBuilder<i32> = GenericStringBuilder::new(0);
let elem_builder: GenericStringBuilder<i32> =
GenericStringBuilder::with_capacity(0, 0);
let mut expected_builder = ListBuilder::new(elem_builder);
expected_builder.append(false);
expected_builder.values().append_value("7");
Expand Down
2 changes: 1 addition & 1 deletion arrow/src/compute/kernels/take.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2040,7 +2040,7 @@ mod tests {
#[test]
fn test_take_dict() {
let keys_builder = Int16Builder::new();
let values_builder = StringBuilder::new(4);
let values_builder = StringBuilder::new();

let mut dict_builder = StringDictionaryBuilder::new(keys_builder, values_builder);

Expand Down