Skip to content

Commit

Permalink
use page row limit fix ut
Browse files Browse the repository at this point in the history
Signed-off-by: yangjiang <yangjiang@ebay.com>
  • Loading branch information
Ted-Jiang committed Nov 5, 2022
1 parent 1c97dc4 commit 005b101
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 6 deletions.
4 changes: 1 addition & 3 deletions datafusion/core/src/datasource/file_format/parquet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -538,9 +538,7 @@ pub(crate) mod test_util {
// All batches write in to one file, each batch must have same schema.
let mut output = NamedTempFile::new().expect("creating temp file");
let mut builder = WriterProperties::builder();
// todo https://github.com/apache/arrow-rs/issues/2941 release change to row limit.
builder = builder.set_data_pagesize_limit(1);
builder = builder.set_write_batch_size(1);
builder = builder.set_data_page_row_count_limit(2);
let proper = builder.build();
let mut writer =
ArrowWriter::try_new(&mut output, batches[0].schema(), Some(proper))
Expand Down
5 changes: 2 additions & 3 deletions datafusion/core/src/physical_plan/file_format/parquet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1726,11 +1726,10 @@ mod tests {
// todo fix this https://github.com/apache/arrow-rs/issues/2941 release change to row limit.
// assert the batches and some metrics
let expected = vec![
"+-----+", "| int |", "+-----+", "| |", "| 1 |", "| 2 |", "| 3 |",
"| 4 |", "| 5 |", "+-----+",
"+-----+", "| int |", "+-----+", "| 3 |", "| 4 |", "| 5 |", "+-----+",
];
assert_batches_sorted_eq!(expected, &rt.batches.unwrap());
assert_eq!(get_value(&metrics, "page_index_rows_filtered"), 0);
assert_eq!(get_value(&metrics, "page_index_rows_filtered"), 3);
assert!(
get_value(&metrics, "page_index_eval_time") > 0,
"no eval time in metrics: {:#?}",
Expand Down

0 comments on commit 005b101

Please sign in to comment.