Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Correct schema nullability declaration in tests #2802

Merged
merged 1 commit into from Jun 28, 2022
Merged

Correct schema nullability declaration in tests #2802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion datafusion/core/src/datasource/listing/helpers.rs
View file
Open in desktop
Expand Up @@ -272,7 +272,7 @@ fn paths_to_batch(
let mut fields = vec![
Field::new(FILE_PATH_COLUMN_NAME, DataType::Utf8, false),
Field::new(FILE_SIZE_COLUMN_NAME, DataType::UInt64, false),
Field::new(FILE_MODIFIED_COLUMN_NAME, DataType::Date64, false),
Field::new(FILE_MODIFIED_COLUMN_NAME, DataType::Date64, true),
];
for pn in table_partition_cols {
fields.push(Field::new(pn, DataType::Utf8, false));
Expand Down
4 changes: 2 additions & 2 deletions datafusion/core/src/physical_optimizer/aggregate_statistics.rs
View file
Open in desktop
Expand Up @@ -276,8 +276,8 @@ mod tests {
/// Mock data using a MemoryExec which has an exact count statistic
fn mock_data() -> Result<Arc<MemoryExec>> {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
Field::new("a", DataType::Int32, true),
Field::new("b", DataType::Int32, true),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a pretty easy to understand example of the issue -- prior to this PR, the fields "a" and "b" are declared as "nullable=false" but then 5 lines lower NULL data is inserted 🤦

        let batch = RecordBatch::try_new(
            Arc::clone(&schema),
            vec![
                Arc::new(Int32Array::from(vec![Some(1), Some(2), None])),
                Arc::new(Int32Array::from(vec![Some(4), None, Some(6)])),
            ],
        )?;

Now that RecordBatch::try_new validates the nullability, the schema must match the data otherwise an error results

]));

let batch = RecordBatch::try_new(
Expand Down
4 changes: 2 additions & 2 deletions datafusion/core/src/physical_plan/hash_join.rs
View file
Open in desktop
Expand Up @@ -2388,8 +2388,8 @@ mod tests {
},
];
let intermediate_schema = Schema::new(vec![
Field::new("c", DataType::Int32, false),
Field::new("c", DataType::Int32, false),
Field::new("c", DataType::Int32, true),
Field::new("c", DataType::Int32, true),
]);
let filter_expression = Arc::new(BinaryExpr::new(
Arc::new(Column::new("c", 0)),
Expand Down
4 changes: 2 additions & 2 deletions datafusion/core/src/physical_plan/memory.rs
View file
Open in desktop
Expand Up @@ -231,8 +231,8 @@ mod tests {
let schema = Arc::new(Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
Field::new("c", DataType::Int32, false),
Field::new("d", DataType::Int32, true),
Field::new("c", DataType::Int32, true),
Field::new("d", DataType::Int32, false),
]));

let batch = RecordBatch::try_new(
Expand Down
16 changes: 8 additions & 8 deletions datafusion/physical-expr/src/aggregate/correlation.rs
View file
Open in desktop
Expand Up @@ -336,8 +336,8 @@ mod tests {
let b: ArrayRef = Arc::new(Int32Array::from(vec![Some(4), Some(5), Some(6)]));

let schema = Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
Field::new("a", DataType::Int32, true),
Field::new("b", DataType::Int32, true),
]);
let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;

Expand All @@ -359,8 +359,8 @@ mod tests {
let b: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));

let schema = Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
Field::new("a", DataType::Int32, true),
Field::new("b", DataType::Int32, true),
]);
let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;

Expand All @@ -384,8 +384,8 @@ mod tests {
let d = Arc::new(Float64Array::from(vec![4.4_f64, 5.5_f64, 9.9_f64]));

let schema = Schema::new(vec![
Field::new("a", DataType::Float64, false),
Field::new("b", DataType::Float64, false),
Field::new("a", DataType::Float64, true),
Field::new("b", DataType::Float64, true),
]);

let batch1 = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;
Expand Down Expand Up @@ -419,8 +419,8 @@ mod tests {
let d = Arc::new(Float64Array::from(vec![None]));

let schema = Schema::new(vec![
Field::new("a", DataType::Float64, false),
Field::new("b", DataType::Float64, false),
Field::new("a", DataType::Float64, true),
Field::new("b", DataType::Float64, true),
]);

let batch1 = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;
Expand Down
16 changes: 8 additions & 8 deletions datafusion/physical-expr/src/aggregate/covariance.rs
View file
Open in desktop
Expand Up @@ -511,8 +511,8 @@ mod tests {
let b: ArrayRef = Arc::new(Int32Array::from(vec![Some(4), Some(5), Some(6)]));

let schema = Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
Field::new("a", DataType::Int32, true),
Field::new("b", DataType::Int32, true),
]);
let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;

Expand All @@ -534,8 +534,8 @@ mod tests {
let b: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));

let schema = Schema::new(vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Int32, false),
Field::new("a", DataType::Int32, true),
Field::new("b", DataType::Int32, true),
]);
let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;

Expand All @@ -559,8 +559,8 @@ mod tests {
let d = Arc::new(Float64Array::from(vec![4.4_f64, 5.5_f64, 6.6_f64]));

let schema = Schema::new(vec![
Field::new("a", DataType::Float64, false),
Field::new("b", DataType::Float64, false),
Field::new("a", DataType::Float64, true),
Field::new("b", DataType::Float64, true),
]);

let batch1 = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;
Expand Down Expand Up @@ -594,8 +594,8 @@ mod tests {
let d = Arc::new(Float64Array::from(vec![None]));

let schema = Schema::new(vec![
Field::new("a", DataType::Float64, false),
Field::new("b", DataType::Float64, false),
Field::new("a", DataType::Float64, true),
Field::new("b", DataType::Float64, true),
]);

let batch1 = RecordBatch::try_new(Arc::new(schema.clone()), vec![a, b])?;
Expand Down
4 changes: 2 additions & 2 deletions datafusion/physical-expr/src/aggregate/stddev.rs
View file
Open in desktop
Expand Up @@ -348,7 +348,7 @@ mod tests {
#[test]
fn stddev_i32_all_nulls() -> Result<()> {
let a: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a])?;

let agg = Arc::new(Stddev::new(
Expand Down Expand Up @@ -395,7 +395,7 @@ mod tests {
let a = Arc::new(Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]));
let b = Arc::new(Float64Array::from(vec![None]));

let schema = Schema::new(vec![Field::new("a", DataType::Float64, false)]);
let schema = Schema::new(vec![Field::new("a", DataType::Float64, true)]);

let batch1 = RecordBatch::try_new(Arc::new(schema.clone()), vec![a])?;
let batch2 = RecordBatch::try_new(Arc::new(schema.clone()), vec![b])?;
Expand Down
2 changes: 1 addition & 1 deletion datafusion/physical-expr/src/aggregate/sum_distinct.rs
View file
Open in desktop
Expand Up @@ -199,7 +199,7 @@ mod tests {

macro_rules! generic_test_sum_distinct {
($ARRAY:expr, $DATATYPE:expr, $EXPECTED:expr, $EXPECTED_DATATYPE:expr) => {{
let schema = Schema::new(vec![Field::new("a", $DATATYPE, false)]);
let schema = Schema::new(vec![Field::new("a", $DATATYPE, true)]);

let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![$ARRAY])?;

Expand Down
4 changes: 2 additions & 2 deletions datafusion/physical-expr/src/aggregate/variance.rs
View file
Open in desktop
Expand Up @@ -429,7 +429,7 @@ mod tests {
#[test]
fn variance_i32_all_nulls() -> Result<()> {
let a: ArrayRef = Arc::new(Int32Array::from(vec![None, None]));
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a])?;

let agg = Arc::new(Variance::new(
Expand Down Expand Up @@ -476,7 +476,7 @@ mod tests {
let a = Arc::new(Float64Array::from(vec![1_f64, 2_f64, 3_f64, 4_f64, 5_f64]));
let b = Arc::new(Float64Array::from(vec![None]));

let schema = Schema::new(vec![Field::new("a", DataType::Float64, false)]);
let schema = Schema::new(vec![Field::new("a", DataType::Float64, true)]);

let batch1 = RecordBatch::try_new(Arc::new(schema.clone()), vec![a])?;
let batch2 = RecordBatch::try_new(Arc::new(schema.clone()), vec![b])?;
Expand Down
6 changes: 3 additions & 3 deletions datafusion/physical-expr/src/expressions/binary.rs
View file
Open in desktop
Expand Up @@ -2113,8 +2113,8 @@ mod tests {
/// b: [true, NULL, false, true, NULL, false, true, NULL, false]
fn bool_test_arrays() -> (SchemaRef, ArrayRef, ArrayRef) {
let schema = Schema::new(vec![
Field::new("a", DataType::Boolean, false),
Field::new("b", DataType::Boolean, false),
Field::new("a", DataType::Boolean, true),
Field::new("b", DataType::Boolean, true),
]);
let a: BooleanArray = [
Some(true),
Expand Down Expand Up @@ -2147,7 +2147,7 @@ mod tests {

/// Returns (schema, BooleanArray) with [true, NULL, false]
fn scalar_bool_test_array() -> (SchemaRef, ArrayRef) {
let schema = Schema::new(vec![Field::new("a", DataType::Boolean, false)]);
let schema = Schema::new(vec![Field::new("a", DataType::Boolean, true)]);
let a: BooleanArray = vec![Some(true), None, Some(false)].iter().collect();
(Arc::new(schema), Arc::new(a))
}
Expand Down
4 changes: 2 additions & 2 deletions datafusion/physical-expr/src/expressions/cast.rs
View file
Open in desktop
Expand Up @@ -177,7 +177,7 @@ mod tests {
// 5. verify that the resulting values are downcastable and correct
macro_rules! generic_decimal_to_other_test_cast {
($DECIMAL_ARRAY:ident, $A_TYPE:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr,$CAST_OPTIONS:expr) => {{
let schema = Schema::new(vec![Field::new("a", $A_TYPE, false)]);
let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
let batch = RecordBatch::try_new(
Arc::new(schema.clone()),
vec![Arc::new($DECIMAL_ARRAY)],
Expand Down Expand Up @@ -225,7 +225,7 @@ mod tests {
// 5. verify that the resulting values are downcastable and correct
macro_rules! generic_test_cast {
($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr, $CAST_OPTIONS:expr) => {{
let schema = Schema::new(vec![Field::new("a", $A_TYPE, false)]);
let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
let a = $A_ARRAY::from($A_VEC);
let batch =
RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
Expand Down
2 changes: 1 addition & 1 deletion datafusion/physical-expr/src/expressions/literal.rs
View file
Open in desktop
Expand Up @@ -88,7 +88,7 @@ mod tests {
#[test]
fn literal_i32() -> Result<()> {
// create an arbitrary record bacth
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
let schema = Schema::new(vec![Field::new("a", DataType::Int32, true)]);
let a = Int32Array::from(vec![Some(1), None, Some(3), Some(4), Some(5)]);
let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(a)])?;

Expand Down
6 changes: 3 additions & 3 deletions datafusion/physical-expr/src/expressions/mod.rs
View file
Open in desktop
Expand Up @@ -100,7 +100,7 @@ pub(crate) mod tests {
#[macro_export]
macro_rules! generic_test_op {
($ARRAY:expr, $DATATYPE:expr, $OP:ident, $EXPECTED:expr, $EXPECTED_DATATYPE:expr) => {{
let schema = Schema::new(vec![Field::new("a", $DATATYPE, false)]);
let schema = Schema::new(vec![Field::new("a", $DATATYPE, true)]);

let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![$ARRAY])?;

Expand All @@ -123,8 +123,8 @@ pub(crate) mod tests {
macro_rules! generic_test_op2 {
($ARRAY1:expr, $ARRAY2:expr, $DATATYPE1:expr, $DATATYPE2:expr, $OP:ident, $EXPECTED:expr, $EXPECTED_DATATYPE:expr) => {{
let schema = Schema::new(vec![
Field::new("a", $DATATYPE1, false),
Field::new("b", $DATATYPE2, false),
Field::new("a", $DATATYPE1, true),
Field::new("b", $DATATYPE2, true),
]);
let batch =
RecordBatch::try_new(Arc::new(schema.clone()), vec![$ARRAY1, $ARRAY2])?;
Expand Down
4 changes: 2 additions & 2 deletions datafusion/physical-expr/src/expressions/try_cast.rs
View file
Open in desktop
Expand Up @@ -138,7 +138,7 @@ mod tests {
// 5. verify that the resulting values are downcastable and correct
macro_rules! generic_decimal_to_other_test_cast {
($DECIMAL_ARRAY:ident, $A_TYPE:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
let schema = Schema::new(vec![Field::new("a", $A_TYPE, false)]);
let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
let batch = RecordBatch::try_new(
Arc::new(schema.clone()),
vec![Arc::new($DECIMAL_ARRAY)],
Expand Down Expand Up @@ -185,7 +185,7 @@ mod tests {
// 5. verify that the resulting values are downcastable and correct
macro_rules! generic_test_cast {
($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
let schema = Schema::new(vec![Field::new("a", $A_TYPE, false)]);
let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
let a = $A_ARRAY::from($A_VEC);
let batch =
RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
Expand Down
2 changes: 1 addition & 1 deletion datafusion/physical-expr/src/window/row_number.rs
View file
Open in desktop
Expand Up @@ -93,7 +93,7 @@ mod tests {
let arr: ArrayRef = Arc::new(BooleanArray::from(vec![
None, None, None, None, None, None, None, None,
]));
let schema = Schema::new(vec![Field::new("arr", DataType::Boolean, false)]);
let schema = Schema::new(vec![Field::new("arr", DataType::Boolean, true)]);
let batch = RecordBatch::try_new(Arc::new(schema), vec![arr])?;
let row_number = RowNumber::new("row_number".to_owned());
let result = row_number.create_evaluator(&batch)?.evaluate(vec![0..8])?;
Expand Down