Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update ahash requirement from 0.7 to 0.8 #3161

Merged
merged 4 commits into from
Aug 16, 2022
Merged

Update ahash requirement from 0.7 to 0.8 #3161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d1a295c
Update ahash requirement from 0.7 to 0.8
dependabot[bot] Aug 10, 2022
621b0c5
Merge remote-tracking branch 'apache/master' into alamb/update_ahash
alamb Aug 15, 2022
ac18c75
Update to new API
alamb Aug 15, 2022
7b3f4f4
Fix cargo.toml formatting
alamb Aug 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion datafusion/core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ simd = ["arrow/simd"]
unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion-sql/unicode_expressions"]

[dependencies]
ahash = { version = "0.7", default-features = false }
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
apache-avro = { version = "0.14", optional = true }
arrow = { version = "20.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
Expand Down
60 changes: 24 additions & 36 deletions datafusion/core/src/physical_plan/hash_utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
//! Functionality used both on logical and physical plans

use crate::error::{DataFusionError, Result};
use ahash::{CallHasher, RandomState};
use ahash::RandomState;
use arrow::array::{
Array, ArrayRef, BasicDecimalArray, BooleanArray, Date32Array, Date64Array,
Decimal128Array, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array,
Expand All @@ -43,11 +43,11 @@ fn hash_null(random_state: &RandomState, hashes_buffer: &'_ mut [u64], mul_col:
if mul_col {
hashes_buffer.iter_mut().for_each(|hash| {
// stable hash for null value
*hash = combine_hashes(i128::get_hash(&1, random_state), *hash);
*hash = combine_hashes(random_state.hash_one(&1), *hash);
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These changes are required due to tkaitchuck/aHash#121 which appears to have made CallHasher crate private

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Largely the same change @Dandandan applied in apache/arrow-rs#2410

})
} else {
hashes_buffer.iter_mut().for_each(|hash| {
*hash = i128::get_hash(&1, random_state);
*hash = random_state.hash_one(&1);
})
}
}
Expand All @@ -63,28 +63,28 @@ fn hash_decimal128<'a>(
if mul_col {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
*hash = combine_hashes(
i128::get_hash(&array.value(i).as_i128(), random_state),
random_state.hash_one(&array.value(i).as_i128()),
*hash,
);
}
} else {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
*hash = i128::get_hash(&array.value(i).as_i128(), random_state);
*hash = random_state.hash_one(&array.value(i).as_i128());
}
}
} else if mul_col {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
*hash = combine_hashes(
i128::get_hash(&array.value(i).as_i128(), random_state),
random_state.hash_one(&array.value(i).as_i128()),
*hash,
);
}
}
} else {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
if !array.is_null(i) {
*hash = i128::get_hash(&array.value(i).as_i128(), random_state);
*hash = random_state.hash_one(&array.value(i).as_i128());
}
}
}
Expand All @@ -96,30 +96,28 @@ macro_rules! hash_array {
if array.null_count() == 0 {
if $multi_col {
for (i, hash) in $hashes.iter_mut().enumerate() {
*hash = combine_hashes(
<$ty>::get_hash(&array.value(i), $random_state),
*hash,
);
*hash =
combine_hashes($random_state.hash_one(&array.value(i)), *hash);
}
} else {
for (i, hash) in $hashes.iter_mut().enumerate() {
*hash = <$ty>::get_hash(&array.value(i), $random_state);
*hash = $random_state.hash_one(&array.value(i));
}
}
} else {
if $multi_col {
for (i, hash) in $hashes.iter_mut().enumerate() {
if !array.is_null(i) {
*hash = combine_hashes(
<$ty>::get_hash(&array.value(i), $random_state),
$random_state.hash_one(&array.value(i)),
*hash,
);
}
}
} else {
for (i, hash) in $hashes.iter_mut().enumerate() {
if !array.is_null(i) {
*hash = <$ty>::get_hash(&array.value(i), $random_state);
*hash = $random_state.hash_one(&array.value(i));
}
}
}
Expand All @@ -135,11 +133,11 @@ macro_rules! hash_array_primitive {
if array.null_count() == 0 {
if $multi_col {
for (hash, value) in $hashes.iter_mut().zip(values.iter()) {
*hash = combine_hashes($ty::get_hash(value, $random_state), *hash);
*hash = combine_hashes($random_state.hash_one(value), *hash);
}
} else {
for (hash, value) in $hashes.iter_mut().zip(values.iter()) {
*hash = $ty::get_hash(value, $random_state)
*hash = $random_state.hash_one(value)
}
}
} else {
Expand All @@ -148,16 +146,15 @@ macro_rules! hash_array_primitive {
$hashes.iter_mut().zip(values.iter()).enumerate()
{
if !array.is_null(i) {
*hash =
combine_hashes($ty::get_hash(value, $random_state), *hash);
*hash = combine_hashes($random_state.hash_one(value), *hash);
}
}
} else {
for (i, (hash, value)) in
$hashes.iter_mut().zip(values.iter()).enumerate()
{
if !array.is_null(i) {
*hash = $ty::get_hash(value, $random_state);
*hash = $random_state.hash_one(value);
}
}
}
Expand All @@ -174,19 +171,14 @@ macro_rules! hash_array_float {
if $multi_col {
for (hash, value) in $hashes.iter_mut().zip(values.iter()) {
*hash = combine_hashes(
$ty::get_hash(
&$ty::from_le_bytes(value.to_le_bytes()),
$random_state,
),
$random_state.hash_one(&$ty::from_le_bytes(value.to_le_bytes())),
*hash,
);
}
} else {
for (hash, value) in $hashes.iter_mut().zip(values.iter()) {
*hash = $ty::get_hash(
&$ty::from_le_bytes(value.to_le_bytes()),
$random_state,
)
*hash =
$random_state.hash_one(&$ty::from_le_bytes(value.to_le_bytes()))
}
}
} else {
Expand All @@ -196,10 +188,8 @@ macro_rules! hash_array_float {
{
if !array.is_null(i) {
*hash = combine_hashes(
$ty::get_hash(
&$ty::from_le_bytes(value.to_le_bytes()),
$random_state,
),
$random_state
.hash_one(&$ty::from_le_bytes(value.to_le_bytes())),
*hash,
);
}
Expand All @@ -209,10 +199,8 @@ macro_rules! hash_array_float {
$hashes.iter_mut().zip(values.iter()).enumerate()
{
if !array.is_null(i) {
*hash = $ty::get_hash(
&$ty::from_le_bytes(value.to_le_bytes()),
$random_state,
);
*hash = $random_state
.hash_one(&$ty::from_le_bytes(value.to_le_bytes()));
}
}
}
Expand Down Expand Up @@ -312,7 +300,7 @@ pub fn create_row_hashes<'a>(
*hash = 0
}
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
*hash = <Vec<u8>>::get_hash(&rows[i], random_state);
*hash = random_state.hash_one(&rows[i]);
}
Ok(hashes_buffer)
}
Expand Down
2 changes: 1 addition & 1 deletion datafusion/expr/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ path = "src/lib.rs"
[features]

[dependencies]
ahash = { version = "0.7", default-features = false }
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "20.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "10.0.0" }
sqlparser = "0.20"
2 changes: 1 addition & 1 deletion datafusion/physical-expr/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ regex_expressions = ["regex"]
unicode_expressions = ["unicode-segmentation"]

[dependencies]
ahash = { version = "0.7", default-features = false }
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "20.0.0", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
Expand Down
2 changes: 1 addition & 1 deletion datafusion/sql/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []

[dependencies]
ahash = { version = "0.7", default-features = false }
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "20.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "10.0.0" }
datafusion-expr = { path = "../expr", version = "10.0.0" }
Expand Down