Skip to content

Commit

Permalink
[SPARK-33725][BUILD] Upgrade snappy-java to 1.1.8.2
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?

This upgrades snappy-java to 1.1.8.2.

### Why are the changes needed?

Minor version upgrade that includes:

- [Fixed](xerial/snappy-java#265) an initialization issue when using a recent Mac OS X version
- Support Apple Silicon (M1, Mac-aarch64)
- Fixed the pure-java Snappy fallback logic when no native library for your platform is found.

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Unit test.

Closes #30690 from viirya/upgrade-snappy.

Authored-by: Liang-Chi Hsieh <viirya@gmail.com>
Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
  • Loading branch information
a0x8o committed Dec 9, 2020
1 parent a9b49c3 commit 41a51c0
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 5 deletions.
2 changes: 1 addition & 1 deletion dev/deps/spark-deps-hadoop-2.7-hive-2.3
Expand Up @@ -221,7 +221,7 @@ shims/0.9.0//shims-0.9.0.jar
slf4j-api/1.7.30//slf4j-api-1.7.30.jar
slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
snakeyaml/1.26//snakeyaml-1.26.jar
snappy-java/1.1.8//snappy-java-1.1.8.jar
snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
Expand Down
2 changes: 1 addition & 1 deletion dev/deps/spark-deps-hadoop-3.2-hive-2.3
Expand Up @@ -236,7 +236,7 @@ shims/0.9.0//shims-0.9.0.jar
slf4j-api/1.7.30//slf4j-api-1.7.30.jar
slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
snakeyaml/1.26//snakeyaml-1.26.jar
snappy-java/1.1.8//snappy-java-1.1.8.jar
snappy-java/1.1.8.2//snappy-java-1.1.8.2.jar
spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Expand Up @@ -171,7 +171,7 @@
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
<fasterxml.jackson.version>2.10.5</fasterxml.jackson.version>
<fasterxml.jackson-databind.version>2.10.5.1</fasterxml.jackson-databind.version>
<snappy.version>1.1.8</snappy.version>
<snappy.version>1.1.8.2</snappy.version>
<netlib.java.version>1.1.2</netlib.java.version>
<commons-codec.version>1.10</commons-codec.version>
<commons-compress.version>1.20</commons-compress.version>
Expand Down
Expand Up @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, CaseWhen, Expression, If}
import org.apache.spark.sql.catalyst.expressions.{LambdaFunction, Literal, MapFilter, Or}
import org.apache.spark.sql.catalyst.expressions.Literal.FalseLiteral
import org.apache.spark.sql.catalyst.plans.logical.{Filter, Join, LogicalPlan}
import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, Filter, Join, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.types.BooleanType
import org.apache.spark.util.Utils
Expand Down Expand Up @@ -53,6 +53,7 @@ object ReplaceNullWithFalseInPredicate extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case f @ Filter(cond, _) => f.copy(condition = replaceNullWithFalse(cond))
case j @ Join(_, _, _, Some(cond), _) => j.copy(condition = Some(replaceNullWithFalse(cond)))
case d @ DeleteFromTable(_, Some(cond)) => d.copy(condition = Some(replaceNullWithFalse(cond)))
case p: LogicalPlan => p transformExpressions {
case i @ If(pred, _, _) => i.copy(predicate = replaceNullWithFalse(pred))
case cw @ CaseWhen(branches, _) =>
Expand Down
Expand Up @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.dsl.plans._
import org.apache.spark.sql.catalyst.expressions.{And, ArrayExists, ArrayFilter, ArrayTransform, CaseWhen, Expression, GreaterThan, If, LambdaFunction, Literal, MapFilter, NamedExpression, Or, UnresolvedNamedLambdaVariable}
import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
import org.apache.spark.sql.catalyst.plans.{Inner, PlanTest}
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.plans.logical.{DeleteFromTable, LocalRelation, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{BooleanType, IntegerType}
Expand All @@ -48,6 +48,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
test("replace null inside filter and join conditions") {
testFilter(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
testJoin(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
testDelete(originalCond = Literal(null, BooleanType), expectedCond = FalseLiteral)
}

test("Not expected type - replaceNullWithFalse") {
Expand All @@ -64,6 +65,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
Literal(null, BooleanType))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace nulls in nested expressions in branches of If") {
Expand All @@ -73,6 +75,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
UnresolvedAttribute("b") && Literal(null, BooleanType))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace null in elseValue of CaseWhen") {
Expand All @@ -83,6 +86,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
val expectedCond = CaseWhen(branches, FalseLiteral)
testFilter(originalCond, expectedCond)
testJoin(originalCond, expectedCond)
testDelete(originalCond, expectedCond)
}

test("replace null in branch values of CaseWhen") {
Expand All @@ -92,6 +96,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
val originalCond = CaseWhen(branches, Literal(null))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace null in branches of If inside CaseWhen") {
Expand All @@ -108,6 +113,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {

testFilter(originalCond, expectedCond)
testJoin(originalCond, expectedCond)
testDelete(originalCond, expectedCond)
}

test("replace null in complex CaseWhen expressions") {
Expand All @@ -127,19 +133,22 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {

testFilter(originalCond, expectedCond)
testJoin(originalCond, expectedCond)
testDelete(originalCond, expectedCond)
}

test("replace null in Or") {
val originalCond = Or(UnresolvedAttribute("b"), Literal(null))
val expectedCond = UnresolvedAttribute("b")
testFilter(originalCond, expectedCond)
testJoin(originalCond, expectedCond)
testDelete(originalCond, expectedCond)
}

test("replace null in And") {
val originalCond = And(UnresolvedAttribute("b"), Literal(null))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace nulls in nested And/Or expressions") {
Expand All @@ -148,6 +157,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
Or(Literal(null), And(Literal(null), And(UnresolvedAttribute("b"), Literal(null)))))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace null in And inside branches of If") {
Expand All @@ -157,6 +167,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
And(UnresolvedAttribute("b"), Literal(null, BooleanType)))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace null in branches of If inside And") {
Expand All @@ -168,6 +179,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
And(FalseLiteral, UnresolvedAttribute("b"))))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace null in branches of If inside another If") {
Expand All @@ -177,13 +189,15 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
Literal(null))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("replace null in CaseWhen inside another CaseWhen") {
val nestedCaseWhen = CaseWhen(Seq(UnresolvedAttribute("b") -> FalseLiteral), Literal(null))
val originalCond = CaseWhen(Seq(nestedCaseWhen -> TrueLiteral), Literal(null))
testFilter(originalCond, expectedCond = FalseLiteral)
testJoin(originalCond, expectedCond = FalseLiteral)
testDelete(originalCond, expectedCond = FalseLiteral)
}

test("inability to replace null in non-boolean branches of If") {
Expand All @@ -196,6 +210,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
FalseLiteral)
testFilter(originalCond = condition, expectedCond = condition)
testJoin(originalCond = condition, expectedCond = condition)
testDelete(originalCond = condition, expectedCond = condition)
}

test("inability to replace null in non-boolean values of CaseWhen") {
Expand All @@ -210,6 +225,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
val condition = CaseWhen(branches)
testFilter(originalCond = condition, expectedCond = condition)
testJoin(originalCond = condition, expectedCond = condition)
testDelete(originalCond = condition, expectedCond = condition)
}

test("inability to replace null in non-boolean branches of If inside another If") {
Expand All @@ -222,6 +238,7 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
FalseLiteral)
testFilter(originalCond = condition, expectedCond = condition)
testJoin(originalCond = condition, expectedCond = condition)
testDelete(originalCond = condition, expectedCond = condition)
}

test("replace null in If used as a join condition") {
Expand Down Expand Up @@ -353,6 +370,10 @@ class ReplaceNullWithFalseInPredicateSuite extends PlanTest {
test((rel, exp) => rel.select(exp), originalExpr, expectedExpr)
}

private def testDelete(originalCond: Expression, expectedCond: Expression): Unit = {
test((rel, expr) => DeleteFromTable(rel, Some(expr)), originalCond, expectedCond)
}

private def testHigherOrderFunc(
argument: Expression,
createExpr: (Expression, Expression) => Expression,
Expand Down

0 comments on commit 41a51c0

Please sign in to comment.