Merge pull request #8585 from NVIDIA/branch-23.06

Merge branch-23.06 to main [skip ci]
NVIDIA · Jun 19, 2023 · 643370d · 643370d
2 parents 5806439 + a84e13a
commit 643370d
Show file tree

Hide file tree

Showing 9 changed files with 13 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,5 @@
 # Change log
-Generated on 2023-06-13
+Generated on 2023-06-19
 
 ## Release 23.06
 
@@ -115,6 +115,8 @@ Generated on 2023-06-13
 ### PRs
 |||
 |:---|:---|
+|[#8581](https://github.com/NVIDIA/spark-rapids/pull/8581)|Fix 321db 330db shims build errors caused by DB updates|
+|[#8570](https://github.com/NVIDIA/spark-rapids/pull/8570)|Update changelog to latest [skip ci]|
 |[#8567](https://github.com/NVIDIA/spark-rapids/pull/8567)|Fixed a link in config doc[skip ci]|
 |[#8562](https://github.com/NVIDIA/spark-rapids/pull/8562)|Update changelog to latest 230612 [skip ci]|
 |[#8560](https://github.com/NVIDIA/spark-rapids/pull/8560)|Fix relative path in config doc [skip ci]|

diff --git a/...in/databricks/scala/com/databricks/sql/transaction/tahoe/rapids/GpuMergeIntoCommand.scala b/...in/databricks/scala/com/databricks/sql/transaction/tahoe/rapids/GpuMergeIntoCommand.scala
@@ -343,7 +343,7 @@ case class GpuMergeIntoCommand(
         deltaTxn.commit(
           deltaActions,
           DeltaOperations.Merge(
-            Option(condition.sql),
+            Option(condition),
             matchedClauses.map(DeltaOperations.MergePredicate(_)),
             notMatchedClauses.map(DeltaOperations.MergePredicate(_))))
 

diff --git a/...rk321db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuDeleteCommand.scala b/...rk321db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuDeleteCommand.scala
@@ -101,7 +101,7 @@ case class GpuDeleteCommand(
       gpuDeltaLog.withNewTransaction { txn =>
         val deleteActions = performDelete(sparkSession, deltaLog, txn)
         if (deleteActions.nonEmpty) {
-          txn.commit(deleteActions, DeltaOperations.Delete(condition.map(_.sql).toSeq))
+          txn.commit(deleteActions, DeltaOperations.Delete(condition.toSeq))
         }
       }
       // Re-cache all cached plans(including this relation itself, if it's cached) that refer to

diff --git a/...21db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuOptimizeExecutor.scala b/...21db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuOptimizeExecutor.scala
@@ -85,7 +85,7 @@ class GpuOptimizeExecutor(
       val addedFiles = updates.collect { case a: AddFile => a }
       val removedFiles = updates.collect { case r: RemoveFile => r }
       if (addedFiles.nonEmpty) {
-        val operation = DeltaOperations.Optimize(partitionPredicate.map(_.sql), zOrderByColumns)
+        val operation = DeltaOperations.Optimize(partitionPredicate, zOrderByColumns)
         val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles)
         commitAndRetry(txn, operation, updates, metrics) { newTxn =>
           val newPartitionSchema = newTxn.metadata.partitionSchema
@@ -398,4 +398,4 @@ class GpuOptimizeExecutor(
       "numRemovedBytes" ->
         setAndReturnMetric("total number of bytes removed", totalSize(removedFiles)))
   }
-}
+}
diff --git a/...rk321db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuUpdateCommand.scala b/...rk321db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuUpdateCommand.scala
@@ -205,7 +205,7 @@ case class GpuUpdateCommand(
           metrics("numTouchedRows").value - metrics("numUpdatedRows").value)
       }
       txn.registerSQLMetrics(sparkSession, metrics)
-      txn.commit(totalActions, DeltaOperations.Update(condition.map(_.toString)))
+      txn.commit(totalActions, DeltaOperations.Update(condition))
       // This is needed to make the SQL metrics visible in the Spark UI
       val executionId = sparkSession.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
       SQLMetrics.postDriverMetricUpdates(

diff --git a/...rk330db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuDeleteCommand.scala b/...rk330db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuDeleteCommand.scala
@@ -70,7 +70,7 @@ case class GpuDeleteCommand(
       gpuDeltaLog.withNewTransaction { txn =>
         val deleteActions = performDelete(sparkSession, deltaLog, txn)
         if (deleteActions.nonEmpty) {
-          txn.commit(deleteActions, DeltaOperations.Delete(condition.map(_.sql).toSeq))
+          txn.commit(deleteActions, DeltaOperations.Delete(condition.toSeq))
         }
       }
       // Re-cache all cached plans(including this relation itself, if it's cached) that refer to

diff --git a/...30db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuOptimizeExecutor.scala b/...30db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuOptimizeExecutor.scala
@@ -86,7 +86,7 @@ class GpuOptimizeExecutor(
       val addedFiles = updates.collect { case a: AddFile => a }
       val removedFiles = updates.collect { case r: RemoveFile => r }
       if (addedFiles.nonEmpty) {
-        val operation = DeltaOperations.Optimize(partitionPredicate.map(_.sql), zOrderByColumns)
+        val operation = DeltaOperations.Optimize(partitionPredicate, zOrderByColumns)
         val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles)
         commitAndRetry(txn, operation, updates, metrics) { newTxn =>
           val newPartitionSchema = newTxn.metadata.partitionSchema
@@ -402,4 +402,4 @@ class GpuOptimizeExecutor(
       "numRemovedBytes" ->
         setAndReturnMetric("total number of bytes removed", totalSize(removedFiles)))
   }
-}
+}
diff --git a/...rk330db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuUpdateCommand.scala b/...rk330db/src/main/scala/com/databricks/sql/transaction/tahoe/rapids/GpuUpdateCommand.scala
@@ -203,7 +203,7 @@ case class GpuUpdateCommand(
           metrics("numTouchedRows").value - metrics("numUpdatedRows").value)
       }
       txn.registerSQLMetrics(sparkSession, metrics)
-      txn.commit(totalActions, DeltaOperations.Update(condition.map(_.toString)))
+      txn.commit(totalActions, DeltaOperations.Update(condition))
       // This is needed to make the SQL metrics visible in the Spark UI
       val executionId = sparkSession.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
       SQLMetrics.postDriverMetricUpdates(

diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh
@@ -182,7 +182,7 @@ set_sw_versions()
             sw_versions[JSON4S_AST]="3.7.0-M11"
             sw_versions[JSON4S_CORE]="3.7.0-M11"
             sw_versions[KRYO]="4.0.2"
-            sw_versions[ORC]="1.6.13"
+            sw_versions[ORC]="1.6.14"
             sw_versions[PARQUET]="1.12.0"
             sw_versions[PROTOBUF]="2.6.1"
             ;;