From 86ad2a56cda825e19ed6ff62f91650030f08fcfe Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Fri, 16 Feb 2024 21:49:30 -0800 Subject: [PATCH 1/2] Reverts NVIDIA/spark-rapids#10232 and fixes the plugin build on Databricks 11.3 [databricks] (#10439) * Reverts NVIDIA/spark-rapids#10232 and fixes the plugin build on Databricks 11.3 Signed-off-by: raza jafri * changed copyrights back --------- Signed-off-by: raza jafri --- .../sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala | 1 + .../execution/python/shims/GpuCoGroupedArrowPythonRunner.scala | 1 + .../sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala | 1 + .../execution/python/shims/GpuGroupUDFArrowPythonRunner.scala | 1 + .../sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala | 1 - .../sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala | 1 - .../execution/python/shims/GpuCoGroupedArrowPythonRunner.scala | 1 - .../execution/python/shims/GpuGroupUDFArrowPythonRunner.scala | 1 - 8 files changed, 4 insertions(+), 4 deletions(-) rename sql-plugin/src/main/{spark330db => spark341db}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala (99%) rename sql-plugin/src/main/{spark330db => spark341db}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala (99%) rename sql-plugin/src/main/{spark330db => spark341db}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala (99%) rename sql-plugin/src/main/{spark330db => spark341db}/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala (99%) diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala index c211b29a9f0..68decf764a3 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala @@ -27,6 +27,7 @@ {"spark": "324"} {"spark": "330"} {"spark": "330cdh"} +{"spark": "330db"} {"spark": "331"} {"spark": "332"} {"spark": "332cdh"} diff --git a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala index 1fb974ec9bd..f5c4aaf77d9 100644 --- a/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark311/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala @@ -27,6 +27,7 @@ {"spark": "324"} {"spark": "330"} {"spark": "330cdh"} +{"spark": "330db"} {"spark": "331"} {"spark": "332"} {"spark": "332cdh"} diff --git a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala index 064701b811c..41d297e520f 100644 --- a/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala +++ b/sql-plugin/src/main/spark320/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala @@ -24,6 +24,7 @@ {"spark": "324"} {"spark": "330"} {"spark": "330cdh"} +{"spark": "330db"} {"spark": "331"} {"spark": "332"} {"spark": "332cdh"} diff --git a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala index 6623fc8765f..9165bc80b9d 100644 --- a/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark321db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala @@ -19,6 +19,7 @@ /*** spark-rapids-shim-json-lines {"spark": "321db"} +{"spark": "330db"} {"spark": "332db"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala similarity index 99% rename from sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala rename to sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala index f229e50528a..2a351f66329 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonOutput.scala @@ -15,7 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "330db"} {"spark": "341db"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala similarity index 99% rename from sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala rename to sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala index fc79236095a..17106833a99 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuArrowPythonRunner.scala @@ -14,7 +14,6 @@ * limitations under the License. */ /*** spark-rapids-shim-json-lines -{"spark": "330db"} {"spark": "341db"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala similarity index 99% rename from sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala rename to sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala index 6f657950a91..7e52ce7a3d0 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuCoGroupedArrowPythonRunner.scala @@ -15,7 +15,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "330db"} {"spark": "341db"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims diff --git a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala similarity index 99% rename from sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala rename to sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala index ec3bde02434..4393c8b7057 100644 --- a/sql-plugin/src/main/spark330db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala +++ b/sql-plugin/src/main/spark341db/scala/org/apache/spark/sql/rapids/execution/python/shims/GpuGroupUDFArrowPythonRunner.scala @@ -18,7 +18,6 @@ */ /*** spark-rapids-shim-json-lines -{"spark": "330db"} {"spark": "341db"} spark-rapids-shim-json-lines ***/ package org.apache.spark.sql.rapids.execution.python.shims From fce531d390df4f83b668a09e92309948d587e9cc Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Sat, 17 Feb 2024 14:49:35 +0800 Subject: [PATCH 2/2] Update change log for v24.02.0 release (#10443) Signed-off-by: Tim Liu --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0783ace096..fdb53243cdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Change log -Generated on 2024-02-15 +Generated on 2024-02-17 ## Release 24.02 @@ -119,6 +119,8 @@ Generated on 2024-02-15 ### PRs ||| |:---|:---| +|[#10439](https://github.com/NVIDIA/spark-rapids/pull/10439)|Reverts NVIDIA/spark-rapids#10232 and fixes the plugin build on Databricks 11.3| +|[#10380](https://github.com/NVIDIA/spark-rapids/pull/10380)|Init changelog 24.02 [skip ci]| |[#10367](https://github.com/NVIDIA/spark-rapids/pull/10367)|Update rapids JNI and private version to release 24.02.0| |[#10414](https://github.com/NVIDIA/spark-rapids/pull/10414)|[DOC] Fix 24.02.0 documentation errors [skip ci]| |[#10403](https://github.com/NVIDIA/spark-rapids/pull/10403)|Cherry-pick: Fix a memory leak in json tuple (#10360)|