Merge branch 'NVIDIA:branch-24.02' into parse_url_protocol

NVIDIA · Dec 4, 2023 · e4fdf13 · e4fdf13
2 parents 7eca922 + 557680b
commit e4fdf13
Show file tree

Hide file tree

Showing 191 changed files with 2,152 additions and 1,853 deletions.
diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml
@@ -30,7 +30,49 @@ env:
     -Dmaven.scaladoc.skip
 
 jobs:
+  cache-dependencies:
+    runs-on: ubuntu-latest
+    outputs:
+      dailyCacheKey: ${{ steps.generateCacheKey.outputs.dailyCacheKey }}
+    steps:
+      - uses: actions/checkout@v3 # refs/pull/:prNumber/merge
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: 8
+      - name: Generate daily cache key
+        id: generateCacheKey
+        run: |
+          set -x
+          cacheKey="${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-$(date +'%Y-%m-%d')"
+          echo "dailyCacheKey=$cacheKey" | tee $GITHUB_ENV $GITHUB_OUTPUT
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ env.dailyCacheKey }}
+      - name: populate-daily-cache
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          set -x
+          max_retry=3; delay=30; i=1
+          while true; do
+            for pom in pom.xml scala2.13/pom.xml; do 
+              mvn -f $pom help:evaluate -pl dist -Dexpression=included_buildvers \
+              -DforceStdout -PnoSnapshots -q | tr -d ',' | \
+              xargs -n 1 bash -c 'mvn -f $1 initialize -pl sql-plugin-api -am -Dbuildver=$2' _ $pom;
+            done && break || {
+            if [[ $i -le $max_retry ]]; then
+              echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2))
+            else
+              echo "mvn command failed. Exit 1"; exit 1
+            fi
+          }
+          done 
+
   get-shim-versions-from-dist:
+    needs: cache-dependencies
     runs-on: ubuntu-latest
     outputs:
       defaultSparkVersion: ${{ steps.allShimVersionsStep.outputs.defaultSparkVersion }}
@@ -97,7 +139,7 @@ jobs:
           echo "scala213Versions=$svJsonStr" >> $GITHUB_OUTPUT
 
   package-tests:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     continue-on-error: ${{ matrix.isSnapshot }}
     strategy:
       matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkTailVersions) }}
@@ -112,6 +154,12 @@ jobs:
           distribution: adopt
           java-version: 8
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: check runtime before tests
         run: |
           env | grep JAVA
@@ -142,7 +190,7 @@ jobs:
           done
 
   package-tests-scala213:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     continue-on-error: ${{ matrix.isSnapshot }}
     strategy:
       matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.scala213Versions) }}
@@ -157,6 +205,12 @@ jobs:
           distribution: adopt
           java-version: 8
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: check runtime before tests
         run: |
           env | grep JAVA
@@ -198,7 +252,7 @@ jobs:
 
 
   verify-all-modules:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     runs-on: ubuntu-latest
     strategy:
       matrix: ${{ fromJSON(needs.get-shim-versions-from-dist.outputs.sparkJDKVersions) }}
@@ -211,6 +265,12 @@ jobs:
           distribution: adopt
           java-version: ${{ matrix.java-version }}
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: check runtime before tests
         run: |
           env | grep JAVA
@@ -240,7 +300,7 @@ jobs:
           done
 
   install-modules:
-    needs: get-shim-versions-from-dist
+    needs: [get-shim-versions-from-dist, cache-dependencies]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -254,6 +314,12 @@ jobs:
           distribution: adopt
           java-version: 11
 
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: ${{ needs.cache-dependencies.outputs.dailyCacheKey }}
+
       - name: Setup Maven Wrapper
         run: mvn wrapper:wrapper -Dmaven=${{ matrix.maven-version }}
 

diff --git a/README.md b/README.md
@@ -73,7 +73,7 @@ as a `provided` dependency.
 <dependency>
     <groupId>com.nvidia</groupId>
     <artifactId>rapids-4-spark_2.12</artifactId>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
     <scope>provided</scope>
 </dependency>
 ```
diff --git a/aggregator/pom.xml b/aggregator/pom.xml
@@ -22,13 +22,13 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../jdk-profiles/pom.xml</relativePath>
     </parent>
     <artifactId>rapids-4-spark-aggregator_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Aggregator</name>
     <description>Creates an aggregated shaded package of the RAPIDS plugin for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>aggregator</rapids.module>

diff --git a/api_validation/pom.xml b/api_validation/pom.xml
@@ -22,11 +22,11 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-shim-deps-parent_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../shim-deps/pom.xml</relativePath>
     </parent>
     <artifactId>rapids-4-spark-api-validation_2.12</artifactId>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>api_validation</rapids.module>

diff --git a/datagen/README.md b/datagen/README.md
@@ -24,12 +24,12 @@ Where `$SPARK_VERSION` is a compressed version number, like 330 for Spark 3.3.0.
 
 After this the jar should be at
 `target/datagen_2.12-$PLUGIN_VERSION-spark$SPARK_VERSION.jar`
-for example a Spark 3.3.0 jar for the 23.12.0 release would be
-`target/datagen_2.12-23.12.0-spark330.jar`
+for example a Spark 3.3.0 jar for the 24.02.0 release would be
+`target/datagen_2.12-24.02.0-spark330.jar`
 
 To get a spark shell with this you can run
 ```shell
-spark-shell --jars target/datagen_2.12-23.12.0-spark330.jar
+spark-shell --jars target/datagen_2.12-24.02.0-spark330.jar
 ```
 
 After that you should be good to go.

diff --git a/datagen/ScaleTest.md b/datagen/ScaleTest.md
@@ -44,7 +44,7 @@ $SPARK_HOME/bin/spark-submit \
 --conf spark.sql.parquet.datetimeRebaseModeInWrite=CORRECTED \
 --class com.nvidia.rapids.tests.scaletest.ScaleTestDataGen \ # the main class
 --jars $SPARK_HOME/examples/jars/scopt_2.12-3.7.1.jar \ # one dependency jar just shipped with Spark under $SPARK_HOME
-./target/datagen_2.12-23.12.0-SNAPSHOT-spark332.jar \
+./target/datagen_2.12-24.02.0-SNAPSHOT-spark332.jar \
 1 \
 10 \
 parquet \

diff --git a/datagen/pom.xml b/datagen/pom.xml
@@ -21,13 +21,13 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-shim-deps-parent_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../shim-deps/pom.xml</relativePath>
     </parent>
     <artifactId>datagen_2.12</artifactId>
     <name>Data Generator</name>
     <description>Tools for generating large amounts of data</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
     <properties>
         <rapids.module>datagen</rapids.module>
         <target.classifier/>

diff --git a/delta-lake/delta-20x/pom.xml b/delta-lake/delta-20x/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-20x_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Delta Lake 2.0.x Support</name>
     <description>Delta Lake 2.0.x support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-20x</rapids.module>

diff --git a/delta-lake/delta-21x/pom.xml b/delta-lake/delta-21x/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-21x_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Delta Lake 2.1.x Support</name>
     <description>Delta Lake 2.1.x support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-21x</rapids.module>

diff --git a/delta-lake/delta-22x/pom.xml b/delta-lake/delta-22x/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-22x_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Delta Lake 2.2.x Support</name>
     <description>Delta Lake 2.2.x support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-22x</rapids.module>

diff --git a/delta-lake/delta-23x/pom.xml b/delta-lake/delta-23x/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-parent_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-23x_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Delta Lake 2.3.x Support</name>
     <description>Delta Lake 2.3.x support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-23x</rapids.module>

diff --git a/delta-lake/delta-24x/pom.xml b/delta-lake/delta-24x/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-24x_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Delta Lake 2.4.x Support</name>
     <description>Delta Lake 2.4.x support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-24x</rapids.module>

diff --git a/delta-lake/delta-spark321db/pom.xml b/delta-lake/delta-spark321db/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-spark321db_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Databricks 10.4 Delta Lake Support</name>
     <description>Databricks 10.4 Delta Lake support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-spark321db</rapids.module>

diff --git a/delta-lake/delta-spark330db/pom.xml b/delta-lake/delta-spark330db/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-spark330db_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Databricks 11.3 Delta Lake Support</name>
     <description>Databricks 11.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-spark330db</rapids.module>

diff --git a/delta-lake/delta-spark332db/pom.xml b/delta-lake/delta-spark332db/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-spark332db_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Databricks 12.2 Delta Lake Support</name>
     <description>Databricks 12.2 Delta Lake support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-spark332db</rapids.module>

diff --git a/delta-lake/delta-spark341db/pom.xml b/delta-lake/delta-spark341db/pom.xml
@@ -22,14 +22,14 @@
     <parent>
 	<groupId>com.nvidia</groupId>
 	<artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-spark341db_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Databricks 13.3 Delta Lake Support</name>
     <description>Databricks 13.3 Delta Lake support for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.compressed.artifact>false</rapids.compressed.artifact>

diff --git a/delta-lake/delta-stub/pom.xml b/delta-lake/delta-stub/pom.xml
@@ -22,14 +22,14 @@
     <parent>
         <groupId>com.nvidia</groupId>
         <artifactId>rapids-4-spark-jdk-profiles_2.12</artifactId>
-        <version>23.12.0-SNAPSHOT</version>
+        <version>24.02.0-SNAPSHOT</version>
         <relativePath>../../jdk-profiles/pom.xml</relativePath>
     </parent>
 
     <artifactId>rapids-4-spark-delta-stub_2.12</artifactId>
     <name>RAPIDS Accelerator for Apache Spark Delta Lake Stub</name>
     <description>Delta Lake stub for the RAPIDS Accelerator for Apache Spark</description>
-    <version>23.12.0-SNAPSHOT</version>
+    <version>24.02.0-SNAPSHOT</version>
 
     <properties>
         <rapids.module>../delta-lake/delta-stub</rapids.module>