From 718bcc1fa239b94bd6b36c7b7388836a00ef8df4 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Wed, 25 Dec 2024 19:20:22 +0800 Subject: [PATCH 1/3] Deploy the sources and Javadoc JARs in the nightly CICD Deploy the sources and javadoc JARS to make sure nightly CICD includes all jars required by Sonatype release Deploy dist jars in the final step to ensure that the POM files are not overwritten Signed-off-by: Tim Liu --- jenkins/spark-nightly-build.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index 00735e02c84..84028e750b3 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 DIST_PL="dist" DIST_PATH="$DIST_PL" # The path of the dist module is used only outside of the mvn cmd +SQL_PLGUIN_PATH="sql-plugin" SCALA_BINARY_VER=${SCALA_BINARY_VER:-"2.12"} if [ $SCALA_BINARY_VER == "2.13" ]; then # Run scala2.13 build and test against JDK17 @@ -33,6 +34,7 @@ if [ $SCALA_BINARY_VER == "2.13" ]; then MVN="$MVN -f scala2.13/" DIST_PATH="scala2.13/$DIST_PL" + SQL_PLGUIN_PATH="scala2.13/$SQL_PLGUIN_PATH" fi WORKSPACE=${WORKSPACE:-$(pwd)} @@ -180,7 +182,6 @@ installDistArtifact ${DEFAULT_CUDA_CLASSIFIER} distWithReducedPom "install" if [[ $SKIP_DEPLOY != 'true' ]]; then - distWithReducedPom "deploy" # this deploys selected submodules that is unconditionally built with Spark 3.2.0 $MVN -B deploy -pl "!${DIST_PL}" \ @@ -189,6 +190,18 @@ if [[ $SKIP_DEPLOY != 'true' ]]; then -Dmaven.scaladoc.skip -Dmaven.scalastyle.skip=true \ $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR \ -Dcuda.version=$DEFAULT_CUDA_CLASSIFIER + + # dist module does not have javadoc and sources jars, use 'sql-plugin' ones instead + SQL_ART_PATH="$(echo -n $SQL_PLGUIN_PATH/target/spark*)/rapids-4-spark-sql_${SCALA_BINARY_VER}-${ART_VER}" + cp $SQL_ART_PATH-sources.jar $DIST_PATH/target/${ART_ID}-${ART_VER}-sources.jar + cp $SQL_ART_PATH-javadoc.jar $DIST_PATH/target/${ART_ID}-${ART_VER}-javadoc.jar + # Deploy the sources and javadoc to make sure nightly CICD includes all jars required by Sonatype release + DEPLOY_TYPES="${DEPLOY_TYPES},jar,jar" + DEPLOY_FILES="${DEPLOY_FILES},$DIST_PL/target/${ART_ID}-${ART_VER}-sources.jar,$DIST_PL/target/${ART_ID}-${ART_VER}-javadoc.jar" + DEPLOY_CLASSIFIERS="${DEPLOY_CLASSIFIERS},sources,javadoc" + ls ${DIST_PATH}/target/ + # Deploy dist jars in the final step to ensure that the POM files are not overwritten + distWithReducedPom "deploy" fi # Parse Spark files from local mvn repo From ab041ff4e6e70cf12edf01f17f2f2cca8b4cd044 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Tue, 7 Jan 2025 16:12:56 +0800 Subject: [PATCH 2/3] Share the deploy scripts for nightly and release CI Support deploying both Scala 2.12 and Scala 2.13 artifacts in jenkins/deploy.sh Signed-off-by: Tim Liu --- jenkins/deploy.sh | 15 +++++++++++---- jenkins/spark-nightly-build.sh | 35 +++++----------------------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh index 8eb336678fa..091b5e48671 100755 --- a/jenkins/deploy.sh +++ b/jenkins/deploy.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,6 +42,9 @@ DIST_PL=${DIST_PL:-"dist"} ###### Build the path of jar(s) to be deployed ###### MVN_SETTINGS=${MVN_SETTINGS:-"jenkins/settings.xml"} MVN="mvn -B -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 -s $MVN_SETTINGS" +SCALA_BINARY_VER=${SCALA_BINARY_VER:-"2.12"} +[ $SCALA_BINARY_VER == "2.13" ] && MVN="$MVN -f scala2.13/" + function mvnEval { $MVN help:evaluate -q -DforceStdout -pl $1 -Dexpression=$2 } @@ -73,9 +76,13 @@ DEPLOY_FILES=$(echo $CLASSIFIERS | sed -e "s;\([^,]*\);${FPATH}-\1.jar;g") SQL_ART_ID=$(mvnEval $SQL_PL project.artifactId) SQL_ART_VER=$(mvnEval $SQL_PL project.version) JS_FPATH="$(echo -n ${SQL_PL}/target/spark*)/${SQL_ART_ID}-${SQL_ART_VER}" -cp $JS_FPATH-sources.jar $FPATH-sources.jar -cp $JS_FPATH-javadoc.jar $FPATH-javadoc.jar - +if [ $SCALA_BINARY_VER == "2.13" ]; then + cp scala2.13/$JS_FPATH-sources.jar scala2.13/$FPATH-sources.jar + cp scala2.13/$JS_FPATH-javadoc.jar scala2.13/$FPATH-javadoc.jar +else + cp $JS_FPATH-sources.jar $FPATH-sources.jar + cp $JS_FPATH-javadoc.jar $FPATH-javadoc.jar +fi echo "Plan to deploy ${FPATH}.jar to $SERVER_URL (ID:$SERVER_ID)" GPG_PLUGIN="org.apache.maven.plugins:maven-gpg-plugin:3.1.0:sign-and-deploy-file" diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index 84028e750b3..dbcfe79bfb9 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -24,7 +24,6 @@ MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 DIST_PL="dist" DIST_PATH="$DIST_PL" # The path of the dist module is used only outside of the mvn cmd -SQL_PLGUIN_PATH="sql-plugin" SCALA_BINARY_VER=${SCALA_BINARY_VER:-"2.12"} if [ $SCALA_BINARY_VER == "2.13" ]; then # Run scala2.13 build and test against JDK17 @@ -34,7 +33,6 @@ if [ $SCALA_BINARY_VER == "2.13" ]; then MVN="$MVN -f scala2.13/" DIST_PATH="scala2.13/$DIST_PL" - SQL_PLGUIN_PATH="scala2.13/$SQL_PLGUIN_PATH" fi WORKSPACE=${WORKSPACE:-$(pwd)} @@ -63,9 +61,6 @@ if [[ "$DIST_INCLUDES_DATABRICKS" == "true" ]] && [[ -n ${SPARK_SHIM_VERSIONS_DA DIST_PROFILE_OPT="$DIST_PROFILE_OPT,"$(IFS=,; echo "${SPARK_SHIM_VERSIONS_DATABRICKS[*]}") fi -DEPLOY_TYPES='jar' -DEPLOY_FILES="${DIST_FPATH}.jar" -DEPLOY_CLASSIFIERS="$DEFAULT_CUDA_CLASSIFIER" # Make sure that the local m2 repo on the build machine has the same pom # installed as the one being pushed to the remote repo. This to prevent # discrepancies between the build machines regardless of how the local repo was populated. @@ -78,16 +73,6 @@ function distWithReducedPom { mvnCmd="install:install-file" mvnExtraFlags="-Dpackaging=jar" ;; - - deploy) - mvnCmd="deploy:deploy-file" - if (( ${#CLASSIFIERS_ARR[@]} > 1 )); then - # try move tmp artifacts back to target folder for simplifying separate release process - mv ${TMP_PATH}/${ART_ID}-${ART_VER}-*.jar ${DIST_PATH}/target/ - fi - mvnExtraFlags="-Durl=${URM_URL}-local -DrepositoryId=snapshots -Dtypes=${DEPLOY_TYPES} -Dfiles=${DEPLOY_FILES} -Dclassifiers=${DEPLOY_CLASSIFIERS}" - ;; - *) echo "Unknown command: $cmd" ;; @@ -170,10 +155,6 @@ if (( ${#CLASSIFIERS_ARR[@]} > 1 )); then # move artifacts to temp for deployment later artifactFile="${ART_ID}-${ART_VER}-${classifier}.jar" mv ${DIST_PATH}/target/${artifactFile} ${TMP_PATH}/ - # update deployment properties - DEPLOY_TYPES="${DEPLOY_TYPES},jar" - DEPLOY_FILES="${DEPLOY_FILES},${DIST_PL}/target/${artifactFile}" - DEPLOY_CLASSIFIERS="${DEPLOY_CLASSIFIERS},${classifier}" done fi # build dist w/ default cuda classifier @@ -182,7 +163,6 @@ installDistArtifact ${DEFAULT_CUDA_CLASSIFIER} distWithReducedPom "install" if [[ $SKIP_DEPLOY != 'true' ]]; then - # this deploys selected submodules that is unconditionally built with Spark 3.2.0 $MVN -B deploy -pl "!${DIST_PL}" \ -Dbuildver=$SPARK_BASE_SHIM_VERSION \ @@ -191,17 +171,12 @@ if [[ $SKIP_DEPLOY != 'true' ]]; then $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR \ -Dcuda.version=$DEFAULT_CUDA_CLASSIFIER - # dist module does not have javadoc and sources jars, use 'sql-plugin' ones instead - SQL_ART_PATH="$(echo -n $SQL_PLGUIN_PATH/target/spark*)/rapids-4-spark-sql_${SCALA_BINARY_VER}-${ART_VER}" - cp $SQL_ART_PATH-sources.jar $DIST_PATH/target/${ART_ID}-${ART_VER}-sources.jar - cp $SQL_ART_PATH-javadoc.jar $DIST_PATH/target/${ART_ID}-${ART_VER}-javadoc.jar - # Deploy the sources and javadoc to make sure nightly CICD includes all jars required by Sonatype release - DEPLOY_TYPES="${DEPLOY_TYPES},jar,jar" - DEPLOY_FILES="${DEPLOY_FILES},$DIST_PL/target/${ART_ID}-${ART_VER}-sources.jar,$DIST_PL/target/${ART_ID}-${ART_VER}-javadoc.jar" - DEPLOY_CLASSIFIERS="${DEPLOY_CLASSIFIERS},sources,javadoc" - ls ${DIST_PATH}/target/ + # try move tmp artifacts back to target folder for simplifying separate release process + if (( ${#CLASSIFIERS_ARR[@]} > 1 )); then + mv ${TMP_PATH}/${ART_ID}-${ART_VER}-*.jar ${DIST_PATH}/target/ + fi # Deploy dist jars in the final step to ensure that the POM files are not overwritten - distWithReducedPom "deploy" + jenkins/deploy.sh fi # Parse Spark files from local mvn repo From be7279ae7197fd533c9214416796b7efb7114b21 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Tue, 7 Jan 2025 17:18:08 +0800 Subject: [PATCH 3/3] Set default environments to deploy artifacts to the internal maven repo Signed-off-by: Tim Liu --- jenkins/spark-nightly-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index dbcfe79bfb9..d8f50141e88 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -176,7 +176,7 @@ if [[ $SKIP_DEPLOY != 'true' ]]; then mv ${TMP_PATH}/${ART_ID}-${ART_VER}-*.jar ${DIST_PATH}/target/ fi # Deploy dist jars in the final step to ensure that the POM files are not overwritten - jenkins/deploy.sh + SERVER_URL=${SERVER_URL:-"$URM_URL"} SERVER_ID=${SERVER_ID:-"snapshots"} jenkins/deploy.sh fi # Parse Spark files from local mvn repo