From 9cc7cdc62f58c15c9e51365b978f43a4b00fb278 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 17 Jun 2022 19:16:55 +0900 Subject: [PATCH] [SPARK-39502][BUILD] Downgrade scala-maven-plugin to 4.6.1 ### What changes were proposed in this pull request? SPARK-39409 upgrade `scala-maven-plugin` to 4.6.1, but I found that there is a compilation issue when run **`mvn test` with Java 8.** The reproduction steps are as follows: ``` mvn clean install -DskipTests -pl core -am mvn test -pl core ``` ``` [ERROR] ## Exception when compiling 669 sources to /basedir/spark-mine/core/target/scala-2.12/classes java.lang.RuntimeException: rt.jar (class sbt.internal.inc.DummyVirtualFile) is not supported scala.sys.package$.error(package.scala:27) sbt.internal.inc.Locate$.definesClass(Locate.scala:92) sbt.internal.inc.Locate.definesClass(Locate.scala) sbt_inc.SbtIncrementalCompiler$1.definesClass(SbtIncrementalCompiler.java:119) sbt.internal.inc.Locate$.$anonfun$entry$1(Locate.scala:60) scala.collection.Iterator$$anon$9.next(Iterator.scala:575) scala.collection.IterableOnceOps.collectFirst(IterableOnce.scala:1079) scala.collection.IterableOnceOps.collectFirst$(IterableOnce.scala:1071) scala.collection.AbstractIterator.collectFirst(Iterator.scala:1288) sbt.internal.inc.Locate$.$anonfun$entry$2(Locate.scala:67) sbt.internal.inc.LookupImpl.lookupOnClasspath(LookupImpl.scala:51) sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$3(IncrementalCommon.scala:764) sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$3$adapted(IncrementalCommon.scala:754) scala.collection.IterableOnceOps.exists(IterableOnce.scala:591) scala.collection.IterableOnceOps.exists$(IterableOnce.scala:588) scala.collection.AbstractIterable.exists(Iterable.scala:919) sbt.internal.inc.IncrementalCommon$.isLibraryChanged$1(IncrementalCommon.scala:754) sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$1(IncrementalCommon.scala:774) sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$1$adapted(IncrementalCommon.scala:732) scala.collection.parallel.AugmentedIterableIterator.filter2combiner(RemainsIterator.scala:136) scala.collection.parallel.AugmentedIterableIterator.filter2combiner$(RemainsIterator.scala:133) scala.collection.parallel.immutable.ParVector$ParVectorIterator.filter2combiner(ParVector.scala:72) scala.collection.parallel.ParIterableLike$Filter.leaf(ParIterableLike.scala:1083) scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:52) scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:97) scala.collection.parallel.Task.tryLeaf(Tasks.scala:55) scala.collection.parallel.Task.tryLeaf$(Tasks.scala:49) scala.collection.parallel.ParIterableLike$Filter.tryLeaf(ParIterableLike.scala:1079) scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:159) scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal$(Tasks.scala:156) scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.internal(Tasks.scala:303) scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:149) scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:148) scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:303) java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389) java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719) scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:242) scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:242) scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:303) scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:286) scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:279) scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:59) scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult(Tasks.scala:409) scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult$(Tasks.scala:409) scala.collection.parallel.ExecutionContextTaskSupport.executeAndWaitResult(TaskSupport.scala:75) scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:932) scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:52) scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:97) scala.collection.parallel.Task.tryLeaf(Tasks.scala:55) scala.collection.parallel.Task.tryLeaf$(Tasks.scala:49) scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:927) scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:152) scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:148) scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:303) java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189) java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) ``` Use Java 11 and 17 no this issue and downgrade to scala-maven-plugin to 4.6.1 can avoid this issue. So this pr downgrade scala-maven-plugin to 4.6.1. ### Why are the changes needed? Fix compilation issue when run `mvn test` with Java 8. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GAs Closes #36899 from LuciferYang/SPARK-39502. Authored-by: yangjie01 Signed-off-by: Hyukjin Kwon --- pom.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9d52a9f504019..6749dfd8422df 100644 --- a/pom.xml +++ b/pom.xml @@ -165,8 +165,11 @@ errors building different Hadoop versions. See: SPARK-36547, SPARK-38394. --> - - 4.6.2 + + 4.6.1 true 1.9.13