[SPARK-39502][BUILD] Downgrade scala-maven-plugin to 4.6.1

### What changes were proposed in this pull request?
SPARK-39409 upgrade `scala-maven-plugin` to 4.6.1, but I found that there is a compilation issue when run **`mvn test` with Java 8.**

The reproduction steps are as follows:

```
mvn clean install -DskipTests -pl core -am
mvn test -pl core
```

```
[ERROR] ## Exception when compiling 669 sources to /basedir/spark-mine/core/target/scala-2.12/classes
java.lang.RuntimeException: rt.jar (class sbt.internal.inc.DummyVirtualFile) is not supported
scala.sys.package$.error(package.scala:27)
sbt.internal.inc.Locate$.definesClass(Locate.scala:92)
sbt.internal.inc.Locate.definesClass(Locate.scala)
sbt_inc.SbtIncrementalCompiler$1.definesClass(SbtIncrementalCompiler.java:119)
sbt.internal.inc.Locate$.$anonfun$entry$1(Locate.scala:60)
scala.collection.Iterator$$anon$9.next(Iterator.scala:575)
scala.collection.IterableOnceOps.collectFirst(IterableOnce.scala:1079)
scala.collection.IterableOnceOps.collectFirst$(IterableOnce.scala:1071)
scala.collection.AbstractIterator.collectFirst(Iterator.scala:1288)
sbt.internal.inc.Locate$.$anonfun$entry$2(Locate.scala:67)
sbt.internal.inc.LookupImpl.lookupOnClasspath(LookupImpl.scala:51)
sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$3(IncrementalCommon.scala:764)
sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$3$adapted(IncrementalCommon.scala:754)
scala.collection.IterableOnceOps.exists(IterableOnce.scala:591)
scala.collection.IterableOnceOps.exists$(IterableOnce.scala:588)
scala.collection.AbstractIterable.exists(Iterable.scala:919)
sbt.internal.inc.IncrementalCommon$.isLibraryChanged$1(IncrementalCommon.scala:754)
sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$1(IncrementalCommon.scala:774)
sbt.internal.inc.IncrementalCommon$.$anonfun$isLibraryModified$1$adapted(IncrementalCommon.scala:732)
scala.collection.parallel.AugmentedIterableIterator.filter2combiner(RemainsIterator.scala:136)
scala.collection.parallel.AugmentedIterableIterator.filter2combiner$(RemainsIterator.scala:133)
scala.collection.parallel.immutable.ParVector$ParVectorIterator.filter2combiner(ParVector.scala:72)
scala.collection.parallel.ParIterableLike$Filter.leaf(ParIterableLike.scala:1083)
scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:52)
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:97)
scala.collection.parallel.Task.tryLeaf(Tasks.scala:55)
scala.collection.parallel.Task.tryLeaf$(Tasks.scala:49)
scala.collection.parallel.ParIterableLike$Filter.tryLeaf(ParIterableLike.scala:1079)
scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal(Tasks.scala:159)
scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.internal$(Tasks.scala:156)
scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.internal(Tasks.scala:303)
scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:149)
scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:148)
scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:303)
java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
java.util.concurrent.ForkJoinTask.doJoin(ForkJoinTask.java:389)
java.util.concurrent.ForkJoinTask.join(ForkJoinTask.java:719)
scala.collection.parallel.ForkJoinTasks$WrappedTask.sync(Tasks.scala:242)
scala.collection.parallel.ForkJoinTasks$WrappedTask.sync$(Tasks.scala:242)
scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.sync(Tasks.scala:303)
scala.collection.parallel.ForkJoinTasks.executeAndWaitResult(Tasks.scala:286)
scala.collection.parallel.ForkJoinTasks.executeAndWaitResult$(Tasks.scala:279)
scala.collection.parallel.ForkJoinTaskSupport.executeAndWaitResult(TaskSupport.scala:59)
scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult(Tasks.scala:409)
scala.collection.parallel.ExecutionContextTasks.executeAndWaitResult$(Tasks.scala:409)
scala.collection.parallel.ExecutionContextTaskSupport.executeAndWaitResult(TaskSupport.scala:75)
scala.collection.parallel.ParIterableLike$ResultMapping.leaf(ParIterableLike.scala:932)
scala.collection.parallel.Task.$anonfun$tryLeaf$1(Tasks.scala:52)
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
scala.util.control.Breaks$$anon$1.catchBreak(Breaks.scala:97)
scala.collection.parallel.Task.tryLeaf(Tasks.scala:55)
scala.collection.parallel.Task.tryLeaf$(Tasks.scala:49)
scala.collection.parallel.ParIterableLike$ResultMapping.tryLeaf(ParIterableLike.scala:927)
scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute(Tasks.scala:152)
scala.collection.parallel.AdaptiveWorkStealingTasks$WrappedTask.compute$(Tasks.scala:148)
scala.collection.parallel.AdaptiveWorkStealingForkJoinTasks$WrappedTask.compute(Tasks.scala:303)
java.util.concurrent.RecursiveAction.exec(RecursiveAction.java:189)
java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
```

Use Java 11 and 17 no this issue and downgrade to scala-maven-plugin to 4.6.1 can avoid this issue.

So this pr downgrade scala-maven-plugin to 4.6.1.

### Why are the changes needed?
Fix compilation issue when run `mvn test` with Java 8.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
Pass GAs

Closes #36899 from LuciferYang/SPARK-39502.

Authored-by: yangjie01 <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>

pom.xml

@@ -165,8 +165,11 @@
       errors building different Hadoop versions.
       See: SPARK-36547, SPARK-38394.
        -->
-
-    <scala-maven-plugin.version>4.6.2</scala-maven-plugin.version>
+    <!--
+      SPARK-39502: Please don't upgrade the version to 4.6.2 due to
+      there is a compilation issue when run `mvn test` with Java 8
+      -->
+    <scala-maven-plugin.version>4.6.1</scala-maven-plugin.version>
     <!-- for now, not running scalafmt as part of default verify pipeline -->
     <scalafmt.skip>true</scalafmt.skip>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>