From e5febdaceb2d7c43764427af74d550a9954a717a Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 17 Jun 2022 19:15:53 +0900 Subject: [PATCH] [SPARK-39501][TESTS] Propagate `java.net.preferIPv6Addresses=true` in SBT tests ### What changes were proposed in this pull request? This PR aims to propagate `java.net.preferIPv6Addresses=true` in SBT tests. - This also fixes several `hive` module (which downloads Hive or Hadoop dependencies) failures. - `Maven` will be handled separately. ### Why are the changes needed? To test IPv6 SBT tests, we need to set consistently with the `SBT_OPTS`. - https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/net/doc-files/net-properties.html **java.net.preferIPv6Addresses (default: false)** > When dealing with a host which has both IPv4 and IPv6 addresses, and if IPv6 is available on the operating system, the default behavior is to prefer using IPv4 addresses over IPv6 ones. This is to ensure backward compatibility, for example applications that depend on the representation of an IPv4 address (e.g. 192.168.1.1). This property can be set to true to change that preference and use IPv6 addresses over IPv4 ones where possible, or system to preserve the order of the addresses as returned by the operating system. ### Does this PR introduce _any_ user-facing change? This is a test-only change. ### How was this patch tested? Pass the GitHub Action with new test cases. 1. New test case will be disabled by default or when `java.net.preferIPv6Addresses` is not `true`. 2. With `java.net.preferIPv6Addresses=true` in IPv6 environment, new test case will pass. ``` [info] SparkSubmitUtilsSuite: :: loading settings :: url = jar:file:/Users/dongjoon/Library/Caches/Coursier/v1/https/maven-central.storage-download.googleapis.com/maven2/org/apache/ivy/ivy/2.5.0/ivy-2.5.0.jar!/org/apache/ivy/core/settings/ivysettings.xml [info] - SPARK-39501: Resolve maven dependenicy in IPv6 (243 milliseconds) [info] Run completed in 1 second, 141 milliseconds. [info] Total number of tests run: 1 [info] Suites: completed 1, aborted 0 [info] Tests: succeeded 1, failed 0, canceled 0, ignored 0, pending 0 [info] All tests passed. ``` Closes #36898 from dongjoon-hyun/SPARK-39501. Authored-by: Dongjoon Hyun Signed-off-by: Hyukjin Kwon --- core/src/main/scala/org/apache/spark/util/Utils.scala | 5 +++++ .../org/apache/spark/deploy/SparkSubmitUtilsSuite.scala | 8 +++++++- project/SparkBuild.scala | 7 +++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index cf93897f97d86..56392f0239c83 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1988,6 +1988,11 @@ private[spark] object Utils extends Logging { */ val isMacOnAppleSilicon = SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64") + /** + * Whether the underlying JVM prefer IPv6 addresses. + */ + val preferIPv6 = "true".equals(System.getProperty("java.net.preferIPv6Addresses")) + /** * Pattern for matching a Windows drive, which contains only a single alphabet character. */ diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala index 8945885347925..c118c58cafea2 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitUtilsSuite.scala @@ -32,7 +32,7 @@ import org.scalatest.BeforeAndAfterAll import org.apache.spark.SparkFunSuite import org.apache.spark.deploy.SparkSubmitUtils.MavenCoordinate -import org.apache.spark.util.Utils +import org.apache.spark.util.{DependencyUtils, Utils} class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { @@ -304,4 +304,10 @@ class SparkSubmitUtilsSuite extends SparkFunSuite with BeforeAndAfterAll { s" Resolved jars are: $jarPath") } } + + test("SPARK-39501: Resolve maven dependenicy in IPv6") { + assume(Utils.preferIPv6) + DependencyUtils.resolveMavenDependencies( + URI.create("ivy://org.apache.logging.log4j:log4j-api:2.17.2")) + } } diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 568cb57976b92..6a33aae6b4116 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1165,6 +1165,13 @@ object TestSettings { (Test / javaOptions) += "-Dsun.io.serialization.extendedDebugInfo=false", (Test / javaOptions) += "-Dderby.system.durability=test", (Test / javaOptions) += "-Dio.netty.tryReflectionSetAccessible=true", + (Test / javaOptions) ++= { + if ("true".equals(System.getProperty("java.net.preferIPv6Addresses"))) { + Seq("-Djava.net.preferIPv6Addresses=true") + } else { + Seq.empty + } + }, (Test / javaOptions) ++= System.getProperties.asScala.filter(_._1.startsWith("spark")) .map { case (k,v) => s"-D$k=$v" }.toSeq, (Test / javaOptions) += "-ea",