From 4731c4ac4914c9a7fea8cb128e8f63fab9e25d1a Mon Sep 17 00:00:00 2001 From: Zheng Zhu Date: Tue, 11 Aug 2020 23:47:55 -0500 Subject: [PATCH] [BUG]: Must run spark-submit in the same directory as published assembly or UDF's fail with serialization errors Code change to unzip contents of assembly.zip to current directory instead. --- .../scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala b/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala index 5925dcca9..1dee553bf 100644 --- a/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala +++ b/src/scala/microsoft-spark-2.4.x/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala @@ -60,7 +60,10 @@ object DotnetRunner extends Logging { var zipFileName = args(0) val zipFileUri = Try(new URI(zipFileName)).getOrElse(new File(zipFileName).toURI) val workingDir = new File("").getAbsoluteFile - val driverDir = new File(workingDir, FilenameUtils.getBaseName(zipFileUri.getPath())) + //val driverDir = new File(workingDir, FilenameUtils.getBaseName(zipFileUri.getPath())) + // ZZ: By unzipping into the working directory, this ensures currentdir is the dir containing the files that is needed to properly handle UDF's, without the need to set ASSEMBLY_PATHS + // The alternative here would be to change current working directory to the subdirectory path, but that seems like it could have more unintended side effects + val driverDir = workingDir // Standalone cluster mode where .NET application is remotely located. if (zipFileUri.getScheme() != "file") {