Performance improvement for BinaryFileReader and ImageReader

Temporary workaround for a bug introduced in spark 2.1 (from 2.0).
microsoft · Aug 15, 2017 · 2426bf0 · 2426bf0
1 parent d8eb988
commit 2426bf0
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/readers/src/main/scala/BinaryFileReader.scala b/src/readers/src/main/scala/BinaryFileReader.scala
@@ -40,6 +40,7 @@ object BinaryFileReader {
     var data: RDD[(String, Array[Byte])] = null
     try {
       val streams = spark.sparkContext.binaryFiles(path, spark.sparkContext.defaultParallelism)
+        .repartition(spark.sparkContext.defaultParallelism)
 
       // Create files RDD and load bytes
       data = if (!inspectZip) {