fix infinite case

mrpowers-io · Jan 5, 2025 · 72e7c26 · 72e7c26
1 parent d69be53
commit 72e7c26
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala
@@ -280,7 +280,7 @@ object DataFrameExt {
 
       @tailrec
       def flatten(df: DataFrame): DataFrame = {
-        if (StructTypeHelpers.containComplexFields(df.schema)) {
+        if (StructTypeHelpers.containComplexFields(df.schema, flattenArrayType)) {
           val renamedCols = StructTypeHelpers
             .flattenSchema(df.schema, "", flattenArrayType)
             .map { case (c, n) => c.as(sanitizeColName(n, delimiter)) }

diff --git a/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala
@@ -41,11 +41,11 @@ object StructTypeHelpers {
     }
   }
 
-  def containComplexFields(schema: StructType): Boolean = {
+  def containComplexFields(schema: StructType, isArrayComplex: Boolean): Boolean = {
     schema.fields.exists {
-      case StructField(_, _: StructType, _, _) => true
-      case StructField(_, _: ArrayType, _, _)  => true
-      case _                                   => false
+      case StructField(_, _: StructType, _, _)                   => true
+      case StructField(_, _: ArrayType, _, _) if isArrayComplex  => true
+      case _                                                     => false
     }
   }