From 72e7c26de73f54064ad6f407ddc5075587afcaa7 Mon Sep 17 00:00:00 2001 From: Tuan Pham Date: Mon, 6 Jan 2025 08:51:38 +1100 Subject: [PATCH] fix infinite case --- .../github/mrpowers/spark/daria/sql/DataFrameExt.scala | 2 +- .../spark/daria/sql/types/StructTypeHelpers.scala | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala index 5d76559..cc81df2 100644 --- a/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala +++ b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala @@ -280,7 +280,7 @@ object DataFrameExt { @tailrec def flatten(df: DataFrame): DataFrame = { - if (StructTypeHelpers.containComplexFields(df.schema)) { + if (StructTypeHelpers.containComplexFields(df.schema, flattenArrayType)) { val renamedCols = StructTypeHelpers .flattenSchema(df.schema, "", flattenArrayType) .map { case (c, n) => c.as(sanitizeColName(n, delimiter)) } diff --git a/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala index 187919b..a83987f 100644 --- a/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala +++ b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala @@ -41,11 +41,11 @@ object StructTypeHelpers { } } - def containComplexFields(schema: StructType): Boolean = { + def containComplexFields(schema: StructType, isArrayComplex: Boolean): Boolean = { schema.fields.exists { - case StructField(_, _: StructType, _, _) => true - case StructField(_, _: ArrayType, _, _) => true - case _ => false + case StructField(_, _: StructType, _, _) => true + case StructField(_, _: ArrayType, _, _) if isArrayComplex => true + case _ => false } }