Revert "[SPARK-48898][SQL] Set nullability correctly in the Variant s…

…chema" This reverts commit 9394b35.
szehon-ho · Dec 11, 2024 · b2c8b30 · b2c8b30
1 parent 3bb9a72
commit b2c8b30
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 34 deletions.
diff --git a/...c/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkShreddingUtils.scala b/...c/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkShreddingUtils.scala
@@ -81,11 +81,8 @@ case object SparkShreddingUtils {
           StructField(TypedValueFieldName, arrayShreddingSchema, nullable = true)
         )
       case StructType(fields) =>
-        // The field name level is always non-nullable: Variant null values are represented in the
-        // "value" columna as "00", and missing values are represented by setting both "value" and
-        // "typed_value" to null.
         val objectShreddingSchema = StructType(fields.map(f =>
-            f.copy(dataType = variantShreddingSchema(f.dataType, false), nullable = false)))
+            f.copy(dataType = variantShreddingSchema(f.dataType, false))))
         Seq(
           StructField(VariantValueFieldName, BinaryType, nullable = true),
           StructField(TypedValueFieldName, objectShreddingSchema, nullable = true)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantWriteShreddingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/VariantWriteShreddingSuite.scala
@@ -67,36 +67,6 @@ class VariantWriteShreddingSuite extends SparkFunSuite with ExpressionEvalHelper
 
   private val emptyMetadata: Array[Byte] = parseJson("null").getMetadata
 
-  test("variantShreddingSchema") {
-    // Validate the schema produced by SparkShreddingUtils.variantShreddingSchema for a few simple
-    // cases.
-    // metadata is always non-nullable.
-    assert(SparkShreddingUtils.variantShreddingSchema(IntegerType) ==
-      StructType(Seq(
-        StructField("metadata", BinaryType, nullable = false),
-        StructField("value", BinaryType, nullable = true),
-        StructField("typed_value", IntegerType, nullable = true))))
-
-    val fieldA = StructType(Seq(
-      StructField("value", BinaryType, nullable = true),
-      StructField("typed_value", TimestampNTZType, nullable = true)))
-    val arrayType = ArrayType(StructType(Seq(
-      StructField("value", BinaryType, nullable = true),
-      StructField("typed_value", StringType, nullable = true))))
-    val fieldB = StructType(Seq(
-      StructField("value", BinaryType, nullable = true),
-      StructField("typed_value", arrayType, nullable = true)))
-    val objectType = StructType(Seq(
-      StructField("a", fieldA, nullable = false),
-      StructField("b", fieldB, nullable = false)))
-    val structSchema = DataType.fromDDL("a timestamp_ntz, b array<string>")
-    assert(SparkShreddingUtils.variantShreddingSchema(structSchema) ==
-      StructType(Seq(
-        StructField("metadata", BinaryType, nullable = false),
-        StructField("value", BinaryType, nullable = true),
-        StructField("typed_value", objectType, nullable = true))))
-  }
-
   test("shredding as fixed numeric types") {
     /* Cast integer to any wider numeric type. */
     testWithSchema("1", IntegerType, Row(emptyMetadata, null, 1))