Bug fix: MergeHiveSchemaWithAvro should retain avro properties for li… (

#125) * Bug fix: MergeHiveSchemaWithAvro should retain avro properties for list and map when they are nullable
linkedin · Sep 22, 2022 · fed806b · fed806b
1 parent 5f214b1
commit fed806b
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 1 deletion.
diff --git a/...k-core/src/main/java/org/apache/iceberg/hivelink/core/schema/MergeHiveSchemaWithAvro.java b/...k-core/src/main/java/org/apache/iceberg/hivelink/core/schema/MergeHiveSchemaWithAvro.java
@@ -157,7 +157,14 @@ private Schema checkCompatibilityAndPromote(Schema schema, Schema partner) {
 
   private static void copySchemaProps(Schema from, Schema to) {
     if (from != null) {
-      for (Map.Entry<String, Object> prop : from.getObjectProps().entrySet()) {
+      Schema sanitizedFrom;
+      if (AvroSchemaUtil.isOptionSchema(from)) {
+        // extract the actual type from the nullable union
+        sanitizedFrom = AvroSchemaUtil.fromOption(from);
+      } else {
+        sanitizedFrom = from;
+      }
+      for (Map.Entry<String, Object> prop : sanitizedFrom.getObjectProps().entrySet()) {
         to.addProp(prop.getKey(), prop.getValue());
       }
     }

diff --git a/...link-core/src/test/java/org/apache/iceberg/hivelink/core/TestMergeHiveSchemaWithAvro.java b/...link-core/src/test/java/org/apache/iceberg/hivelink/core/TestMergeHiveSchemaWithAvro.java
@@ -301,6 +301,17 @@ public void shouldRetainMapProp() {
     assertSchema(avro, merge(hive, avro));
   }
 
+  @Test
+  public void shouldRetainNullableMapProp() {
+    String hive = "struct<fa:map<string,int>>";
+    Schema fa = map(Schema.Type.INT);
+    fa.addProp("key-id", 1);
+    fa.addProp("value-id", 2);
+    Schema avro = struct("r1", optional("fa", fa));
+
+    assertSchema(avro, merge(hive, avro));
+  }
+
   @Test
   public void shouldRetainListProp() {
     String hive = "struct<fa:array<int>>";
@@ -311,6 +322,16 @@ public void shouldRetainListProp() {
     assertSchema(avro, merge(hive, avro));
   }
 
+  @Test
+  public void shouldRetainNullableListProp() {
+    String hive = "struct<fa:array<int>>";
+    Schema fa = array(Schema.Type.INT);
+    fa.addProp("element-id", 1);
+    Schema avro = struct("r1", optional("fA", fa));
+
+    assertSchema(avro, merge(hive, avro));
+  }
+
   @Test
   public void shouldRecoverLogicalType() {
     String hive = "struct<fa:date,fb:timestamp,fc:decimal(4,2)>";