Add session property to configure ORC rowGroupMaxRowCount in Iceberg

mayankvadariya · Oct 22, 2024 · cabe3ae · cabe3ae
1 parent 4d8b8d1
commit cabe3ae
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java
@@ -61,6 +61,7 @@
 import static io.trino.plugin.iceberg.IcebergSessionProperties.getCompressionCodec;
 import static io.trino.plugin.iceberg.IcebergSessionProperties.getOrcStringStatisticsLimit;
 import static io.trino.plugin.iceberg.IcebergSessionProperties.getOrcWriterMaxDictionaryMemory;
+import static io.trino.plugin.iceberg.IcebergSessionProperties.getOrcWriterMaxRowGroupRows;
 import static io.trino.plugin.iceberg.IcebergSessionProperties.getOrcWriterMaxStripeRows;
 import static io.trino.plugin.iceberg.IcebergSessionProperties.getOrcWriterMaxStripeSize;
 import static io.trino.plugin.iceberg.IcebergSessionProperties.getOrcWriterMinStripeSize;
@@ -244,6 +245,7 @@ private IcebergFileWriter createOrcWriter(
                             .withStripeMinSize(getOrcWriterMinStripeSize(session))
                             .withStripeMaxSize(getOrcWriterMaxStripeSize(session))
                             .withStripeMaxRowCount(getOrcWriterMaxStripeRows(session))
+                            .withRowGroupMaxRowCount(getOrcWriterMaxRowGroupRows(session))
                             .withDictionaryMaxMemory(getOrcWriterMaxDictionaryMemory(session))
                             .withMaxStringStatisticsLimit(stringStatisticsLimit),
                     IntStream.range(0, fileColumnNames.size()).toArray(),

diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java
@@ -81,6 +81,7 @@ public final class IcebergSessionProperties
     private static final String ORC_WRITER_MIN_STRIPE_SIZE = "orc_writer_min_stripe_size";
     private static final String ORC_WRITER_MAX_STRIPE_SIZE = "orc_writer_max_stripe_size";
     private static final String ORC_WRITER_MAX_STRIPE_ROWS = "orc_writer_max_stripe_rows";
+    private static final String ORC_WRITER_MAX_ROW_GROUP_ROWS = "orc_writer_max_row_group_rows";
     private static final String ORC_WRITER_MAX_DICTIONARY_MEMORY = "orc_writer_max_dictionary_memory";
     private static final String PARQUET_MAX_READ_BLOCK_SIZE = "parquet_max_read_block_size";
     private static final String PARQUET_USE_BLOOM_FILTER = "parquet_use_bloom_filter";
@@ -217,6 +218,11 @@ public IcebergSessionProperties(
                         "ORC: Max stripe row count",
                         orcWriterConfig.getStripeMaxRowCount(),
                         false))
+                .add(integerProperty(
+                        ORC_WRITER_MAX_ROW_GROUP_ROWS,
+                        "ORC: Max number of rows in a row group",
+                        orcWriterConfig.getRowGroupMaxRowCount(),
+                        false))
                 .add(dataSizeProperty(
                         ORC_WRITER_MAX_DICTIONARY_MEMORY,
                         "ORC: Max dictionary memory",
@@ -472,6 +478,11 @@ public static int getOrcWriterMaxStripeRows(ConnectorSession session)
         return session.getProperty(ORC_WRITER_MAX_STRIPE_ROWS, Integer.class);
     }
 
+    public static int getOrcWriterMaxRowGroupRows(ConnectorSession session)
+    {
+        return session.getProperty(ORC_WRITER_MAX_ROW_GROUP_ROWS, Integer.class);
+    }
+
     public static DataSize getOrcWriterMaxDictionaryMemory(ConnectorSession session)
     {
         return session.getProperty(ORC_WRITER_MAX_DICTIONARY_MEMORY, DataSize.class);