From 778a89bc7f2e6061b44735806f6b7b2f39133094 Mon Sep 17 00:00:00 2001 From: Jialiang Tan Date: Fri, 25 Oct 2024 15:57:27 -0700 Subject: [PATCH] [native] Add session property native_writer_flush_threshold_bytes --- .../sphinx/presto_cpp/properties-session.rst | 11 ++++++----- .../presto/SystemSessionProperties.java | 7 +++++++ .../presto_cpp/main/SessionProperties.cpp | 19 ++++++++++--------- .../presto_cpp/main/SessionProperties.h | 8 ++++---- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index ebd6c4153070f..017f513be1588 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -233,13 +233,14 @@ Native Execution only. Enable topN row number spilling on native engine. Native Execution only. Enable window spilling on native engine. -``native_writer_spill_enabled`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``native_writer_flush_threshold_bytes`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* **Type:** ``boolean`` -* **Default value:** ``true`` +* **Type:** ``bigint`` +* **Default value:** ``100663296`` -Native Execution only. Enable writer spilling on native engine. +Minimum memory footprint size required to reclaim memory from a file writer by flushing its buffered data to disk. +Default is 96MB. ``native_max_output_buffer_size`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index c22c7e80f06a8..cd4a110eb9e62 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -345,6 +345,7 @@ public final class SystemSessionProperties public static final String NATIVE_JOIN_SPILL_ENABLED = "native_join_spill_enabled"; public static final String NATIVE_WINDOW_SPILL_ENABLED = "native_window_spill_enabled"; public static final String NATIVE_WRITER_SPILL_ENABLED = "native_writer_spill_enabled"; + public static final String NATIVE_WRITER_FLUSH_THRESHOLD_BYTES = "native_writer_flush_threshold_bytes"; public static final String NATIVE_ROW_NUMBER_SPILL_ENABLED = "native_row_number_spill_enabled"; public static final String NATIVE_TOPN_ROW_NUMBER_SPILL_ENABLED = "native_topn_row_number_spill_enabled"; public static final String NATIVE_SPILLER_NUM_PARTITION_BITS = "native_spiller_num_partition_bits"; @@ -1709,6 +1710,12 @@ public SystemSessionProperties( "Native Execution only. Enable writer spilling on native engine", false, false), + longProperty( + NATIVE_WRITER_FLUSH_THRESHOLD_BYTES, + "Native Execution only. Minimum memory footprint size required to reclaim memory from a file " + + "writer by flushing its buffered data to disk.", + 96L << 20, + false), booleanProperty( NATIVE_ROW_NUMBER_SPILL_ENABLED, "Native Execution only. Enable row number spilling on native engine", diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index cf63b5a75c395..3ca4a98ee0e65 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -167,6 +167,16 @@ SessionProperties::SessionProperties() { QueryConfig::kWriterSpillEnabled, boolToString(c.writerSpillEnabled())); + addSessionProperty( + kWriterFlushThresholdBytes, + "Native Execution only. Minimum memory footprint size required " + "to reclaim memory from a file writer by flushing its buffered data to " + "disk.", + BIGINT(), + false, + QueryConfig::kWriterFlushThresholdBytes, + std::to_string(c.writerFlushThresholdBytes())); + addSessionProperty( kRowNumberSpillEnabled, "Native Execution only. Enable row number spilling on native engine", @@ -175,15 +185,6 @@ SessionProperties::SessionProperties() { QueryConfig::kRowNumberSpillEnabled, boolToString(c.rowNumberSpillEnabled())); - addSessionProperty( - kJoinSpillPartitionBits, - "Native Execution only. The number of bits (N) used to calculate the " - "spilling partition number for hash join and RowNumber: 2 ^ N", - INTEGER(), - false, - QueryConfig::kJoinSpillPartitionBits, - std::to_string(c.rowNumberSpillEnabled())); - addSessionProperty( kSpillerNumPartitionBits, "none", diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index c9a5db98d3326..b135fcff27fcf 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -131,10 +131,10 @@ class SessionProperties { static constexpr const char* kWriterSpillEnabled = "native_writer_spill_enabled"; - /// The number of bits (N) used to calculate the spilling - /// partition number for hash join and RowNumber: 2 ^ N - static constexpr const char* kJoinSpillPartitionBits = - "native_join_spiller_partition_bits"; + /// Minimum memory footprint size required to reclaim memory from a file + /// writer by flushing its buffered data to disk. + static constexpr const char* kWriterFlushThresholdBytes = + "native_writer_flush_threshold_bytes"; static constexpr const char* kSpillerNumPartitionBits = "native_spiller_num_partition_bits";