Apply suggestions from code review

Co-authored-by: Piotr Findeisen <[email protected]>
adriangb · Nov 1, 2024 · d0aee77 · d0aee77
1 parent e3f3171
commit d0aee77
Showing 1 changed file with 2 additions and 6 deletions.
diff --git a/datafusion/core/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs
@@ -1713,13 +1713,9 @@ fn increment_utf8(data: &str) -> Option<String> {
     fn is_valid_unicode(c: char) -> bool {
         let cp = c as u32;
 
-        // Filter out Unicode surrogate pairs range
-        if (0xD800..=0xDFFF).contains(&cp) {
-            return false;
-        }
 
-        // Filter out non-characters
-        if (0xFDD0..=0xFDEF).contains(&cp) {
+        // Filter out non-characters (https://www.unicode.org/versions/corrigendum9.html)
+        if [0xFFFE, 0xFFFF].contains(&cp) || (0xFDD0..=0xFDEF).contains(&cp) {
             return false;
         }