Fix DictionaryArray::normalized_keys (#4788)

arrow-array/src/array/dictionary_array.rs

@@ -1005,7 +1005,7 @@ impl<K: ArrowDictionaryKeyType> AnyDictionaryArray for DictionaryArray<K> {
         let v_len = self.values().len();
         assert_ne!(v_len, 0);
         let iter = self.keys().values().iter();
-        iter.map(|x| x.as_usize().min(v_len)).collect()
+        iter.map(|x| x.as_usize().min(v_len - 1)).collect()
     }
 
     fn with_values(&self, values: ArrayRef) -> ArrayRef {
@@ -1385,4 +1385,13 @@ mod tests {
             .collect();
         assert_eq!(values, &[Some(50), None, None, Some(2)])
     }
+
+    #[test]
+    fn test_normalized_keys() {
+        let values = vec![132, 0, 1].into();
+        let nulls = NullBuffer::from(vec![false, true, true]);
+        let keys = Int32Array::new(values, Some(nulls));
+        let dictionary = DictionaryArray::new(keys, Arc::new(Int32Array::new_null(2)));
+        assert_eq!(&dictionary.normalized_keys(), &[1, 0, 1])
+    }
 }

arrow-ord/src/cmp.rs

@@ -555,7 +555,7 @@ impl<'a> ArrayOrd for &'a FixedSizeBinaryArray {
 mod tests {
     use std::sync::Arc;
 
-    use arrow_array::{DictionaryArray, Int32Array, Scalar};
+    use arrow_array::{DictionaryArray, Int32Array, Scalar, StringArray};
 
     use super::*;
 
@@ -708,4 +708,16 @@ mod tests {
         let r = eq(&b, &a).unwrap();
         assert_eq!(r.len(), 0);
     }
+
+    #[test]
+    fn test_dictionary_nulls() {
+        let values = StringArray::from(vec![Some("us-west"), Some("us-east")]);
+        let nulls = NullBuffer::from(vec![false, true, true]);
+
+        let key_values = vec![100i32, 1i32, 0i32].into();
+        let keys = Int32Array::new(key_values, Some(nulls));
+        let col = DictionaryArray::try_new(keys, Arc::new(values)).unwrap();
+
+        neq(&col.slice(0, col.len() - 1), &col.slice(1, col.len() - 1)).unwrap();
+    }
 }