Allow testing records with sibling whitespace in SLT tests and add mo…

…re string tests (#13197)

* empty

* Allow testing values with trailing whitespace in SLT tests

* Update SLT tests for "Allow testing values with trailing whitespace ..."

* Add empty string to string test data

datafusion/sqllogictest/bin/sqllogictests.rs

@@ -22,6 +22,7 @@ use std::path::{Path, PathBuf};
 use clap::Parser;
 use datafusion_sqllogictest::{DataFusion, TestContext};
 use futures::stream::StreamExt;
+use itertools::Itertools;
 use log::info;
 use sqllogictest::strict_column_validator;
 
@@ -39,6 +40,23 @@ pub fn main() -> Result<()> {
         .block_on(run_tests())
 }
 
+fn value_validator(actual: &[Vec<String>], expected: &[String]) -> bool {
+    let expected = expected
+        .iter()
+        // Trailing whitespace from lines in SLT will typically be removed, but do not fail if it is not
+        // If particular test wants to cover trailing whitespace on a value,
+        // it should project additional non-whitespace column on the right.
+        .map(|s| s.trim_end().to_owned())
+        .collect::<Vec<_>>();
+    let actual = actual
+        .iter()
+        .map(|strs| strs.iter().join(" "))
+        // Editors do not preserve trailing whitespace, so expected may or may not lack it included
+        .map(|s| s.trim_end().to_owned())
+        .collect::<Vec<_>>();
+    actual == expected
+}
+
 /// Sets up an empty directory at test_files/scratch/<name>
 /// creating it if needed and clearing any file contents if it exists
 /// This allows tests for inserting to external tables or copy to
@@ -140,6 +158,7 @@ async fn run_test_file(test_file: TestFile) -> Result<()> {
         ))
     });
     runner.with_column_validator(strict_column_validator);
+    runner.with_validator(value_validator);
     runner
         .run_file_async(path)
         .await
@@ -158,6 +177,7 @@ async fn run_test_file_with_postgres(test_file: TestFile) -> Result<()> {
     let mut runner =
         sqllogictest::Runner::new(|| Postgres::connect(relative_path.clone()));
     runner.with_column_validator(strict_column_validator);
+    runner.with_validator(value_validator);
     runner
         .run_file_async(path)
         .await
@@ -176,7 +196,6 @@ async fn run_complete_file(test_file: TestFile) -> Result<()> {
         path,
         relative_path,
     } = test_file;
-    use sqllogictest::default_validator;
 
     info!("Using complete mode to complete: {}", path.display());
 
@@ -196,7 +215,7 @@ async fn run_complete_file(test_file: TestFile) -> Result<()> {
         .update_test_file(
             path,
             col_separator,
-            default_validator,
+            value_validator,
             strict_column_validator,
         )
         .await

datafusion/sqllogictest/test_files/avro.slt

@@ -198,22 +198,22 @@ NULL
 query IT
 SELECT id, CAST(string_col AS varchar) FROM alltypes_plain_multi_files
 ----
-4  0
-5  1
-6  0
-7  1
-2  0
-3  1
-0  0
-1  1
-4  0
-5  1
-6  0
-7  1
-2  0
-3  1
-0  0
-1  1
+4 0
+5 1
+6 0
+7 1
+2 0
+3 1
+0 0
+1 1
+4 0
+5 1
+6 0
+7 1
+2 0
+3 1
+0 0
+1 1
 
 # test avro nested records
 query ????

datafusion/sqllogictest/test_files/map.slt

@@ -433,7 +433,7 @@ SELECT MAP {'a': 1, null: 2}
 query ?
 SELECT MAP {[1,2,3]:1, [2,4]:2};
 ----
- {[1, 2, 3]: 1, [2, 4]: 2}
+{[1, 2, 3]: 1, [2, 4]: 2}
 
 # array with different type as key
 # expect to fail due to type coercion error
@@ -483,7 +483,7 @@ SELECT MAP { MAP {1:'a', 2:'b'}:1, MAP {1:'c', 2:'d'}:2 };
 query ?
 SELECT MAP { MAP {1:'a', 2:'b', 3:'c'}:1, MAP {2:'c', 4:'d'}:2 };
 ----
- {{1: a, 2: b, 3: c}: 1, {2: c, 4: d}: 2}
+{{1: a, 2: b, 3: c}: 1, {2: c, 4: d}: 2}
 
 # map as value
 query ?

datafusion/sqllogictest/test_files/pg_compat/pg_compat_types.slt

@@ -24,18 +24,18 @@ NULL
 query TT
 select 'a'::VARCHAR, ''::VARCHAR
 ----
-a  (empty)
+a (empty)
 
 skipif postgres
 query TT
 select 'a'::CHAR, ''::CHAR
 ----
-a  (empty)
+a (empty)
 
 query TT
 select 'a'::TEXT, ''::TEXT
 ----
-a  (empty)
+a (empty)
 
 skipif postgres
 query I

datafusion/sqllogictest/test_files/select.slt

@@ -459,7 +459,7 @@ VALUES (-1)
 query IIB
 VALUES (2+1,2-1,2>1)
 ----
-3    1    true
+3 1 true
 
 # multiple rows values
 query I rowsort
@@ -472,8 +472,8 @@ VALUES (1),(2)
 query IT rowsort
 VALUES (1,'a'),(2,'b')
 ----
-1   a
-2   b
+1 a
+2 b
 
 # table foo for distinct order by
 statement ok

datafusion/sqllogictest/test_files/string/dictionary_utf8.slt

@@ -58,6 +58,8 @@ Xiangpeng datafusion数据融合 false true false true
 Raphael datafusionДатаФусион false false false false
 under_score un iść core false false false false
 percent pan Tadeusz ma iść w kąt false false false false
+(empty) (empty) false false false false
+NULL NULL NULL NULL NULL NULL
 NULL NULL NULL NULL NULL NULL
 
 #

datafusion/sqllogictest/test_files/string/init_data.slt.part

@@ -22,6 +22,8 @@ create table test_source as values
   ('Raphael', 'R', 'datafusionДатаФусион', 'аФус'),
   ('under_score', 'un_____core', 'un iść core', 'chrząszcz na łące w 東京都'),
   ('percent', 'p%t', 'pan Tadeusz ma iść w kąt', 'Pan Tadeusz ma frunąć stąd w kąt'),
+  ('', '%', '', ''),
+  (NULL, '%', NULL, NULL),
   (NULL, 'R', NULL, '🔥');
 
 # --------------------------------------

datafusion/sqllogictest/test_files/string/large_string.slt

@@ -43,6 +43,8 @@ Xiangpeng Xiangpeng datafusion数据融合 datafusion数据融合
 Raphael R datafusionДатаФусион аФус
 under_score un_____core un iść core chrząszcz na łące w 東京都
 percent p%t pan Tadeusz ma iść w kąt Pan Tadeusz ma frunąć stąd w kąt
+(empty) % (empty) (empty)
+NULL % NULL NULL
 NULL R NULL 🔥
 
 # TODO: move it back to `string_query.slt.part` after fixing the issue
@@ -61,6 +63,8 @@ Xiangpeng datafusion数据融合 false true false true
 Raphael datafusionДатаФусион false false false false
 under_score un iść core false false false false
 percent pan Tadeusz ma iść w kąt false false false false
+(empty) (empty) false false false false
+NULL NULL NULL NULL NULL NULL
 NULL NULL NULL NULL NULL NULL
 
 #

datafusion/sqllogictest/test_files/string/string.slt

@@ -50,6 +50,8 @@ Xiangpeng datafusion数据融合 false true false true
 Raphael datafusionДатаФусион false false false false
 under_score un iść core false false false false
 percent pan Tadeusz ma iść w kąt false false false false
+(empty) (empty) false false false false
+NULL NULL NULL NULL NULL NULL
 NULL NULL NULL NULL NULL NULL
 
 #
@@ -72,6 +74,9 @@ SELECT unicode_2, 'is LIKE', ascii_2 FROM test_basic_operator WHERE unicode_2 LI
 UNION ALL
 SELECT unicode_2, 'is NOT LIKE', ascii_2 FROM test_basic_operator WHERE unicode_2 NOT LIKE ascii_2
 ----
+(empty) is LIKE %
+(empty) is LIKE %
+(empty) is LIKE %
 Andrew is NOT LIKE X
 Pan Tadeusz ma frunąć stąd w kąt is NOT LIKE p%t
 Raphael is NOT LIKE R
@@ -100,7 +105,9 @@ SELECT
     (unicode_2 LIKE ascii_2) AS unicode_2_like_ascii_2
 FROM test_basic_operator
 ----
+(empty) % (empty) (empty) true false true true
 Andrew X datafusion📊🔥 🔥 false false false false
+NULL % NULL NULL NULL NULL NULL NULL
 NULL R NULL 🔥 NULL NULL NULL false
 Raphael R datafusionДатаФусион аФус false false false false
 Xiangpeng Xiangpeng datafusion数据融合 datafusion数据融合 true true false false
@@ -122,6 +129,9 @@ SELECT unicode_2, 'is ILIKE', ascii_2 FROM test_basic_operator WHERE unicode_2 I
 UNION ALL
 SELECT unicode_2, 'is NOT ILIKE', ascii_2 FROM test_basic_operator WHERE unicode_2 NOT ILIKE ascii_2
 ----
+(empty) is ILIKE %
+(empty) is ILIKE %
+(empty) is ILIKE %
 Andrew is NOT ILIKE X
 Pan Tadeusz ma frunąć stąd w kąt is ILIKE p%t
 Raphael is NOT ILIKE R
@@ -150,7 +160,9 @@ SELECT
     (unicode_2 ILIKE ascii_2) AS unicode_2_ilike_ascii_2
 FROM test_basic_operator
 ----
+(empty) % (empty) (empty) true false true true
 Andrew X datafusion📊🔥 🔥 false false false false
+NULL % NULL NULL NULL NULL NULL NULL
 NULL R NULL 🔥 NULL NULL NULL false
 Raphael R datafusionДатаФусион аФус false false false false
 Xiangpeng Xiangpeng datafusion数据融合 datafusion数据融合 true true false false

datafusion/sqllogictest/test_files/string/string_literal.slt

@@ -816,3 +816,8 @@ query B
 SELECT starts_with('foobar', 'bar')
 ----
 false
+
+query TT
+select '   ', '|'
+----
+    |