temporary commit: \0 becomes replacement char

and all other control chars too, even tab and newline

have CI include these tests as well but only if the normal tests pass

this is just a proof of concept commit, i expect to undo it soon

.github/workflows/rust.yml

@@ -21,4 +21,6 @@ jobs:
     - name: Build
       run: cargo build --features "toml markdown ansi termion-backend crossterm-backend" --no-default-features --verbose
     - name: Run tests
-      run: cargo test --features "toml markdown ansi termion-backend crossterm-backend" --no-default-features --verbose
+      run: >
+        cargo test --features "toml markdown ansi termion-backend crossterm-backend" --no-default-features --verbose &&
+        cargo test --example select_test -- --nocapture

cursive-core/src/buffer.rs

@@ -179,14 +179,32 @@ impl PrintBuffer {
 
         let mut pos = start;
 
+        fn is_control_char(g: &str) -> bool {
+            g.chars().all(|c| {
+                (
+                    c <= '\u{001F}'
+                    //&& (c != '\u{0009}' && c != '\u{000A}') //TODO: special case for tab or newline?
+                ) || (c >= '\u{007F}' && c <= '\u{009F}')
+            })
+        }
         // Fill our active buffer
         // TODO: Use some WithWidth(&str, usize) to not re-compute width a thousand times
         for g in text.graphemes(true) {
             let width = g.width();
+            //if (width == 0) || ("\0" == g) {
             if width == 0 {
                 continue;
             }
-            self.set_cell(pos, g, CellWidth::from_usize(width), style);
+            if is_control_char(g) {
+                debug_assert_eq!(
+                    1, width,
+                    "Control character '{g}' should've had a width of 1"
+                );
+                debug_assert_eq!(1, "\u{FFFD}".width(), "\\0 should've had a width of 1");
+                self.set_cell(pos, "\u{fffd}", CellWidth::from_usize(width), style);
+            } else {
+                self.set_cell(pos, g, CellWidth::from_usize(width), style);
+            }
             pos.x += width;
         }
     }

cursive-core/src/utils/lines/spans/tests.rs

@@ -16,6 +16,50 @@ fn input() -> StyledString {
     text
 }
 
+#[test]
+fn test_nuls_have_width_1() {
+    use unicode_width::UnicodeWidthStr;
+    assert_eq!("\0".width(), 1, "nul chars should have width 1 since unicode-width 0.1.13, seen here: https://github.com/unicode-rs/unicode-width/commit/4efb1803faa054f1bea3c0457275ad3c8610170b#diff-2ad10836ccce5ac2056d5679cc92449d9ff9094d4ff5c5803f65b5dd1d52ef19R224");
+    let replacement_char = "\u{FFFD}";
+    assert_eq!(
+        replacement_char.width(),
+        1,
+        "REPLACEMENT CHAR='{replacement_char}' for nul='\0' chars should be width 1"
+    );
+}
+
+#[test]
+fn test_control_chars_have_width_1() {
+    use unicode_width::UnicodeWidthStr;
+    let control_chars = [
+        "\u{0000}", "\u{0001}", "\u{0002}", "\u{0003}", "\u{0004}", "\u{0005}", "\u{0006}",
+        "\u{0007}", "\u{0008}", "\u{0009}", "\u{000A}", "\u{000B}", "\u{000C}", "\u{000D}",
+        "\u{000E}", "\u{000F}", "\u{0010}", "\u{0011}", "\u{0012}", "\u{0013}", "\u{0014}",
+        "\u{0015}", "\u{0016}", "\u{0017}", "\u{0018}", "\u{0019}", "\u{001A}", "\u{001B}",
+        "\u{001C}", "\u{001D}", "\u{001E}", "\u{001F}", "\u{007F}", "\u{0080}", "\u{0081}",
+        "\u{0082}", "\u{0083}", "\u{0084}", "\u{0085}", "\u{0086}", "\u{0087}", "\u{0088}",
+        "\u{0089}", "\u{008A}", "\u{008B}", "\u{008C}", "\u{008D}", "\u{008E}", "\u{008F}",
+        "\u{0090}", "\u{0091}", "\u{0092}", "\u{0093}", "\u{0094}", "\u{0095}", "\u{0096}",
+        "\u{0097}", "\u{0098}", "\u{0099}", "\u{009A}", "\u{009B}", "\u{009C}", "\u{009D}",
+        "\u{009E}", "\u{009F}",
+    ];
+
+    for c in &control_chars {
+        let width = c.width();
+        assert_eq!(
+            c.chars().count(),
+            1,
+            "it's supposed to be a string of 1 char"
+        );
+        let unicode_escape = format!("\\u{{{:04X}}}", c.chars().last().unwrap() as u32);
+        debug_assert_eq!(
+            width, 1,
+            "Width of control character {} is not 1",
+            unicode_escape
+        );
+    }
+}
+
 #[test]
 fn test_next_line_char() {
     use unicode_width::UnicodeWidthStr;

cursive/examples/select_test.rs

@@ -40,6 +40,16 @@ pub mod tests {
             // (We include the file at compile-time to avoid runtime read errors.)
             let content = include_str!("assets/cities.txt");
             select.add_all_str(content.lines());
+            select.add_item_str("short \0nul\0 1str");
+            select.add_item_str("1\x01\x02\x03\x04\x05\x06\x07\x08thru8");
+            select.add_item_str("tab\x09and\x0Anewline");
+            select.add_item_str("b\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15thru15");
+            select.add_item_str("16\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1Fthru1F");
+            select.add_item_str("7F\x7Fonly");
+            select.add_item_str("80\u{0080}\u{0081}\u{0082}\u{0083}\u{0084}\u{0085}\u{0086}\u{0087}\u{0088}\u{0089}thru89");
+            select.add_item_str("8A\u{008A}\u{008B}\u{008C}\u{008D}\u{008E}\u{008F}\u{0090}\u{0091}\u{0092}\u{0093}thru93");
+            select.add_item_str("94\u{0094}\u{0095}\u{0096}\u{0097}\u{0098}\u{0099}\u{009A}\u{009B}\u{009C}\u{009D}thru9D");
+            select.add_item_str("9E\u{009E}\u{009F}thru9F");
 
             // Sets the callback for when "Enter" is pressed.
             select.set_on_submit(show_next_window);
@@ -114,6 +124,71 @@ pub mod tests {
         assert_eq!(screen.find_occurences("Some random string").len(), 0);
     }
 
+    #[test]
+    fn nuls_become_replacement_char() {
+        let mut s = BasicSetup::new();
+        s.hit_keystroke(Key::End);
+        let screen = s.last_screen().unwrap();
+        s.dump_debug();
+        assert_eq!(
+            screen
+                .find_occurences("short \u{fffd}nul\u{FFFD} 1str")
+                .len(),
+            1,
+            "nuls aka \\0 in strings are supposed to become the replacement char '\u{fffd}'"
+        );
+    }
+    #[test]
+    fn control_chars_become_replacement_char() {
+        let mut s = BasicSetup::new();
+        s.hit_keystroke(Key::End);
+        let screen = s.last_screen().unwrap();
+        s.dump_debug();
+        let replacement_char = "\u{FFFD} aka \\u{FFFD}";
+        assert_eq!(
+            screen.find_occurences("tab�and�newline").len(),
+            1,
+            "tabs and newline should've been replaced with replacement char {}",
+            replacement_char
+        );
+        assert_eq!(
+            screen.find_occurences("b�����������thru15").len(),
+            1,
+            "control chars \\x0B thru \\x15 should've been replaced with the replacement char {}",
+            replacement_char
+        );
+        assert_eq!(
+            screen.find_occurences("16����������thru1F").len(),
+            1,
+            "control chars \\x16 thru \\x1F should've been replaced with the replacement char {}",
+            replacement_char
+        );
+        assert_eq!(
+            screen.find_occurences("7F�only").len(),
+            1,
+            "control char \\x7F should've been replaced with the replacement char {}",
+            replacement_char
+        );
+        assert_eq!(
+            screen.find_occurences("80����������thru89").len(),
+            1,
+            "control chars \\x80 thru \\x89 should've been replaced with the replacement char {}",
+            replacement_char
+        );
+        assert_eq!(
+            screen.find_occurences("8A����������thru93").len(),
+            1,
+            "control chars \\x8A thru \\x93 should've been replaced with the replacement char {}",
+            replacement_char
+        );
+        assert_eq!(
+            screen.find_occurences("9E��thru9F").len(),
+            1,
+            "control chars \\x9E thru \\x9F should've been replaced with the replacement char {}",
+            replacement_char
+        );
+    }
+
     #[test]
     fn interacts() {
         let mut s = BasicSetup::new();