feat: #20 : comma spacing and #498 : Asian commas

harper-core/src/linting/comma_fixes.rs

@@ -0,0 +1,206 @@
+use super::{Lint, LintKind, Linter, Suggestion};
+use crate::{
+    Span,
+    TokenKind::{Space, Unlintable, Word},
+    TokenStringExt,
+};
+
+const MSG_SPACE_BEFORE: &str = "Don't use a space before a comma.";
+const MSG_AVOID_ASIAN: &str = "Avoid East Asian commas in English contexts.";
+const MSG_SPACE_AFTER: &str = "Use a space after a comma.";
+
+/// A linter that fixes common comma errors:
+/// No space after.
+/// Inappropriate space before.
+/// Asian commas instead of English commas.
+/// This linter only Asian commas anywhere, and wrong spacing of commas between words.
+/// Commas between numbers are used differently in different contexts and these are not checked:
+/// Lists of numbers: 1, 2, 3
+/// Thousands separators: 1,000,000
+/// Decimal points used mistakenly by Europeans: 3,14159
+#[derive(Debug, Default)]
+pub struct CommaFixes;
+
+impl Linter for CommaFixes {
+    fn lint(&mut self, document: &crate::Document) -> Vec<Lint> {
+        let mut lints = Vec::new();
+        let source = document.get_source();
+
+        for ci in document.iter_comma_indices() {
+            let mut toks = (None, None, document.get_token(ci).unwrap(), None, None);
+            toks.0 = (ci >= 2).then(|| document.get_token(ci - 2).unwrap());
+            toks.1 = (ci >= 1).then(|| document.get_token(ci - 1).unwrap());
+            toks.3 = document.get_token(ci + 1);
+            toks.4 = document.get_token(ci + 2);
+
+            let kinds = (
+                toks.0.map(|t| &t.kind),
+                toks.1.map(|t| &t.kind),
+                *toks.2.span.get_content(source).first().unwrap(),
+                toks.3.map(|t| &t.kind),
+                toks.4.map(|t| &t.kind),
+            );
+
+            let (span, suggestion, message) = match kinds {
+                (_, Some(Word(_)), '、' | '，', Some(Space(_)), Some(Word(_))) => (
+                    toks.2.span,
+                    Suggestion::ReplaceWith(vec![',']),
+                    vec![MSG_AVOID_ASIAN],
+                ),
+
+                (Some(Word(_)), Some(Space(_)), ',', Some(Space(_)), Some(Word(_))) => (
+                    toks.1.unwrap().span,
+                    Suggestion::Remove,
+                    vec![MSG_SPACE_BEFORE],
+                ),
+
+                (Some(Word(_)), Some(Space(_)), '、' | '，', Some(Space(_)), Some(Word(_))) => (
+                    Span::new(toks.1.unwrap().span.start, toks.2.span.end),
+                    Suggestion::ReplaceWith(vec![',']),
+                    vec![MSG_SPACE_BEFORE, MSG_AVOID_ASIAN],
+                ),
+
+                (_, Some(Word(_)), ',', Some(Word(_)), _) => (
+                    toks.2.span,
+                    Suggestion::InsertAfter(vec![' ']),
+                    vec![MSG_SPACE_AFTER],
+                ),
+
+                (_, Some(Word(_)), '、' | '，', Some(Word(_)), _) => (
+                    toks.2.span,
+                    Suggestion::ReplaceWith(vec![',', ' ']),
+                    vec![MSG_AVOID_ASIAN, MSG_SPACE_AFTER],
+                ),
+
+                (Some(Word(_)), Some(Space(_)), ',', Some(Word(_)), _) => (
+                    Span::new(toks.1.unwrap().span.start, toks.2.span.end),
+                    Suggestion::ReplaceWith(vec![',', ' ']),
+                    vec![MSG_SPACE_BEFORE, MSG_SPACE_AFTER],
+                ),
+
+                (Some(Word(_)), Some(Space(_)), '、' | '，', Some(Word(_)), _) => (
+                    Span::new(toks.1.unwrap().span.start, toks.2.span.end),
+                    Suggestion::ReplaceWith(vec![',', ' ']),
+                    vec![MSG_SPACE_BEFORE, MSG_AVOID_ASIAN, MSG_SPACE_AFTER],
+                ),
+
+                // Handles Asian commas in all other contexts
+                // Unlintable is used for non-English tokens to prevent changing commas in CJK text
+                (_, Some(Unlintable), '、' | '，', _, _) => continue,
+                (_, _, '、' | '，', Some(Unlintable), _) => continue,
+
+                (_, _, '、' | '，', _, _) => (
+                    toks.2.span,
+                    Suggestion::ReplaceWith(vec![',']),
+                    vec![MSG_AVOID_ASIAN],
+                ),
+
+                _ => continue,
+            };
+
+            lints.push(Lint {
+                span,
+                lint_kind: LintKind::Punctuation,
+                suggestions: vec![suggestion],
+                message: message.join(" "),
+                priority: 32,
+            });
+        }
+
+        lints
+    }
+
+    fn description(&self) -> &'static str {
+        "Fix common comma errors such as no space after, erroneous space before, etc, Asian commas instead of English commas, etc."
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::CommaFixes;
+    use crate::linting::tests::{assert_lint_count, assert_suggestion_result};
+
+    #[test]
+    fn allows_english_comma_atomic() {
+        assert_lint_count(",", CommaFixes, 0);
+    }
+
+    #[test]
+    fn flags_fullwidth_comma_atomic() {
+        assert_lint_count("，", CommaFixes, 1);
+    }
+
+    #[test]
+    fn flags_ideographic_comma_atomic() {
+        assert_lint_count("、", CommaFixes, 1);
+    }
+
+    #[test]
+    fn corrects_fullwidth_comma_real_world() {
+        assert_suggestion_result(
+            "higher 2 bits of the number of nodes， whether abandoned or not decided by .index section",
+            CommaFixes,
+            "higher 2 bits of the number of nodes, whether abandoned or not decided by .index section",
+        );
+    }
+
+    #[test]
+    fn corrects_ideographic_comma_real_world() {
+        assert_suggestion_result("cout、endl、string", CommaFixes, "cout, endl, string")
+    }
+
+    #[test]
+    fn doesnt_flag_comma_space_between_words() {
+        assert_lint_count("foo, bar", CommaFixes, 0);
+    }
+
+    #[test]
+    fn flags_fullwidth_comma_space_between_words() {
+        assert_lint_count("foo， bar", CommaFixes, 1);
+    }
+
+    #[test]
+    fn flags_ideographic_comma_space_between_words() {
+        assert_lint_count("foo、 bar", CommaFixes, 1);
+    }
+
+    #[test]
+    fn doesnt_flag_semicolon_space_between_words() {
+        assert_lint_count("foo; bar", CommaFixes, 0);
+    }
+
+    #[test]
+    fn corrects_comma_between_words_with_no_space() {
+        assert_suggestion_result("foo,bar", CommaFixes, "foo, bar")
+    }
+
+    #[test]
+    fn corrects_asian_comma_between_words_with_no_space() {
+        assert_suggestion_result("foo，bar", CommaFixes, "foo, bar")
+    }
+
+    #[test]
+    fn corrects_space_on_wrong_side_of_comma_between_words() {
+        assert_suggestion_result("foo ,bar", CommaFixes, "foo, bar")
+    }
+
+    #[test]
+    fn corrects_comma_on_wrong_side_of_asian_comma_between_words() {
+        assert_suggestion_result("foo ，bar", CommaFixes, "foo, bar")
+    }
+
+    #[test]
+    fn corrects_comma_between_words_with_space_on_both_sides() {
+        assert_suggestion_result("foo , bar", CommaFixes, "foo, bar")
+    }
+
+    #[test]
+    fn corrects_asian_comma_between_words_with_space_on_both_sides() {
+        assert_suggestion_result("foo 、 bar", CommaFixes, "foo, bar")
+    }
+
+    #[test]
+    fn doesnt_correct_comma_between_non_english_tokens() {
+        assert_lint_count("严禁采摘花、 果、叶，挖掘树根、草药!", CommaFixes, 0);
+    }
+}

harper-core/src/linting/lint_group.rs

@@ -18,6 +18,7 @@ use super::back_in_the_day::BackInTheDay;
 use super::boring_words::BoringWords;
 use super::capitalize_personal_pronouns::CapitalizePersonalPronouns;
 use super::chock_full::ChockFull;
+use super::comma_fixes::CommaFixes;
 use super::compound_nouns::CompoundNouns;
 use super::confident::Confident;
 use super::correct_number_suffix::CorrectNumberSuffix;
@@ -327,6 +328,7 @@ impl LintGroup {
         insert_struct_rule!(AvoidCurses, true);
         insert_pattern_rule!(TerminatingConjunctions, true);
         insert_struct_rule!(EllipsisLength, true);
+        insert_struct_rule!(CommaFixes, true);
         insert_pattern_rule!(DotInitialisms, true);
         insert_pattern_rule!(BoringWords, false);
         insert_pattern_rule!(UseGenitive, false);

harper-core/src/linting/lint_kind.rs

@@ -19,6 +19,7 @@ pub enum LintKind {
     WordChoice,
     #[default]
     Miscellaneous,
+    Punctuation,
 }
 
 impl LintKind {
@@ -34,6 +35,7 @@ impl LintKind {
             LintKind::Enhancement => "Enhancement",
             LintKind::WordChoice => "WordChoice",
             LintKind::Style => "Style",
+            LintKind::Punctuation => "Punctuation",
         }
         .to_owned()
     }
@@ -51,6 +53,7 @@ impl Display for LintKind {
             LintKind::Enhancement => "Enhancement",
             LintKind::WordChoice => "Word Choice",
             LintKind::Style => "Style",
+            LintKind::Punctuation => "Punctuation",
         };
 
         write!(f, "{}", s)

harper-core/src/linting/mod.rs

@@ -10,6 +10,7 @@ mod boring_words;
 mod capitalize_personal_pronouns;
 mod chock_full;
 mod closed_compounds;
+mod comma_fixes;
 mod compound_nouns;
 mod confident;
 mod correct_number_suffix;
@@ -75,6 +76,7 @@ pub use back_in_the_day::BackInTheDay;
 pub use boring_words::BoringWords;
 pub use capitalize_personal_pronouns::CapitalizePersonalPronouns;
 pub use chock_full::ChockFull;
+pub use comma_fixes::CommaFixes;
 pub use compound_nouns::CompoundNouns;
 pub use confident::Confident;
 pub use correct_number_suffix::CorrectNumberSuffix;

harper-core/src/parsers/isolate_english.rs

@@ -25,7 +25,7 @@ impl<D: Dictionary> Parser for IsolateEnglish<D> {
         let mut english_tokens: Vec<Token> = Vec::with_capacity(tokens.len());
 
         for chunk in tokens.iter_chunks() {
-            if chunk.len() < 5 || is_likely_english(chunk, source, &self.dict) {
+            if chunk.len() < 4 || is_likely_english(chunk, source, &self.dict) {
                 english_tokens.extend_from_slice(chunk);
             }
         }

harper-core/src/punctuation.rs

@@ -97,6 +97,8 @@ impl Punctuation {
             ':' => Punctuation::Colon,
             ';' => Punctuation::Semicolon,
             ',' => Punctuation::Comma,
+            '、' => Punctuation::Comma,
+            '，' => Punctuation::Comma,
             '-' => Punctuation::Hyphen,
             '[' => Punctuation::OpenSquare,
             ']' => Punctuation::CloseSquare,

packages/web/src/lib/lintKindColor.ts

@@ -18,6 +18,8 @@ export default function lintKindColor(lintKindKey: string): string {
 			return '#0EAD69';
 		case 'Miscellaneous':
 			return '#3BCEAC';
+		case 'Punctuation':
+			return '#D4850F';
 		default:
 			throw new Error(`Unexpected lint kind: ${lintKindKey}`);
 	}