Abstract regex and add fancy_regex backend

.gitignore

@@ -3,6 +3,7 @@
 
 .vim
 .env
+.venv
 target
 .idea
 Cargo.lock

tokenizers/Cargo.toml

@@ -36,7 +36,6 @@ harness = false
 [dependencies]
 lazy_static = "1.4"
 rand = "0.7"
-onig = { version = "6.0", default-features = false }
 regex = "1.3"
 regex-syntax = "0.6"
 rayon = "1.3"
@@ -59,11 +58,18 @@ cached-path = { version = "0.5", optional = true }
 aho-corasick = "0.7"
 paste = "1.0.6"
 proc_macros = { path = "./src/utils/proc_macros" }
+once_cell = "1.8"
+cfg-if = "1"
+onig = { version = "6.0", default-features = false, optional = true }
+fancy-regex = { version = "0.7", optional = true }
 
 [features]
-default = ["progressbar", "http"]
+default = ["progressbar", "http", "regex-onig"]
 progressbar = ["indicatif"]
 http = ["reqwest", "cached-path"]
+regex-fancy = ["fancy-regex"]
+regex-onig = ["onig"]
+regex-all-test = ["regex-onig", "regex-fancy"]
 
 [dev-dependencies]
 criterion = "0.3"

tokenizers/src/normalizers/replace.rs

@@ -1,5 +1,5 @@
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
-use onig::Regex;
+use crate::utils::regex::Regex;
 use serde::{Deserialize, Serialize};
 
 /// Represents the different patterns that `Replace` can use
@@ -65,8 +65,8 @@ impl Replace {
     pub fn new<I: Into<ReplacePattern>, C: Into<String>>(pattern: I, content: C) -> Result<Self> {
         let pattern: ReplacePattern = pattern.into();
         let regex = match &pattern {
-            ReplacePattern::String(s) => Regex::new(&regex::escape(s))?,
-            ReplacePattern::Regex(r) => Regex::new(r)?,
+            ReplacePattern::String(s) => Regex::new(regex::escape(s)),
+            ReplacePattern::Regex(r) => Regex::new(r.to_owned()),
         };
 
         Ok(Self {

tokenizers/src/pre_tokenizers/byte_level.rs

@@ -1,6 +1,6 @@
 use std::collections::{HashMap, HashSet};
 
-use onig::Regex;
+use crate::utils::regex::Regex;
 use serde::{Deserialize, Serialize};
 
 use crate::tokenizer::{
@@ -34,8 +34,7 @@ fn bytes_char() -> HashMap<u8, char> {
 
 lazy_static! {
     static ref RE: Regex =
-        Regex::new(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+")
-            .unwrap();
+        Regex::new(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+".to_string());
     static ref BYTES_CHAR: HashMap<u8, char> = bytes_char();
     static ref CHAR_BYTES: HashMap<char, u8> =
         bytes_char().into_iter().map(|(c, b)| (b, c)).collect();

tokenizers/src/pre_tokenizers/split.rs

@@ -1,4 +1,4 @@
-use onig::Regex;
+use crate::utils::regex::Regex;
 use serde::{Deserialize, Deserializer, Serialize};
 
 use crate::tokenizer::{
@@ -80,8 +80,8 @@ impl Split {
     ) -> Result<Self> {
         let pattern: SplitPattern = pattern.into();
         let regex = match &pattern {
-            SplitPattern::String(s) => Regex::new(&regex::escape(s))?,
-            SplitPattern::Regex(r) => Regex::new(r)?,
+            SplitPattern::String(s) => Regex::new(regex::escape(s)),
+            SplitPattern::Regex(r) => Regex::new(r.to_owned()),
         };
 
         Ok(Self {

tokenizers/src/tokenizer/pattern.rs

@@ -59,20 +59,20 @@ impl Pattern for &Regex {
     }
 }
 
-impl Pattern for &onig::Regex {
+impl Pattern for &crate::utils::regex::Regex {
     fn find_matches(&self, inside: &str) -> Result<Vec<(Offsets, bool)>> {
         if inside.is_empty() {
             return Ok(vec![((0, 0), false)]);
         }
 
         let mut prev = 0;
         let mut splits = Vec::with_capacity(inside.len());
-        for (start, end) in self.find_iter(inside) {
-            if prev != start {
-                splits.push(((prev, start), false));
+        for m in self.find_iter(inside) {
+            if prev != m.start() {
+                splits.push(((prev, m.start()), false));
             }
-            splits.push(((start, end), true));
-            prev = end;
+            splits.push(((m.start(), m.end()), true));
+            prev = m.end();
         }
         if prev != inside.len() {
             splits.push(((prev, inside.len()), false))
@@ -205,8 +205,8 @@ mod tests {
     }
 
     #[test]
-    fn onig_regex() {
-        let is_whitespace = onig::Regex::new(r"\s+").unwrap();
+    fn abstract_regex() {
+        let is_whitespace = crate::utils::regex::Regex::new(r"\s+".to_string());
         do_test!("a   b", &is_whitespace => vec![((0, 1), false), ((1, 4), true), ((4, 5), false)]);
         do_test!("   a   b   ", &is_whitespace =>
             vec![((0, 3), true), ((3, 4), false), ((4, 7), true), ((7, 8), false), ((8, 11), true)]

tokenizers/src/utils/mod.rs

@@ -6,6 +6,7 @@ pub mod padding;
 pub mod parallelism;
 pub(crate) mod progress;
 pub mod truncation;
+pub mod regex;
 
 use serde::{Serialize, Serializer};
 use std::collections::{BTreeMap, HashMap};