Fix missing sequences to ignore when splitting tokens

26B · Jan 11, 2023 · e274b96 · e274b96
1 parent 74a503f
commit e274b96
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/TextUtilities.php b/src/TextUtilities.php
@@ -184,7 +184,7 @@ private function splitTokens(string $line, int $maxLength): array
         $tokens = explode(" ", $line);
         $newTokens = [];
         foreach ($tokens as $token) {
-            if (mb_strlen($token) > $maxLength) {
+            if (mb_strlen(str_replace($this->config->getSequencesToIgnore(), '', $token)) > $maxLength) {
                 $tokensWithMaxLength = str_split($token, $maxLength);
                 $newTokens = array_merge($newTokens, $tokensWithMaxLength);
             } else {