Update to tokenizers 0.19 (#57)

CHANGELOG.md

@@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support for regular expressions to split pre-tokenizer. See
   `Tokenizers.PreTokenizer.split_regex/3`.
 
+### Removed
+
+- **(Breaking)** `:add_prefix_space` option in favour of `:prepend_scheme` for metaspace
+  decoder and pre-tokenizer
+
 ## [v0.4.0] - 2023-08-09
 
 ### Added

lib/tokenizers/decoder.ex

@@ -74,8 +74,11 @@ defmodule Tokenizers.Decoder do
     * `replacement` - the replacement character. Defaults to `▁`
       (as char)
 
-    * `add_prefix_space` - whether to add a space to the first word.
-      Defaults to `true`
+    * `:prepend_scheme` - whether to add a space to the first word if there
+      isn't already one. This lets us treat "hello" exactly like "say hello".
+      Either of `:always`, `:never`, `:first`. `:first` means the space is
+      only added on the first token (relevant when special tokens are used
+      or other pre_tokenizer are used). Defaults to `:always`
 
   """
   @spec metaspace(keyword()) :: t()

lib/tokenizers/pre_tokenizer.ex

@@ -103,9 +103,11 @@ defmodule Tokenizers.PreTokenizer do
 
     * `:replacement` - the replacement character to use. Defaults to `"▁"`
 
-    * `:add_prefix_space` - whether to add a space to the first word
-      if there isn’t already one. This lets us treat hello exactly
-      like say hello. Defaults to `true`
+    * `:prepend_scheme` - whether to add a space to the first word if there
+      isn't already one. This lets us treat "hello" exactly like "say hello".
+      Either of `:always`, `:never`, `:first`. `:first` means the space is
+      only added on the first token (relevant when special tokens are used
+      or other pre_tokenizer are used). Defaults to `:always`
 
   """
   @spec metaspace(keyword()) :: t()

native/ex_tokenizers/Cargo.toml

@@ -13,5 +13,5 @@ crate-type = ["cdylib"]
 anyhow = "1"
 rustler = "0.29.1"
 thiserror = "1"
-tokenizers = { version = "0.15.0", default-features = false, features = ["onig", "esaxx_fast"]}
+tokenizers = { version = "0.19.1", default-features = false, features = ["onig", "esaxx_fast"]}
 serde = { version = "1.0", features = [ "rc", "derive" ] }