src/SequenceTokenizers.jl: Handle indexes out of range which can occu…

…r due to zero masking for padding.
mashu · Aug 31, 2024 · 3a56dfc · 3a56dfc · mashu · Aug 31, 2024
1 parent 0ac7304
commit 3a56dfc
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "SequenceTokenizers"
 uuid = "782fbe24-d740-44eb-8e9f-f0831b1cb5e5"
 authors = ["Mateusz Kaduk <[email protected]>"]
-version = "0.0.4"
+version = "0.0.5"
 
 [deps]
 OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f"

diff --git a/src/SequenceTokenizers.jl b/src/SequenceTokenizers.jl
@@ -178,8 +178,14 @@ module SequenceTokenizers
     println(tokenizer(1))  # Output: 'x' (unknown token)
     ```
     """
-    @inline (tokenizer::SequenceTokenizer)(idx::Integer) = tokenizer.alphabet[idx]
-
+    @inline function (tokenizer::SequenceTokenizer)(idx::Integer)
+        if 1 <= idx <= length(tokenizer.alphabet)
+            return tokenizer.alphabet[idx]
+        else
+            return tokenizer.unksym
+        end
+    end
+
     """
         (tokenizer::SequenceTokenizer{T})(input::AbstractString) where T