We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent c8bff72 commit f38814cCopy full SHA for f38814c
src/pre_tokenizer.cpp
@@ -78,7 +78,8 @@ PreTokenizerConfig& PreTokenizerConfig::parse_json(const json& json_config) {
78
type = json_config.at("type");
79
if (type == "Split") {
80
try {
81
- pattern = json_config.at("pattern");
+ pattern = json_config.at("pattern").at("Regex");
82
+ std::cout << "pattern: " << pattern.value() << std::endl;
83
} catch (json::out_of_range&) {
84
}
85
} else if (type == "Digits") {
0 commit comments