Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature] Allow unconstrained whitespace in JSON schema #123

Merged
merged 3 commits into from
Dec 7, 2024
Merged

[Feature] Allow unconstrained whitespace in JSON schema #123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cf12027
first checkpoint
Ubospica Dec 7, 2024
76439ca
tmp
Ubospica Dec 7, 2024
59410a4
finish
Ubospica Dec 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions cpp/compiler.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ class GrammarCompiler::Impl {

CompiledGrammar CompileJSONSchema(
const std::string& schema,
bool any_whitespace,
std::optional<int> indent,
std::optional<std::pair<std::string, std::string>> separators,
bool strict_mode = true
Expand All @@ -342,16 +343,17 @@ class GrammarCompiler::Impl {
private:
/*! \brief The cache for the compiled grammar of a JSON schema. */
using SchemaKey =
std::tuple<std::string, std::optional<int>, std::pair<std::string, std::string>, bool>;
std::tuple<std::string, bool, std::optional<int>, std::pair<std::string, std::string>, bool>;

std::function<CompiledGrammar(const SchemaKey&)> GetCompileJSONSchemaCacheFunc(bool cache_enabled
) {
if (!cache_enabled) {
return nullptr;
}
return [&](const SchemaKey& key) {
auto [schema, indent, separators, strict_mode] = key;
auto grammar = Grammar::FromJSONSchema(schema, indent, separators, strict_mode);
auto [schema, any_whitespace, indent, separators, strict_mode] = key;
auto grammar =
Grammar::FromJSONSchema(schema, any_whitespace, indent, separators, strict_mode);
return MultiThreadCompileGrammar(grammar, tokenizer_info_, max_threads_);
};
}
Expand Down Expand Up @@ -404,21 +406,22 @@ CompiledGrammar GrammarCompiler::Impl::CompileBuiltinJSONGrammar() {

CompiledGrammar GrammarCompiler::Impl::CompileJSONSchema(
const std::string& schema,
bool any_whitespace,
std::optional<int> indent,
std::optional<std::pair<std::string, std::string>> separators,
bool strict_mode
) {
if (!cache_enabled_) {
return MultiThreadCompileGrammar(
Grammar::FromJSONSchema(schema, indent, separators, strict_mode),
Grammar::FromJSONSchema(schema, any_whitespace, indent, separators, strict_mode),
tokenizer_info_,
max_threads_
);
}
auto separators_value = separators.value_or(
(indent == std::nullopt) ? std::make_pair(", ", ": ") : std::make_pair(",", ": ")
);
auto key = std::make_tuple(schema, indent, separators_value, strict_mode);
auto key = std::make_tuple(schema, any_whitespace, indent, separators_value, strict_mode);
return compile_json_schema_cache_.Get(key);
}

Expand All @@ -444,11 +447,12 @@ GrammarCompiler::GrammarCompiler(

CompiledGrammar GrammarCompiler::CompileJSONSchema(
const std::string& schema,
bool any_whitespace,
std::optional<int> indent,
std::optional<std::pair<std::string, std::string>> separators,
bool strict_mode
) {
return pimpl_->CompileJSONSchema(schema, indent, separators, strict_mode);
return pimpl_->CompileJSONSchema(schema, any_whitespace, indent, separators, strict_mode);
}

CompiledGrammar GrammarCompiler::CompileBuiltinJSONGrammar() {
Expand Down
3 changes: 2 additions & 1 deletion cpp/grammar.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@ Grammar Grammar::FromEBNF(const std::string& ebnf_string, const std::string& roo

Grammar Grammar::FromJSONSchema(
const std::string& schema,
bool any_whitespace,
std::optional<int> indent,
std::optional<std::pair<std::string, std::string>> separators,
bool strict_mode
) {
auto ebnf_string = JSONSchemaToEBNF(schema, indent, separators, strict_mode);
auto ebnf_string = JSONSchemaToEBNF(schema, any_whitespace, indent, separators, strict_mode);
return FromEBNF(ebnf_string);
}

Expand Down
Loading
Loading