vllm 后端支持 guided_decoding,

shell-nlp · Nov 18, 2024 · 32ba876 · 32ba876
1 parent 214a291
commit 32ba876
Showing 1 changed file with 16 additions and 20 deletions.
diff --git a/gpt_server/model_backend/vllm_backend.py b/gpt_server/model_backend/vllm_backend.py
@@ -111,27 +111,23 @@ async def stream_chat(self, params: Dict[str, Any]) -> AsyncGenerator:
         guided_decoding = None
         guided_json = None
         # ---- 支持 response_format,但是官方对BPE分词器的支持仍然太差 ----
-        # if response_format is not None:
-        #     if response_format["type"] == "json_object":
-        #         guided_json_object = True
-        #     if response_format["type"] == "json_schema":
-        #         json_schema = response_format["json_schema"]
-        #         assert json_schema is not None
-        #         guided_json = json_schema["schema"]
+        if response_format is not None:
+            if response_format["type"] == "json_object":
+                guided_json_object = True
+            if response_format["type"] == "json_schema":
+                json_schema = response_format["json_schema"]
+                assert json_schema is not None
+                guided_json = json_schema["schema"]
 
-        #     guided_decoding = GuidedDecodingParams.from_optional(
-        #         json=guided_json,
-        #         regex=None,
-        #         choice=None,
-        #         grammar=None,
-        #         json_object=guided_json_object,
-        #         backend=(
-        #             self.engine_args.guided_decoding_backend
-        #             if self.engine_args.guided_decoding_backend
-        #             else "lm-format-enforcer"
-        #         ),
-        #         whitespace_pattern=None,
-        #     )
+            guided_decoding = GuidedDecodingParams.from_optional(
+                json=guided_json,
+                regex=None,
+                choice=None,
+                grammar=None,
+                json_object=guided_json_object,
+                backend="lm-format-enforcer",
+                whitespace_pattern=None,
+            )
         # ---- 支持 response_format,但是官方对BPE分词器的支持仍然太差 ----
         sampling = SamplingParams(
             top_p=top_p,