fix streaming issue (opea-project#81)

Signed-off-by: letonghan <[email protected]>
poussa · May 22, 2024 · 1cf449b · 1cf449b
1 parent 7d823cf
commit 1cf449b
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/comps/cores/mega/orchestrator.py b/comps/cores/mega/orchestrator.py
@@ -73,7 +73,7 @@ async def execute(self, cur_node: str, inputs: Dict):
 
             def generate():
                 if response:
-                    for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
+                    for chunk in response.iter_content(chunk_size=None):
                         if chunk:
                             yield chunk
 

diff --git a/comps/llms/langchain/llm_tgi.py b/comps/llms/langchain/llm_tgi.py
@@ -55,9 +55,9 @@ def llm_generate(input: LLMParamsDoc):
 
     if input.streaming:
 
-        def stream_generator():
+        async def stream_generator():
             chat_response = ""
-            for text in llm.stream(input.query):
+            async for text in llm.astream(input.query):
                 chat_response += text
                 processed_text = post_process_text(text)
                 if text and processed_text: