update func export_codec_vllm

FunAudioLLM · Feb 26, 2025 · f280558 · f280558
1 parent f6a18ee
commit f280558
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 2 deletions.
diff --git a/cosyvoice/cli/cosyvoice.py b/cosyvoice/cli/cosyvoice.py
@@ -156,7 +156,7 @@ def __init__(self, model_dir, load_jit=False, load_trt=False, fp16=False, use_vl
             self.model.export_codec_vllm(''.join([model_dir, '/codec_vllm_model']))
             engine_args = EngineArgs(model=''.join([model_dir, '/codec_vllm_model']),
                          skip_tokenizer_init=True,
-                         gpu_memory_utilization=0.1)
+                         gpu_memory_utilization=0.2)
             self.vllm_codec_engine = LLMEngine.from_engine_args(engine_args)
             self.model.vllm_codec_engine = self.vllm_codec_engine
 

diff --git a/cosyvoice/cli/model.py b/cosyvoice/cli/model.py
@@ -347,6 +347,9 @@ def export_codec_vllm(self, model_path):
         self.llm.llm.model.to(dtype)
         tmp_vocab_size = self.llm.llm.model.config.vocab_size
         tmp_tie_embedding = self.llm.llm.model.config.tie_word_embeddings
+        del self.llm.llm.model.generation_config.eos_token_id
+        del self.llm.llm.model.config.bos_token_id
+        del self.llm.llm.model.config.eos_token_id
         self.llm.llm.model.config.vocab_size = pad_vocab_size
         self.llm.llm.model.config.tie_word_embeddings = False
         self.llm.llm.model.config.use_bias = True

diff --git a/cosyvoice/llm/llm.py b/cosyvoice/llm/llm.py
@@ -343,7 +343,7 @@ def inference(
                                             max_tokens=max_len)
             request_id = uuid.uuid4()
             vllm_codec_engine.add_request(request_id,
-                                        {"prompt_embeds": lm_input.to(torch.bfloat16).to(device)},
+                                        {"prompt_embeds": lm_input.squeeze(0).to(torch.bfloat16).to(device)},
                                         sampling_params)
             ## generator
             out_token_ids = []