Merge branch 'development' of https://github.com/vtuber-plan/langport …

…into development
vtuber-plan · Jun 29, 2024 · d8d54e7 · d8d54e7
2 parents 69f6e12 + 101efc3
commit d8d54e7
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 8 deletions.
diff --git a/langport/model/adapters/qwen.py b/langport/model/adapters/qwen.py
@@ -5,7 +5,7 @@
 
 
 class QwenAdapter(BaseAdapter):
-    """The model adapter for Robin"""
+    """The model adapter for Qwen"""
 
     def match(self, model_path: str):
         if model_path.endswith('/'):

diff --git a/langport/model/executor/huggingface.py b/langport/model/executor/huggingface.py
@@ -280,13 +280,17 @@ def load_model(
                         )
                 else:
                     if "8" in quantization:
-                        model, tokenizer = load_compress_model(
-                            model_path=model_path, device=device, compression_config=default_compression_config, **kwargs
-                        )
+                        # model, tokenizer = load_compress_model(
+                        #     model_path=model_path, device=device, compression_config=default_compression_config, **kwargs
+                        # )
+                        kwargs["load_in_8bit"] = True
+                        model, tokenizer = self._load_hf_model(adapter, model_path, kwargs)
                     elif "4" in quantization:
-                        model, tokenizer = load_compress_model(
-                            model_path=model_path, device=device, compression_config=bit4_compression_config, **kwargs
-                        )
+                        # model, tokenizer = load_compress_model(
+                        #     model_path=model_path, device=device, compression_config=bit4_compression_config, **kwargs
+                        # )
+                        kwargs["load_in_4bit"] = True
+                        model, tokenizer = self._load_hf_model(adapter, model_path, kwargs)
                     else:
                         model, tokenizer = load_compress_model(
                             model_path=model_path, device=device, compression_config=default_compression_config, **kwargs
@@ -318,7 +322,7 @@ def load_model(
             ds_engine = deepspeed.init_inference(model=model, config=config)
             model = ds_engine.module
         else:
-            if (device == "cuda" and num_gpus == 1 and not cpu_offloading) or device == "mps":
+            if (device == "cuda" and num_gpus == 1 and not cpu_offloading and quantization is None) or device == "mps":
                 model.to(device)
 
         if debug: