fix embed_out

Signed-off-by: changwangss <[email protected]>
intel · Aug 5, 2024 · a08344a · a08344a
1 parent 0bc8428
commit a08344a
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 3 deletions.
diff --git a/intel_extension_for_transformers/transformers/modeling/modeling_auto.py b/intel_extension_for_transformers/transformers/modeling/modeling_auto.py
@@ -161,7 +161,9 @@ def build_woq_model(model, quantization_config):
     from neural_compressor.adaptor.torch_utils.util import set_module
     weight_dtype = quantization_config.weight_dtype
     for n, m in model.named_modules():
+        print(n)
         if n in quantization_config.llm_int8_skip_modules:
+           # import pdb;pdb.set_trace();
             continue
         if isinstance(m, torch.nn.Linear):
             zp = getattr(

diff --git a/intel_extension_for_transformers/transformers/utils/config.py b/intel_extension_for_transformers/transformers/utils/config.py
@@ -832,9 +832,9 @@ def __init__(
         self.double_quant_use_sym = double_quant_use_sym
         self.double_quant_group_size = double_quant_group_size
         # "transformer.output_layer" for chatglm series model.
-        # "gpt_neox.embed_out" for dolly v2 series model.
-        self.llm_int8_skip_modules = kwargs.get("llm_int8_skip_modules",
-                                                ["lm_head", "transformer.output_layer", "gpt_neox.embed_out"])
+        # "embed_out" for dolly v2 series model.
+        self.llm_int8_skip_modules = kwargs.get("llm_int8_skip_modules", 
+                                                ["lm_head", "transformer.output_layer", "embed_out"])
         self.use_ggml = use_ggml
         self.use_quant = use_quant
         self.use_neural_speed = use_neural_speed