From cd8b7b06b8c173abf57b9b9f5ab1b1e2d39e6ade Mon Sep 17 00:00:00 2001 From: Nikita Malinin Date: Mon, 9 Dec 2024 11:20:29 +0100 Subject: [PATCH 1/2] Update Baichuan2 models default config --- optimum/intel/openvino/configuration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/optimum/intel/openvino/configuration.py b/optimum/intel/openvino/configuration.py index b34cd84cd0..d524c19094 100644 --- a/optimum/intel/openvino/configuration.py +++ b/optimum/intel/openvino/configuration.py @@ -126,8 +126,15 @@ class OVQuantizationMethod(str, Enum): "sym": True, "group_size": 128, "ratio": 0.8, + }, + "baichuan-inc/Baichuan2-13B-Chat": { + "bits": 4, + "sym": True, + "group_size": 128, + "ratio": 1.0, "dataset": "wikitext2", "quant_method": OVQuantizationMethod.AWQ, + "scale_estimation": True, }, "lmsys/longchat-7b-16k": { "bits": 4, From 12afa6b3ae97caa63c40a05945a5664964fcf541 Mon Sep 17 00:00:00 2001 From: Nikita Malinin Date: Mon, 9 Dec 2024 11:23:51 +0100 Subject: [PATCH 2/2] Update configuration.py --- optimum/intel/openvino/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optimum/intel/openvino/configuration.py b/optimum/intel/openvino/configuration.py index d524c19094..f51ec2932a 100644 --- a/optimum/intel/openvino/configuration.py +++ b/optimum/intel/openvino/configuration.py @@ -123,13 +123,13 @@ class OVQuantizationMethod(str, Enum): "mistralai/Mistral-7B-v0.1": {"bits": 4, "sym": True, "group_size": 128, "ratio": 0.9}, "baichuan-inc/Baichuan2-7B-Chat": { "bits": 4, - "sym": True, + "sym": False, "group_size": 128, "ratio": 0.8, }, "baichuan-inc/Baichuan2-13B-Chat": { "bits": 4, - "sym": True, + "sym": False, "group_size": 128, "ratio": 1.0, "dataset": "wikitext2",