diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py
index 7a7da29040..f0230a7ee7 100644
--- a/optimum/intel/openvino/quantization.py
+++ b/optimum/intel/openvino/quantization.py
@@ -260,7 +260,8 @@ def _quantize_ovcausallm(
         save_directory.mkdir(parents=True, exist_ok=True)
 
         if weights_only:
-            self.model.model = nncf.compress_weights(self.model.model)
+            model = nncf.compress_weights(self.model._original_model)
+            self.model.model = model
             self.model.save_pretrained(save_directory)
             return