diff --git a/setup.py b/setup.py
index 7946a0c8c5..00c1c63486 100644
--- a/setup.py
+++ b/setup.py
@@ -42,10 +42,11 @@
         "onnx",
         "onnxruntime<1.15.0",
         "transformers>=4.33.0",
+        "intel-extension-for-pytorch",
     ],
     "openvino": ["openvino>=2023.1.0", "onnx", "onnxruntime", "transformers>=4.33.0"],
     "nncf": ["nncf>=2.6.0"],
-    "ipex": ["transformers", "intel-extension-for-pytorch", "onnx"],
+    "ipex": ["transformers<4.32.0", "intel-extension-for-pytorch", "onnx"],
     "diffusers": ["diffusers"],
     "quality": QUALITY_REQUIRE,
     "tests": TESTS_REQUIRE,