From b1f8d1d6957c12b403a04db1ebd58f6668a4f951 Mon Sep 17 00:00:00 2001
From: Ekaterina Aidova <ekaterina.aidova@intel.com>
Date: Thu, 24 Oct 2024 11:50:11 +0400
Subject: [PATCH] disable warning about tokenizers version for ov tokenizers >=
 2024.5 (#962)

* disable warning about tokenizers version for ov tokenizers >= 2024.5

* increase verbosity for tokenizers test
---
 optimum/exporters/openvino/__main__.py |  4 +++-
 optimum/exporters/openvino/convert.py  |  5 +++--
 optimum/intel/utils/import_utils.py    | 18 ++++++++++++++++++
 tests/openvino/test_exporters_cli.py   |  2 +-
 4 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/optimum/exporters/openvino/__main__.py b/optimum/exporters/openvino/__main__.py
index 69cfec1d9..412ed21f6 100644
--- a/optimum/exporters/openvino/__main__.py
+++ b/optimum/exporters/openvino/__main__.py
@@ -55,6 +55,9 @@
 
 logger = logging.getLogger(__name__)
 
+# init core before import openvino tokenizers to prevent failed attempt loading extension
+core = Core()
+
 
 def infer_task(
     task,
@@ -413,7 +416,6 @@ class StoreAttr(object):
     del model
     gc.collect()
 
-    core = Core()
     for submodel_path in submodel_paths:
         submodel_path = Path(output) / submodel_path
         submodel = core.read_model(submodel_path)
diff --git a/optimum/exporters/openvino/convert.py b/optimum/exporters/openvino/convert.py
index 4e6503b5b..e731cd180 100644
--- a/optimum/exporters/openvino/convert.py
+++ b/optimum/exporters/openvino/convert.py
@@ -41,6 +41,7 @@
     _torch_version,
     _transformers_version,
     compare_versions,
+    is_openvino_tokenizers_version,
     is_tokenizers_version,
     is_transformers_version,
 )
@@ -734,9 +735,9 @@ def export_tokenizer(
     except ModuleNotFoundError:
         return
 
-    if is_tokenizers_version(">", "0.19"):
+    if is_tokenizers_version(">", "0.19") and is_openvino_tokenizers_version("<", "2024.5.0.0"):
         logger.warning(
-            "Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19. "
+            "Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19 and openvino version <= 2024.4. "
             "Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO."
         )
 
diff --git a/optimum/intel/utils/import_utils.py b/optimum/intel/utils/import_utils.py
index 60d20361e..6fa6c590b 100644
--- a/optimum/intel/utils/import_utils.py
+++ b/optimum/intel/utils/import_utils.py
@@ -382,6 +382,24 @@ def is_openvino_version(operation: str, version: str):
     return compare_versions(parse(_openvino_version), operation, version)
 
 
+def is_openvino_tokenizers_version(operation: str, version: str):
+    if not is_openvino_available():
+        return False
+    if not is_openvino_tokenizers_available():
+        return False
+    import openvino_tokenizers
+
+    tokenizers_version = openvino_tokenizers.__version__
+
+    if tokenizers_version == "0.0.0.0":
+        try:
+            tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version
+        except importlib_metadata.PackageNotFoundError:
+            pass
+
+    return compare_versions(parse(tokenizers_version), operation, version)
+
+
 def is_diffusers_version(operation: str, version: str):
     """
     Compare the current diffusers version to a given reference with an operation.
diff --git a/tests/openvino/test_exporters_cli.py b/tests/openvino/test_exporters_cli.py
index 8443f95b3..cea6c94fc 100644
--- a/tests/openvino/test_exporters_cli.py
+++ b/tests/openvino/test_exporters_cli.py
@@ -153,7 +153,7 @@ def test_exporters_cli(self, task: str, model_type: str):
     def test_exporters_cli_tokenizers(self, task: str, model_type: str):
         with TemporaryDirectory() as tmpdir:
             output = subprocess.check_output(
-                f"optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
+                f"TRANSFORMERS_VERBOSITY=debug optimum-cli export openvino --model {MODEL_NAMES[model_type]} --task {task} {tmpdir}",
                 shell=True,
                 stderr=subprocess.STDOUT,
             ).decode()