From e7c15c88cf192246d7253acc661b94177e201462 Mon Sep 17 00:00:00 2001 From: Helena Date: Tue, 9 Jan 2024 12:55:54 +0000 Subject: [PATCH 1/3] f32 precision for compare-with-transformers tests Allow these tests to pass locally on devices where inference is run by default in FP16 (GPU) or BF16 (latest Xeon). --- tests/openvino/test_modeling.py | 66 +++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 0c7f9a8247..0f5de4eb42 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -25,6 +25,7 @@ import torch from datasets import load_dataset from evaluate import evaluator +from openvino.runtime import get_version from parameterized import parameterized from PIL import Image from transformers import ( @@ -125,7 +126,10 @@ def test_load_from_hub_and_save_model(self): loaded_model = OVModelForSequenceClassification.from_pretrained(self.OV_MODEL_ID, ov_config=ov_config) self.assertTrue(manual_openvino_cache_dir.is_dir()) self.assertGreaterEqual(len(list(manual_openvino_cache_dir.glob("*.blob"))), 1) - self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT").name, "THROUGHPUT") + if get_version() < "2023.3": + self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT").name, "THROUGHPUT") + else: + self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT"), "THROUGHPUT") with tempfile.TemporaryDirectory() as tmpdirname: loaded_model.save_pretrained(tmpdirname) @@ -247,7 +251,9 @@ class OVModelForSequenceClassificationIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForSequenceClassification.from_pretrained(model_id, export=True) + ov_model = OVModelForSequenceClassification.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForSequenceClassification.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -313,7 +319,9 @@ class OVModelForQuestionAnsweringIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForQuestionAnswering.from_pretrained(model_id, export=True) + ov_model = OVModelForQuestionAnswering.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForQuestionAnswering.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -386,7 +394,9 @@ class OVModelForTokenClassificationIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForTokenClassification.from_pretrained(model_id, export=True) + ov_model = OVModelForTokenClassification.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForTokenClassification.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -430,7 +440,9 @@ class OVModelForFeatureExtractionIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForFeatureExtraction.from_pretrained(model_id, export=True) + ov_model = OVModelForFeatureExtraction.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModel.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -492,7 +504,9 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForCausalLM.from_pretrained(model_id, export=True) + ov_model = OVModelForCausalLM.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForCausalLM.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -637,7 +651,9 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForMaskedLM.from_pretrained(model_id, export=True) + ov_model = OVModelForMaskedLM.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForMaskedLM.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -693,7 +709,9 @@ class OVModelForImageClassificationIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True) + ov_model = OVModelForImageClassification.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForImageClassification.from_pretrained(model_id) preprocessor = AutoFeatureExtractor.from_pretrained(model_id) @@ -729,7 +747,9 @@ def test_pipeline(self, model_arch): @parameterized.expand(TIMM_MODELS) def test_compare_to_timm(self, model_id): - ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True) + ov_model = OVModelForImageClassification.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) timm_model = timm.create_model(model_id, pretrained=True) preprocessor = TimmImageProcessor.from_pretrained(model_id) @@ -781,7 +801,9 @@ class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True) + ov_model = OVModelForSeq2SeqLM.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.encoder, OVEncoder) self.assertIsInstance(ov_model.decoder, OVDecoder) @@ -920,7 +942,9 @@ def _generate_random_audio_data(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForAudioClassification.from_pretrained(model_id, export=True) + ov_model = OVModelForAudioClassification.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForAudioClassification.from_pretrained(model_id) preprocessor = AutoFeatureExtractor.from_pretrained(model_id) @@ -985,7 +1009,9 @@ def test_load_vanilla_transformers_which_is_not_supported(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForCTC.from_pretrained(model_id, export=True) + ov_model = OVModelForCTC.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) set_seed(SEED) @@ -1037,7 +1063,9 @@ def test_load_vanilla_transformers_which_is_not_supported(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForAudioXVector.from_pretrained(model_id, export=True) + ov_model = OVModelForAudioXVector.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) set_seed(SEED) @@ -1091,7 +1119,9 @@ def test_load_vanilla_transformers_which_is_not_supported(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForAudioFrameClassification.from_pretrained(model_id, export=True) + ov_model = OVModelForAudioFrameClassification.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) set_seed(SEED) @@ -1134,7 +1164,9 @@ class OVModelForPix2StructIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForPix2Struct.from_pretrained(model_id, export=True) + ov_model = OVModelForPix2Struct.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.encoder, OVEncoder) self.assertIsInstance(ov_model.decoder, OVDecoder) @@ -1223,7 +1255,9 @@ def _generate_random_audio_data(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True) + ov_model = OVModelForSpeechSeq2Seq.from_pretrained( + model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + ) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id) processor = get_preprocessor(model_id) From 4f9ea99401cad66515551c4a15fc5b021c44c0b5 Mon Sep 17 00:00:00 2001 From: Helena Date: Tue, 9 Jan 2024 15:52:54 +0000 Subject: [PATCH 2/3] Add F32_CONFIG constant for modeling tests --- tests/openvino/test_modeling.py | 62 +++++++++------------------------ 1 file changed, 17 insertions(+), 45 deletions(-) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index 0f5de4eb42..f44849ba91 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -90,6 +90,8 @@ SEED = 42 +F32_CONFIG = {"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} + class Timer(object): def __enter__(self): @@ -251,9 +253,7 @@ class OVModelForSequenceClassificationIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForSequenceClassification.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForSequenceClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForSequenceClassification.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -319,9 +319,7 @@ class OVModelForQuestionAnsweringIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForQuestionAnswering.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForQuestionAnswering.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForQuestionAnswering.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -394,9 +392,7 @@ class OVModelForTokenClassificationIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForTokenClassification.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForTokenClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForTokenClassification.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -440,9 +436,7 @@ class OVModelForFeatureExtractionIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForFeatureExtraction.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForFeatureExtraction.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModel.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -504,9 +498,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForCausalLM.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForCausalLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForCausalLM.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -651,9 +643,7 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForMaskedLM.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForMaskedLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForMaskedLM.from_pretrained(model_id) tokenizer = AutoTokenizer.from_pretrained(model_id) @@ -709,9 +699,7 @@ class OVModelForImageClassificationIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForImageClassification.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForImageClassification.from_pretrained(model_id) preprocessor = AutoFeatureExtractor.from_pretrained(model_id) @@ -747,9 +735,7 @@ def test_pipeline(self, model_arch): @parameterized.expand(TIMM_MODELS) def test_compare_to_timm(self, model_id): - ov_model = OVModelForImageClassification.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) timm_model = timm.create_model(model_id, pretrained=True) preprocessor = TimmImageProcessor.from_pretrained(model_id) @@ -801,9 +787,7 @@ class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForSeq2SeqLM.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.encoder, OVEncoder) self.assertIsInstance(ov_model.decoder, OVDecoder) @@ -942,9 +926,7 @@ def _generate_random_audio_data(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForAudioClassification.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForAudioClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForAudioClassification.from_pretrained(model_id) preprocessor = AutoFeatureExtractor.from_pretrained(model_id) @@ -1009,9 +991,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForCTC.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForCTC.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) set_seed(SEED) @@ -1063,9 +1043,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForAudioXVector.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForAudioXVector.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) set_seed(SEED) @@ -1119,9 +1097,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForAudioFrameClassification.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForAudioFrameClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) set_seed(SEED) @@ -1164,9 +1140,7 @@ class OVModelForPix2StructIntegrationTest(unittest.TestCase): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForPix2Struct.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForPix2Struct.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.encoder, OVEncoder) self.assertIsInstance(ov_model.decoder, OVDecoder) @@ -1255,9 +1229,7 @@ def _generate_random_audio_data(self): def test_compare_to_transformers(self, model_arch): model_id = MODEL_NAMES[model_arch] set_seed(SEED) - ov_model = OVModelForSpeechSeq2Seq.from_pretrained( - model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"} - ) + ov_model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True, ov_config=F32_CONFIG) self.assertIsInstance(ov_model.config, PretrainedConfig) transformers_model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id) processor = get_preprocessor(model_id) From fcc449747a20724136b3d09c05376443dba6d8af Mon Sep 17 00:00:00 2001 From: Helena Date: Tue, 9 Jan 2024 17:38:27 +0000 Subject: [PATCH 3/3] Replace get_version with is_openvino_version --- tests/openvino/test_modeling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/openvino/test_modeling.py b/tests/openvino/test_modeling.py index f44849ba91..65cd03d090 100644 --- a/tests/openvino/test_modeling.py +++ b/tests/openvino/test_modeling.py @@ -25,7 +25,6 @@ import torch from datasets import load_dataset from evaluate import evaluator -from openvino.runtime import get_version from parameterized import parameterized from PIL import Image from transformers import ( @@ -74,6 +73,7 @@ from optimum.intel.openvino import OV_DECODER_NAME, OV_DECODER_WITH_PAST_NAME, OV_ENCODER_NAME, OV_XML_FILE_NAME from optimum.intel.openvino.modeling_seq2seq import OVDecoder, OVEncoder from optimum.intel.openvino.modeling_timm import TimmImageProcessor +from optimum.intel.utils.import_utils import is_openvino_version from optimum.utils import ( DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER, DIFFUSION_MODEL_UNET_SUBFOLDER, @@ -128,7 +128,7 @@ def test_load_from_hub_and_save_model(self): loaded_model = OVModelForSequenceClassification.from_pretrained(self.OV_MODEL_ID, ov_config=ov_config) self.assertTrue(manual_openvino_cache_dir.is_dir()) self.assertGreaterEqual(len(list(manual_openvino_cache_dir.glob("*.blob"))), 1) - if get_version() < "2023.3": + if is_openvino_version("<", "2023.3"): self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT").name, "THROUGHPUT") else: self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT"), "THROUGHPUT")