Skip to content

Commit

Permalink
Add f32 precision for compare-with-transformers tests (#508)
Browse files Browse the repository at this point in the history 
* f32 precision for compare-with-transformers tests

Allow these tests to pass locally on devices where inference is run by default in FP16 (GPU) or BF16 (latest Xeon).

* Add F32_CONFIG constant for modeling tests

* Replace get_version with is_openvino_version
  • Loading branch information
@helena-intel
helena-intel authored Jan 10, 2024
1 parent c64025d commit aa5b71b
Showing 1 changed file with 22 additions and 16 deletions.
38 changes: 22 additions & 16 deletions tests/openvino/test_modeling.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
from optimum.intel.openvino import OV_DECODER_NAME, OV_DECODER_WITH_PAST_NAME, OV_ENCODER_NAME, OV_XML_FILE_NAME
from optimum.intel.openvino.modeling_seq2seq import OVDecoder, OVEncoder
from optimum.intel.openvino.modeling_timm import TimmImageProcessor
from optimum.intel.utils.import_utils import is_openvino_version
from optimum.utils import (
DIFFUSION_MODEL_TEXT_ENCODER_SUBFOLDER,
DIFFUSION_MODEL_UNET_SUBFOLDER,
Expand All @@ -89,6 +90,8 @@

SEED = 42

F32_CONFIG = {"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}


class Timer(object):
def __enter__(self):
Expand Down Expand Up @@ -125,7 +128,10 @@ def test_load_from_hub_and_save_model(self):
loaded_model = OVModelForSequenceClassification.from_pretrained(self.OV_MODEL_ID, ov_config=ov_config)
self.assertTrue(manual_openvino_cache_dir.is_dir())
self.assertGreaterEqual(len(list(manual_openvino_cache_dir.glob("*.blob"))), 1)
self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT").name, "THROUGHPUT")
if is_openvino_version("<", "2023.3"):
self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT").name, "THROUGHPUT")
else:
self.assertEqual(loaded_model.request.get_property("PERFORMANCE_HINT"), "THROUGHPUT")

with tempfile.TemporaryDirectory() as tmpdirname:
loaded_model.save_pretrained(tmpdirname)
Expand Down Expand Up @@ -247,7 +253,7 @@ class OVModelForSequenceClassificationIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForSequenceClassification.from_pretrained(model_id, export=True)
ov_model = OVModelForSequenceClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForSequenceClassification.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -313,7 +319,7 @@ class OVModelForQuestionAnsweringIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForQuestionAnswering.from_pretrained(model_id, export=True)
ov_model = OVModelForQuestionAnswering.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForQuestionAnswering.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -386,7 +392,7 @@ class OVModelForTokenClassificationIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForTokenClassification.from_pretrained(model_id, export=True)
ov_model = OVModelForTokenClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForTokenClassification.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -430,7 +436,7 @@ class OVModelForFeatureExtractionIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForFeatureExtraction.from_pretrained(model_id, export=True)
ov_model = OVModelForFeatureExtraction.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModel.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -492,7 +498,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForCausalLM.from_pretrained(model_id, export=True)
ov_model = OVModelForCausalLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForCausalLM.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -637,7 +643,7 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForMaskedLM.from_pretrained(model_id, export=True)
ov_model = OVModelForMaskedLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForMaskedLM.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -693,7 +699,7 @@ class OVModelForImageClassificationIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True)
ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForImageClassification.from_pretrained(model_id)
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
Expand Down Expand Up @@ -729,7 +735,7 @@ def test_pipeline(self, model_arch):

@parameterized.expand(TIMM_MODELS)
def test_compare_to_timm(self, model_id):
ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True)
ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
timm_model = timm.create_model(model_id, pretrained=True)
preprocessor = TimmImageProcessor.from_pretrained(model_id)
Expand Down Expand Up @@ -781,7 +787,7 @@ class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True)
ov_model = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)

self.assertIsInstance(ov_model.encoder, OVEncoder)
self.assertIsInstance(ov_model.decoder, OVDecoder)
Expand Down Expand Up @@ -920,7 +926,7 @@ def _generate_random_audio_data(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForAudioClassification.from_pretrained(model_id, export=True)
ov_model = OVModelForAudioClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForAudioClassification.from_pretrained(model_id)
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
Expand Down Expand Up @@ -985,7 +991,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForCTC.from_pretrained(model_id, export=True)
ov_model = OVModelForCTC.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)

set_seed(SEED)
Expand Down Expand Up @@ -1037,7 +1043,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForAudioXVector.from_pretrained(model_id, export=True)
ov_model = OVModelForAudioXVector.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)

set_seed(SEED)
Expand Down Expand Up @@ -1091,7 +1097,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForAudioFrameClassification.from_pretrained(model_id, export=True)
ov_model = OVModelForAudioFrameClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)

set_seed(SEED)
Expand Down Expand Up @@ -1134,7 +1140,7 @@ class OVModelForPix2StructIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForPix2Struct.from_pretrained(model_id, export=True)
ov_model = OVModelForPix2Struct.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)

self.assertIsInstance(ov_model.encoder, OVEncoder)
self.assertIsInstance(ov_model.decoder, OVDecoder)
Expand Down Expand Up @@ -1223,7 +1229,7 @@ def _generate_random_audio_data(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True)
ov_model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id)
processor = get_preprocessor(model_id)
Expand Down

0 comments on commit aa5b71b

Please sign in to comment.