Skip to content

Commit

Permalink
Add F32_CONFIG constant for modeling tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@helena-intel
helena-intel committed Jan 9, 2024
1 parent e7c15c8 commit 4f9ea99
Showing 1 changed file with 17 additions and 45 deletions.
62 changes: 17 additions & 45 deletions tests/openvino/test_modeling.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@

SEED = 42

F32_CONFIG = {"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}


class Timer(object):
def __enter__(self):
Expand Down Expand Up @@ -251,9 +253,7 @@ class OVModelForSequenceClassificationIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForSequenceClassification.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForSequenceClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForSequenceClassification.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -319,9 +319,7 @@ class OVModelForQuestionAnsweringIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForQuestionAnswering.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForQuestionAnswering.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForQuestionAnswering.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -394,9 +392,7 @@ class OVModelForTokenClassificationIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForTokenClassification.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForTokenClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForTokenClassification.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -440,9 +436,7 @@ class OVModelForFeatureExtractionIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForFeatureExtraction.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForFeatureExtraction.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModel.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -504,9 +498,7 @@ class OVModelForCausalLMIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForCausalLM.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForCausalLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForCausalLM.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -651,9 +643,7 @@ class OVModelForMaskedLMIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForMaskedLM.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForMaskedLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForMaskedLM.from_pretrained(model_id)
tokenizer = AutoTokenizer.from_pretrained(model_id)
Expand Down Expand Up @@ -709,9 +699,7 @@ class OVModelForImageClassificationIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForImageClassification.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForImageClassification.from_pretrained(model_id)
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
Expand Down Expand Up @@ -747,9 +735,7 @@ def test_pipeline(self, model_arch):

@parameterized.expand(TIMM_MODELS)
def test_compare_to_timm(self, model_id):
ov_model = OVModelForImageClassification.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForImageClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
timm_model = timm.create_model(model_id, pretrained=True)
preprocessor = TimmImageProcessor.from_pretrained(model_id)
Expand Down Expand Up @@ -801,9 +787,7 @@ class OVModelForSeq2SeqLMIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForSeq2SeqLM.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForSeq2SeqLM.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)

self.assertIsInstance(ov_model.encoder, OVEncoder)
self.assertIsInstance(ov_model.decoder, OVDecoder)
Expand Down Expand Up @@ -942,9 +926,7 @@ def _generate_random_audio_data(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForAudioClassification.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForAudioClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForAudioClassification.from_pretrained(model_id)
preprocessor = AutoFeatureExtractor.from_pretrained(model_id)
Expand Down Expand Up @@ -1009,9 +991,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForCTC.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForCTC.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)

set_seed(SEED)
Expand Down Expand Up @@ -1063,9 +1043,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForAudioXVector.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForAudioXVector.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)

set_seed(SEED)
Expand Down Expand Up @@ -1119,9 +1097,7 @@ def test_load_vanilla_transformers_which_is_not_supported(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForAudioFrameClassification.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForAudioFrameClassification.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)

set_seed(SEED)
Expand Down Expand Up @@ -1164,9 +1140,7 @@ class OVModelForPix2StructIntegrationTest(unittest.TestCase):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForPix2Struct.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForPix2Struct.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)

self.assertIsInstance(ov_model.encoder, OVEncoder)
self.assertIsInstance(ov_model.decoder, OVDecoder)
Expand Down Expand Up @@ -1255,9 +1229,7 @@ def _generate_random_audio_data(self):
def test_compare_to_transformers(self, model_arch):
model_id = MODEL_NAMES[model_arch]
set_seed(SEED)
ov_model = OVModelForSpeechSeq2Seq.from_pretrained(
model_id, export=True, ov_config={"CACHE_DIR": "", "INFERENCE_PRECISION_HINT": "f32"}
)
ov_model = OVModelForSpeechSeq2Seq.from_pretrained(model_id, export=True, ov_config=F32_CONFIG)
self.assertIsInstance(ov_model.config, PretrainedConfig)
transformers_model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id)
processor = get_preprocessor(model_id)
Expand Down

0 comments on commit 4f9ea99

Please sign in to comment.