Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added openvino compiler with tensorflow interface #137

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion nebullvm/operations/inference_learners/openvino.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,4 +400,4 @@ def run(self, *input_tensors: np.ndarray) -> Tuple[np.ndarray, ...]:
DeepLearningFramework.PYTORCH: PytorchOpenVinoInferenceLearner,
DeepLearningFramework.TENSORFLOW: TensorflowOpenVinoInferenceLearner,
DeepLearningFramework.NUMPY: NumpyOpenVinoInferenceLearner,
}
}
12 changes: 10 additions & 2 deletions nebullvm/operations/optimizations/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
TFLiteBuildInferenceLearner,
TensorflowBuildInferenceLearner,
)
from nebullvm.operations.inference_learners.openvino import (
TensorflowOpenVinoInferenceLearner,
)
from nebullvm.operations.measures.measures import MetricDropMeasure
from nebullvm.operations.measures.utils import (
compute_relative_difference,
Expand All @@ -35,6 +38,7 @@
)
from nebullvm.operations.optimizations.compilers.openvino import (
OpenVINOCompiler,
TensorFlowOpenVINOCompiler,
)
from nebullvm.operations.optimizations.compilers.pytorch import (
PytorchBackendCompiler,
Expand Down Expand Up @@ -319,7 +323,10 @@ def get_result(self) -> List:
DeepLearningFramework.NUMPY: ONNXApacheTVMCompiler,
},
ModelCompiler.ONNX_RUNTIME: {DeepLearningFramework.NUMPY: ONNXCompiler},
ModelCompiler.OPENVINO: {DeepLearningFramework.NUMPY: OpenVINOCompiler},
ModelCompiler.OPENVINO: {
DeepLearningFramework.NUMPY: OpenVINOCompiler,
DeepLearningFramework.TENSORFLOW: TensorFlowOpenVINOCompiler,
},
ModelCompiler.TFLITE: {
DeepLearningFramework.TENSORFLOW: TFLiteBackendCompiler
},
Expand Down Expand Up @@ -352,7 +359,8 @@ def get_result(self) -> List:
DeepLearningFramework.NUMPY: ONNXBuildInferenceLearner
},
ModelCompiler.OPENVINO: {
DeepLearningFramework.NUMPY: OpenVINOBuildInferenceLearner
DeepLearningFramework.NUMPY: OpenVINOBuildInferenceLearner,
DeepLearningFramework.TENSORFLOW: TensorflowOpenVinoInferenceLearner,
},
ModelCompiler.TFLITE: {
DeepLearningFramework.TENSORFLOW: TFLiteBuildInferenceLearner
Expand Down
128 changes: 127 additions & 1 deletion nebullvm/operations/optimizations/compilers/openvino.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@
)
from nebullvm.optional_modules.openvino import (
Core,
Model,
CompiledModel,
)
from nebullvm.optional_modules.tensorflow import tensorflow as tf
from nebullvm.tools.base import (
QuantizationType,
ModelParams,
Expand All @@ -25,6 +27,130 @@
from nebullvm.tools.transformations import MultiStageTransformation



class TensorFlowOpenVINOCompiler(Compiler):
supported_ops = {
"cpu": [
None,
# QuantizationType.STATIC,
# QuantizationType.HALF,
],
"gpu": [],
}

def __init__(self):
super().__init__()

def execute(
self,
model: tf.keras.Model,
model_params: ModelParams,
input_tfms: MultiStageTransformation = None,
metric_drop_ths: float = None,
quantization_type: QuantizationType = None,
input_data: DataManager = None,
**kwargs,
):
"""Compile the input model using TF-OPENVINO library.

Args:
model (tf.keras.Model()): The tensorflow model.
marcoschouten marked this conversation as resolved.
Show resolved Hide resolved
model_params (ModelParams): The model parameters.
output_file_path (str or Path): Path where storing the output
TensorFlow file.
input_tfms (MultiStageTransformation, optional): Transformations
to be performed to the model's input tensors in order to
get the prediction. Default: None.
metric_drop_ths (float, optional): Threshold for the accepted drop
in terms of precision. Any optimized model with a higher drop
will be ignored. Default: None.
quantization_type (QuantizationType, optional): The desired
quantization algorithm to be used. Default: None.
input_data (DataManager): User defined data. Default: None
"""

if quantization_type not in self.supported_ops[self.device.value]:
self.compiled_model = None
return

if quantization_type is QuantizationType.STATIC and input_data is None:
raise ValueError("Input data is required for static quantization.")

self.logger.info(
f"Optimizing with {self.__class__.__name__} and "
f"q_type: {quantization_type}."
)

check_quantization(quantization_type, metric_drop_ths)
train_input_data = input_data.get_split("train").get_numpy_list(
QUANTIZATION_DATA_NUM
)


# save model to temporary folder
tmp_dir_path = Compiler.onnx_output_path
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Be careful, the Compiler class doesn't have an onnx_output_path attribute.
onnx_output_path is an argument passed to this method from the caller (it's passed to each execute method of each compiler), so you should add it to the arguments above of the execute (line 46->51) like this:

def execute(
     self,
     model: tf.keras.Model,
     model_params: ModelParams,
     onnx_output_path: str,
     input_tfms: MultiStageTransformation = None,
     metric_drop_ths: float = None,
     quantization_type: QuantizationType = None,
     input_data: DataManager = None,
     **kwargs,
 ):

This parameters was introduced in the DeepSparseCompiler, that needs to export the original PyTorch model to onnx and so it needs the path to a temporary dir to generate the onnx model (that's why it's called onnx_output_path). In this case we are going to use it to convert a model to openvino, so I would suggest to rename it to tmp_dir_path that is a more general name (please reame it also in the DeepSparseCompiler and in the caller).

model.save(tmp_dir_path)


cmd = [
"mo",
"--saved_model_dir",
str(tmp_dir_path),
"--output_dir",
str(tmp_dir_path),
"--input",
",".join(get_input_names(model)),
"--input_shape",
",".join(
[
f"{list((model_params.batch_size,) + shape)}"
for shape in model_params.input_sizes
]
),
]

if quantization_type is QuantizationType.DYNAMIC:
return None

if quantization_type is QuantizationType.HALF:
cmd = cmd + ["--data_type", "FP16"]

process = subprocess.Popen(cmd)
process.wait()
base_path = Path(model).parent
openvino_model_path = base_path / f"{Path(model).stem}.xml"
openvino_model_weights = base_path / f"{Path(model).stem}.bin"

if quantization_type not in [QuantizationType.HALF, None]:
openvino_model_path, openvino_model_weights = self._quantize_model(
model_topology=str(openvino_model_path),
model_weights=str(openvino_model_weights),
input_names=get_input_names(model),
input_data=train_input_data,
)

self.compiled_model = str(
Path(openvino_model_path).parent / Path(openvino_model_path).stem
)

def _compile_model(
self,
model_name: str,
model_weights: str,
network_parameters: ModelParams,
) -> CompiledModel:
core = Core()
model = core.read_model(model=model_name, weights=model_weights)

dynamic_shape = self._get_dynamic_shape(model, network_parameters)

if dynamic_shape is not None:
model.reshape(dynamic_shape)

return core.compile_model(model=model, device_name="CPU")



class OpenVINOCompiler(Compiler):
supported_ops = {
"cpu": [
Expand All @@ -40,7 +166,7 @@ def __init__(self):

def execute(
self,
model: Union[str, Path],
model: tf.keras.Model(),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

model type should be only tf.keras.Model

model_params: ModelParams,
input_tfms: MultiStageTransformation = None,
metric_drop_ths: float = None,
Expand Down
2 changes: 2 additions & 0 deletions nebullvm/operations/optimizations/optimizers.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ def _select_compilers_from_hardware(self):
if tensorflow_is_available():
compilers.append(ModelCompiler.XLA)
compilers.append(ModelCompiler.TFLITE)
if self.device is Device.CPU and openvino_is_available():
compilers.append(ModelCompiler.OPENVINO)
return compilers


Expand Down
2 changes: 1 addition & 1 deletion nebullvm/tools/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,4 +138,4 @@ def recursively_dictionarize(element):
@property
def input_sizes(self):
for input_info in self.input_infos:
yield input_info.size
yield input_info.size