diff --git a/nebullvm/compressors/scripts/__init__.py b/nebullvm/compressors/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nebullvm/optimizers/tensor_rt.py b/nebullvm/optimizers/tensor_rt.py index e2f54369..fbc3eb06 100644 --- a/nebullvm/optimizers/tensor_rt.py +++ b/nebullvm/optimizers/tensor_rt.py @@ -335,7 +335,7 @@ def optimize_from_torch( dataset, batch_size=dataset.batch_size, shuffle=False, - num_workers=1, + num_workers=0, ) calibrator = torch_tensorrt.ptq.DataLoaderCalibrator( @@ -350,8 +350,14 @@ def optimize_from_torch( ): return None # Dynamic quantization is not supported on tensorRT + try: + torch.jit.script(torch_model.eval()) + model = torch_model + except Exception: + model = torch.jit.trace(torch_model, input_data.get_list(1)[0]) + trt_model = torch_tensorrt.compile( - torch_model.eval(), + model.eval(), inputs=[ torch_tensorrt.Input( (model_params.batch_size, *input_info.size), diff --git a/nebullvm/utils/optional_modules.py b/nebullvm/utils/optional_modules.py index a57f8f5f..e8c37b84 100644 --- a/nebullvm/utils/optional_modules.py +++ b/nebullvm/utils/optional_modules.py @@ -3,14 +3,16 @@ from nebullvm.installers.installers import install_tf2onnx, install_tensorflow from nebullvm.utils.general import check_module_version +NoneType = type(None) + class Keras: - Model = None + Model = NoneType class Tensorflow: - Module = None - Tensor = None + Module = NoneType + Tensor = NoneType keras = Keras() @staticmethod