EFRS-885 Import improvements

embedding-calculator/gpu.Dockerfile

@@ -67,7 +67,7 @@ RUN ln -s $(which $PYTHON) /usr/local/bin/python
 
 
 # Variables for MXNET
-ENV MXNET=mxnet_cu101mkl MXNET_CPU_WORKER_NTHREADS=24
+ENV MXNET_CPU_WORKER_NTHREADS=24
 ENV MXNET_ENGINE_TYPE=ThreadedEnginePerDevice MXNET_CUDNN_AUTOTUNE_DEFAULT=0
 
 # No access to GPU devices in the build stage, so skip tests

embedding-calculator/src/_endpoints.py

@@ -19,33 +19,37 @@
 
 from src.constants import ENV
 from src.exceptions import NoFaceFoundError
-from src.services.facescan.plugins import helpers
+from src.services.facescan.plugins import managers
 from src.services.facescan.scanner.facescanners import scanner
 from src.services.flask_.constants import ARG
 from src.services.flask_.needs_attached_file import needs_attached_file
 from src.services.imgtools.read_img import read_img
+from src.services.utils.pyutils import Constants
 
 
 def endpoints(app):
     @app.route('/status')
     def status_get():
-        availiable_plugins = {p.type: str(p) for p in helpers.get_face_plugins()}
-        calculator = helpers.get_calculator()
+        availiable_plugins = {p.slug: str(p)
+                              for p in managers.plugin_manager.plugins}
+        calculator = managers.plugin_manager.calculator
         return jsonify(status='OK', build_version=ENV.BUILD_VERSION,
                        calculator_version=str(calculator),
                        availiable_plugins=availiable_plugins)
 
     @app.route('/find_faces', methods=['POST'])
     @needs_attached_file
     def find_faces_post():
-        detector = helpers.get_detector()
-        face_plugins = helpers.get_face_plugins(_get_face_plugin_names())
+        detector = managers.plugin_manager.detector
+        face_plugins = managers.plugin_manager.filter_face_plugins(
+            _get_face_plugin_names()
+        )
         faces = detector(
             img=read_img(request.files['file']),
             det_prob_threshold=_get_det_prob_threshold(),
             face_plugins=face_plugins
         )
-        plugins_versions = {p.type: str(p) for p in [detector] + face_plugins}
+        plugins_versions = {p.slug: str(p) for p in [detector] + face_plugins}
         return jsonify(results=faces, plugins_versions=plugins_versions)
 
     @app.route('/scan_faces', methods=['POST'])
@@ -73,7 +77,7 @@ def _get_face_plugin_names() -> Optional[List[str]]:
     if ARG.FACE_PLUGINS not in request.values:
         return
     return [
-        name for name in filter(None, request.values[ARG.FACE_PLUGINS].split(','))
+        name for name in Constants.split(request.values[ARG.FACE_PLUGINS])
     ]
 
 

embedding-calculator/src/constants.py

@@ -32,8 +32,6 @@ class ENV(Constants):
     BUILD_VERSION = get_env('APP_VERSION_STRING', 'dev')
 
     GPU_IDX = int(get_env('GPU_IDX', '-1'))
-    CUDA = get_env('CUDA', '')
-    INTEL_OPTIMIZATION = get_env('INTEL_OPTIMIZATION', '')
 
 
 LOGGING_LEVEL = logging._nameToLevel[ENV.LOGGING_LEVEL_NAME]

embedding-calculator/src/services/facescan/plugins/core.py → embedding-calculator/src/services/facescan/plugins/base.py

@@ -50,7 +50,7 @@ def __str__(self):
 
     @property
     def path(self):
-        return Path(MODELS_ROOT) / self.plugin.backend / self.plugin.type / self.name
+        return Path(MODELS_ROOT) / self.plugin.backend / self.plugin.slug / self.name
 
     def exists(self):
         return os.path.exists(self.path)
@@ -83,9 +83,9 @@ def _extract(self, filename: str):
 
 
 class BasePlugin(ABC):
-    dependencies: Tuple[str, ...] = ()
     ml_models: Tuple[Tuple[str, str], ...] = ()
     ml_model: Optional[MLModel] = None
+    plugins_registry = []
 
     def __new__(cls, ml_model_name: str = None):
         """
@@ -94,13 +94,14 @@ def __new__(cls, ml_model_name: str = None):
         """
         if not hasattr(cls, 'instance'):
             cls.instance = super(BasePlugin, cls).__new__(cls)
+            cls.plugins_registry.append(cls.instance)
             if cls.instance.ml_models:
                 cls.instance.ml_model = MLModel(cls.instance, ml_model_name)
         return cls.instance
 
     @property
     @abstractmethod
-    def type(self):
+    def slug(self):
         pass
 
     @property
@@ -123,7 +124,7 @@ def __call__(self, face_img: Array3D) -> plugin_result.PluginResultDTO:
 
 
 class BaseFaceDetector(BasePlugin):
-    type = 'detector'
+    slug = 'detector'
     IMAGE_SIZE: int
     face_plugins: List[BasePlugin] = []
 
@@ -141,7 +142,7 @@ def _fetch_faces(self, img: Array3D, det_prob_threshold: float = None):
         return [
             plugin_result.FaceDTO(
                 img=img, face_img=self.crop_face(img, box), box=box,
-                execution_time={self.type: (time() - start) / len(boxes)}
+                execution_time={self.slug: (time() - start) / len(boxes)}
             ) for box in boxes
         ]
 
@@ -150,11 +151,12 @@ def _apply_face_plugins(self, face: plugin_result.FaceDTO,
         for plugin in face_plugins:
             start = time()
             try:
-                face._plugins_dto.append(plugin(face._face_img))
+                result_dto = plugin(face._face_img)
+                face._plugins_dto.append(result_dto)
             except Exception as e:
                 raise exceptions.PluginError(f'{plugin} error - {e}')
             else:
-                face.execution_time[plugin.type] = time() - start
+                face.execution_time[plugin.slug] = time() - start
 
     @abstractmethod
     def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[BoundingBoxDTO]:
@@ -168,7 +170,7 @@ def crop_face(self, img: Array3D, box: BoundingBoxDTO) -> Array3D:
 
 
 class BaseCalculator(BasePlugin):
-    type = 'calculator'
+    slug = 'calculator'
 
     DIFFERENCE_THRESHOLD: float
 

embedding-calculator/src/services/facescan/plugins/facenet/__init__.py

@@ -12,4 +12,4 @@
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
 
-from .facenet import FaceDetector, Calculator
+requirements = ('tensorflow~=1.15.4', 'facenet~=1.0.5')

embedding-calculator/src/services/facescan/plugins/facenet/facenet.py

@@ -27,19 +27,16 @@
 from src.services.imgtools.types import Array3D
 from src.services.utils.pyutils import get_current_dir
 
-from src.services.facescan.plugins import core
+from src.services.facescan.plugins import base
 
 CURRENT_DIR = get_current_dir(__file__)
-DEPENDENCIES = ('tensorflow~=1.15.4', 'facenet~=1.0.5')
 
 logger = logging.getLogger(__name__)
 _EmbeddingCalculator = namedtuple('_EmbeddingCalculator', 'graph sess')
 _FaceDetectionNets = namedtuple('_FaceDetectionNets', 'pnet rnet onet')
 
 
-class FaceDetector(core.BaseFaceDetector):
-    dependencies = DEPENDENCIES
-
+class FaceDetector(base.BaseFaceDetector):
     BATCH_SIZE = 25
     FACE_MIN_SIZE = 20
     SCALE_FACTOR = 0.709
@@ -103,17 +100,14 @@ def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[Bou
         return filtered_bounding_boxes
 
 
-class Calculator(core.BaseCalculator):
-    dependencies = DEPENDENCIES
+class Calculator(base.BaseCalculator):
     ml_models = (
         # links from https://github.com/davidsandberg/facenet#pre-trained-models
         # VGGFace2 training set, 0.9965 LFW accuracy
         ('20180402-114759', 'https://drive.google.com/uc?id=1EXPBSXwTaqrSC0OhUdXNmKSh9qJUQ55-'),
-        # ('20180402-114759', 'https://file-examples-com.github.io/uploads/2017/02/zip_2MB.zip'),
         # CASIA-WebFace training set, 0.9905 LFW accuracy
         ('20180408-102900', 'https://drive.google.com/uc?id=1R77HmFADxe87GmoLwzfgMu_HY0IhcyBz'),
     )
-
     BATCH_SIZE = 25
     DIFFERENCE_THRESHOLD = 0.2
 

embedding-calculator/src/services/facescan/plugins/insightface/__init__.py

@@ -12,4 +12,24 @@
 #  or implied. See the License for the specific language governing
 #  permissions and limitations under the License.
 
-from .insightface import FaceDetector, Calculator, GenderAgeDetector
+
+from src.constants import ENV
+from src.services.utils.pyutils import get_env, get_env_bool
+
+
+def get_requirements():
+    cuda_version = get_env('CUDA', '').replace('.', '')
+    intel_optimization = get_env_bool('INTEL_OPTIMIZATION', False)
+
+    mxnet_lib = 'mxnet-'
+    if ENV.GPU_IDX > -1 and cuda_version:
+        mxnet_lib += f"cu{cuda_version}"
+    if intel_optimization:
+        mxnet_lib += 'mkl'
+    mxnet_lib = mxnet_lib.rstrip('-')
+    return (
+        f'{mxnet_lib}<1.7',
+        'insightface==0.1.5',
+    )
+
+requirements = get_requirements()