diff --git a/embedding-calculator/Dockerfile b/embedding-calculator/Dockerfile index 0b18319b57..fbe5cfe06a 100644 --- a/embedding-calculator/Dockerfile +++ b/embedding-calculator/Dockerfile @@ -27,7 +27,9 @@ ENV JOBLIB_MULTIPROCESSING=0 # download ML models ARG DETECTION_MODEL="retinaface_r50_v1" ARG CALCULATION_MODEL="arcface_r100_v1" -ENV DETECTION_MODEL=$DETECTION_MODEL CALCULATION_MODEL=$CALCULATION_MODEL +ARG GENDERAGE_MODEL="genderage_v1" +ENV DETECTION_MODEL=$DETECTION_MODEL CALCULATION_MODEL=$CALCULATION_MODEL \ + GENDERAGE_MODEL=$GENDERAGE_MODEL COPY srcext srcext COPY pytest.ini . COPY *.sh ./ diff --git a/embedding-calculator/prepare_scanners.sh b/embedding-calculator/prepare_scanners.sh index 1ab30ec602..99cfc5a65a 100644 --- a/embedding-calculator/prepare_scanners.sh +++ b/embedding-calculator/prepare_scanners.sh @@ -3,7 +3,7 @@ if [[ "$SCANNER" == "InsightFace" ]]; then MODELS_PATH=~/.insightface/models mkdir -p $MODELS_PATH - for MODEL in $DETECTION_MODEL $CALCULATION_MODEL + for MODEL in $DETECTION_MODEL $CALCULATION_MODEL $GENDERAGE_MODEL do # trying to find a pre-downloaded model DIR=~/srcext/insightface/models/$MODEL @@ -19,12 +19,8 @@ if [[ "$SCANNER" == "InsightFace" ]]; then # https://github.com/deepinsight/insightface/issues/764 sed -i 's/limited_workspace/None/g' $MODELS_PATH/$MODEL/*.json done -else - echo " --ignore=src/services/facescan/scanner/insightface" >> pytest.ini fi if [[ "$SCANNER" == "Facenet2018" ]]; then pip install --no-cache-dir tensorflow~=1.15.4 -else - echo " --ignore=src/services/facescan/scanner/facenet" >> pytest.ini fi \ No newline at end of file diff --git a/embedding-calculator/src/_endpoints.py b/embedding-calculator/src/_endpoints.py index 3bdb3c5c44..b8e568ebe5 100644 --- a/embedding-calculator/src/_endpoints.py +++ b/embedding-calculator/src/_endpoints.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing # permissions and limitations under the License. -from typing import List +from typing import List, Optional from flask import request from flask.json import jsonify @@ -19,6 +19,8 @@ from src.constants import ENV from src.exceptions import NoFaceFoundError +from src.services.facescan import helpers +from src.services.facescan.scanner import facescanner as scanners from src.services.facescan.scanner.facescanners import scanner from src.services.flask_.constants import ARG from src.services.flask_.needs_attached_file import needs_attached_file @@ -28,31 +30,37 @@ def endpoints(app): @app.route('/status') def status_get(): + availiable_plugins = {p.name: p.backend for p in helpers.get_plugins()} return jsonify(status='OK', build_version=ENV.BUILD_VERSION, - calculator_version=ENV.SCANNER) + calculator_version=scanner.ID, + availiable_plugins=availiable_plugins) @app.route('/find_faces', methods=['POST']) @needs_attached_file def find_faces_post(): - faces = scanner.find_faces( + detector = helpers.get_detector(_get_det_plugin_name()) + face_plugins = helpers.get_plugins(_get_face_plugin_names()) + + faces = detector( img=read_img(request.files['file']), - det_prob_threshold=_get_det_prob_threshold(request), + det_prob_threshold=_get_det_prob_threshold(), + face_plugins=face_plugins ) - faces = _limit(faces, request.values.get(ARG.LIMIT)) - return jsonify(calculator_version=scanner.ID, result=faces) + plugins_versions = {p.name: p.backend for p in [detector] + face_plugins} + return jsonify(results=faces, plugins_versions=plugins_versions) @app.route('/scan_faces', methods=['POST']) @needs_attached_file def scan_faces_post(): faces = scanner.scan( img=read_img(request.files['file']), - det_prob_threshold=_get_det_prob_threshold(request) + det_prob_threshold=_get_det_prob_threshold() ) faces = _limit(faces, request.values.get(ARG.LIMIT)) return jsonify(calculator_version=scanner.ID, result=faces) -def _get_det_prob_threshold(request): +def _get_det_prob_threshold(): det_prob_threshold_val = request.values.get(ARG.DET_PROB_THRESHOLD) if det_prob_threshold_val is None: return None @@ -62,6 +70,20 @@ def _get_det_prob_threshold(request): return det_prob_threshold +def _get_det_plugin_name() -> Optional[str]: + if ARG.DET_PLUGIN not in request.values: + return + return request.values.get(ARG.DET_PLUGIN) + + +def _get_face_plugin_names() -> Optional[List[str]]: + if ARG.FACE_PLUGINS not in request.values: + return + return [ + name for name in request.values[ARG.FACE_PLUGINS].split(',') if name + ] + + def _limit(faces: List, limit: str = None) -> List: """ >>> _limit([1, 2, 3], None) diff --git a/embedding-calculator/src/constants.py b/embedding-calculator/src/constants.py index 322f516332..9cec5bbab9 100644 --- a/embedding-calculator/src/constants.py +++ b/embedding-calculator/src/constants.py @@ -32,7 +32,29 @@ class ENV(Constants): GPU_ID = int(get_env('GPU_ID', '-1')) DETECTION_MODEL = get_env('DETECTION_MODEL', 'retinaface_r50_v1') CALCULATION_MODEL = get_env('CALCULATION_MODEL', 'arcface_r100_v1') + GENDERAGE_MODEL = get_env('GENDERAGE_MODEL', 'genderage_v1') LOGGING_LEVEL = logging._nameToLevel[ENV.LOGGING_LEVEL_NAME] ENV_MAIN = ENV + + +MXNET_FACE_DETECTORS = ( + '.plugins.insightface.FaceDetector', +) +MXNET_FACE_PLUGINS = ( + '.plugins.insightface.Calculator', + '.plugins.insightface.GenderAgeDetector', +) + +TF_FACE_DETECTORS = ( + '.plugins.facenet.FaceDetector', +) +TF_FACE_PLUGINS = ( + '.plugins.facenet.Calculator', + '.plugins.rude_carnie.AgeDetector', + '.plugins.rude_carnie.GenderDetector', +) + +FACE_DETECTORS = TF_FACE_DETECTORS +FACE_PLUGINS = TF_FACE_PLUGINS \ No newline at end of file diff --git a/embedding-calculator/src/docs/find_faces_post.yml b/embedding-calculator/src/docs/find_faces_post.yml index 9a0d8de8ef..79b2bcd697 100644 --- a/embedding-calculator/src/docs/find_faces_post.yml +++ b/embedding-calculator/src/docs/find_faces_post.yml @@ -2,7 +2,7 @@ tags: - Core summary: 'Find faces in the given image and return their bounding boxes.' description: 'Returns bounding boxes of detected faces on the image.' -operationId: scanFacesPost +operationId: findFacesPost consumes: - multipart/form-data produces: diff --git a/embedding-calculator/src/exceptions.py b/embedding-calculator/src/exceptions.py index c955e47540..7f5fcbecd4 100644 --- a/embedding-calculator/src/exceptions.py +++ b/embedding-calculator/src/exceptions.py @@ -37,10 +37,6 @@ class OneDimensionalImageIsGivenError(BadRequest): description = "Given image has only one dimension" -class MoreThanOneFaceFoundError(BadRequest): - description = "Found more than one face in the given image" - - class ClassifierIsAlreadyTrainingError(Locked): description = "Classifier training is already in progress" @@ -74,3 +70,6 @@ class CouldNotConnectToDatabase(InternalServerError): class NotEnoughUniqueFacesError(BadRequest): description = "Not enough unique faces to start training a new classifier model. " \ "Deleting existing classifiers, if any." + +class InvalidFaceDetectorPlugin(BadRequest): + description = "Invalid face detector plugin" diff --git a/embedding-calculator/src/services/dto/plugin_result.py b/embedding-calculator/src/services/dto/plugin_result.py new file mode 100644 index 0000000000..5cef828c2e --- /dev/null +++ b/embedding-calculator/src/services/dto/plugin_result.py @@ -0,0 +1,44 @@ +import attr +from typing import Tuple, List, Optional, Dict + +from src.services.dto.bounding_box import BoundingBoxDTO +from src.services.dto.json_encodable import JSONEncodable +from src.services.imgtools.types import Array1D, Array3D + + +class PluginResultDTO(JSONEncodable): + def to_json(self) -> dict: + """ Serialize only public properties """ + return {k: v for k, v in self.__dict__.items() if not k.startswith('_')} + + +@attr.s(auto_attribs=True, frozen=True) +class EmbeddingDTO(PluginResultDTO): + embedding: Array1D + + +@attr.s(auto_attribs=True, frozen=True) +class GenderDTO(PluginResultDTO): + gender: str + gender_probability: float = attr.ib(converter=float, default=None) + + +@attr.s(auto_attribs=True, frozen=True) +class AgeDTO(PluginResultDTO): + age: Tuple[int, int] + age_probability: float = attr.ib(converter=float, default=None) + + +@attr.s(auto_attribs=True) +class FaceDTO(PluginResultDTO): + box: BoundingBoxDTO + _img: Optional[Array3D] + _face_img: Optional[Array3D] + _plugins_dto: List[PluginResultDTO] = attr.Factory(list) + execution_time: Dict[str, float] = attr.Factory(dict) + + def to_json(self): + data = super().to_json() + for plugin_dto in self._plugins_dto: + data.update(plugin_dto.to_json()) + return data diff --git a/embedding-calculator/src/services/dto/scanned_face.py b/embedding-calculator/src/services/dto/scanned_face.py index b21cc9f63f..d44b9982e2 100644 --- a/embedding-calculator/src/services/dto/scanned_face.py +++ b/embedding-calculator/src/services/dto/scanned_face.py @@ -12,7 +12,7 @@ # or implied. See the License for the specific language governing # permissions and limitations under the License. -from typing import Union +from typing import Union, Optional import attr @@ -22,23 +22,27 @@ from src.services.imgtools.types import Array1D, Array3D +@attr.s(auto_attribs=True) +class Face(JSONEncodable): + _img: Optional[Array3D] + _face_img: Optional[Array3D] + box: BoundingBoxDTO + + @attr.s(auto_attribs=True, frozen=True) class ScannedFaceDTO(JSONEncodable): box: BoundingBoxDTO embedding: Array1D -class ScannedFace(JSONEncodable): - def __init__(self, box: BoundingBoxDTO, embedding: Array1D, img: Union[Array3D, None], face_img: Array3D = None): - self.box = box - self.embedding = embedding - self.img = img - self._face_img = face_img +@attr.s(auto_attribs=True) +class ScannedFace(Face): + embedding: Array1D @property def face_img(self): if not self._face_img: - self._face_img = crop_img(self.img, self.box) + self._face_img = crop_img(self._img, self.box) return self._face_img @property @@ -53,4 +57,5 @@ def from_request(cls, result): y_min=box_result['y_min'], y_max=box_result['y_max'], probability=box_result['probability']), - embedding=result['embedding'], img=None) + embedding=result['embedding'], + img=None, face_img=None) diff --git a/embedding-calculator/src/services/facescan/conftest.py b/embedding-calculator/src/services/facescan/conftest.py new file mode 100644 index 0000000000..03155dd2af --- /dev/null +++ b/embedding-calculator/src/services/facescan/conftest.py @@ -0,0 +1,32 @@ +# Copyright (c) 2020 the original author or authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. + +import os +from importlib.util import find_spec + +modules_by_lib = { + 'tensorflow': ('facenet', 'rude_carnie'), + 'mexnet': ('insightface',) +} +modules_to_skip = [] +for lib, modules in modules_by_lib.items(): + if find_spec(lib) is None: + modules_to_skip.extend(modules) + + +def pytest_ignore_collect(path): + _, last_path = os.path.split(path) + for module in modules: + if last_path.startswith(module): + return True diff --git a/embedding-calculator/src/services/facescan/core.py b/embedding-calculator/src/services/facescan/core.py new file mode 100644 index 0000000000..ab72c650d6 --- /dev/null +++ b/embedding-calculator/src/services/facescan/core.py @@ -0,0 +1,110 @@ +# Copyright (c) 2020 the original author or authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. + +from time import time +from abc import ABC, abstractmethod +from typing import List + +from src.services.dto.bounding_box import BoundingBoxDTO +from src.services.dto import plugin_result +from src.services.imgtools.types import Array3D + + +class ExecutionContext: + def __enter__(self): + self.start = time() + return self + + def __exit__(self, type, value, traceback): + self.end = time() + + def __float__(self): + return float(self.end - self.start) + + +class BasePlugin(ABC): + + def __new__(cls): + """ + Plugins might cache models in properties so it has to be Singleton. + """ + if not hasattr(cls, 'instance'): + cls.instance = super(BasePlugin, cls).__new__(cls) + return cls.instance + + @property + @abstractmethod + def name(self): + pass + + @property + def backend(self) -> str: + return self.__class__.__module__.rsplit('.', 1)[-1] + + @abstractmethod + def __call__(self, face_img: Array3D) -> plugin_result.PluginResultDTO: + raise NotImplementedError + + +class BaseFaceDetector(BasePlugin): + name = 'detector' + + def __call__(self, img: Array3D, det_prob_threshold: float = None, + face_plugins: List[BasePlugin] = ()): + """ Returns cropped and normalized faces.""" + faces = self._fetch_faces(img, det_prob_threshold) + for face in faces: + self._apply_face_plugins(face, face_plugins) + return faces + + def _fetch_faces(self, img: Array3D, det_prob_threshold: float = None): + start = time() + boxes = self.find_faces(img, det_prob_threshold) + return [ + plugin_result.FaceDTO( + img=img, face_img=self.crop_face(img, box), box=box, + execution_time={self.name: (time() - start) / len(boxes)} + ) for box in boxes + ] + + def _apply_face_plugins(self, face: plugin_result.FaceDTO, + plugins: List[BasePlugin]): + for plugin in plugins: + start = time() + face._plugins_dto.append(plugin(face._face_img)) + face.execution_time[plugin.name] = time() - start + + @abstractmethod + def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[BoundingBoxDTO]: + """ Find face bounding boxes, without calculating embeddings""" + raise NotImplementedError + + @abstractmethod + def crop_face(self, img: Array3D, box: BoundingBoxDTO) -> Array3D: + """ Crop face by bounding box and resize/squish it """ + raise NotImplementedError + + +class BaseCalculator(BasePlugin): + name = 'calculator' + + def __call__(self, face_img: Array3D): + return plugin_result.EmbeddingDTO( + embedding=self.calc_embedding(face_img) + ) + + @abstractmethod + def calc_embedding(self, face_img: Array3D) -> Array3D: + """ Calculate embedding of a given face """ + raise NotImplementedError diff --git a/embedding-calculator/src/services/facescan/helpers.py b/embedding-calculator/src/services/facescan/helpers.py new file mode 100644 index 0000000000..1b92f4747e --- /dev/null +++ b/embedding-calculator/src/services/facescan/helpers.py @@ -0,0 +1,55 @@ +# Copyright (c) 2020 the original author or authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. + +from importlib import import_module +from typing import List, Dict, Type + +from src import constants, exceptions +from src.services.facescan.core import BasePlugin, BaseFaceDetector +from src.services.imgtools.types import Array3D + + +def import_classes(class_path: str): + module, class_name = class_path.rsplit('.', 1) + return getattr(import_module(module, __package__), class_name) + + +def import_plugins(paths: List[str]) -> List[Type[BasePlugin]]: + return [import_classes(path) for path in paths] + + +def get_plugins(names: List[str] = None) -> List[BasePlugin]: + return [fp() for fp in import_plugins(constants.FACE_PLUGINS) + if names is None or fp.name in names] + + +def get_detector(name: str = None) -> BaseFaceDetector: + for face_detector in import_plugins(constants.FACE_DETECTORS): + if name is not None and face_detector.name != name: + continue + return face_detector() + raise exceptions.InvalidFaceDetectorPlugin + + +class Scanner: + """ + Class for backward compatibility. + The scanner does only detection and embedding calculation. + """ + ID = "PluginScanner" + + def scan(self, img: Array3D, det_prob_threshold: float = None): + detector: BaseFaceDetector = get_detector() + face_plugins = get_plugins(['calculator']) + return detector(img, det_prob_threshold, face_plugins) diff --git a/embedding-calculator/src/services/facescan/plugins/__init__.py b/embedding-calculator/src/services/facescan/plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/embedding-calculator/src/services/facescan/plugins/facenet.py b/embedding-calculator/src/services/facescan/plugins/facenet.py new file mode 100644 index 0000000000..10e639bf66 --- /dev/null +++ b/embedding-calculator/src/services/facescan/plugins/facenet.py @@ -0,0 +1,133 @@ +# Copyright (c) 2020 the original author or authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. + +import logging +import math +from collections import namedtuple +from typing import List + +import numpy as np +import tensorflow as tf +from cached_property import cached_property +from srcext.facenet.align import detect_face +from tensorflow.python.platform import gfile + +from src.constants import ENV +from src.services.dto.bounding_box import BoundingBoxDTO +from src.services.facescan.imgscaler.imgscaler import ImgScaler +from src.services.imgtools.proc_img import crop_img, squish_img +from src.services.imgtools.types import Array3D +from src.services.utils.pyutils import get_current_dir + +from src.services.facescan import core + +CURRENT_DIR = get_current_dir(__file__) +logger = logging.getLogger(__name__) +_EmbeddingCalculator = namedtuple('_EmbeddingCalculator', 'graph sess') +_FaceDetectionNets = namedtuple('_FaceDetectionNets', 'pnet rnet onet') + + +class FaceDetector(core.BaseFaceDetector): + BATCH_SIZE = 25 + FACE_MIN_SIZE = 20 + SCALE_FACTOR = 0.709 + BOX_MARGIN = 32 + IMAGE_SIZE = 160 + IMG_LENGTH_LIMIT = ENV.IMG_LENGTH_LIMIT + + # detection settings + det_prob_threshold = 0.65 + det_threshold_a = 0.9436513301 + det_threshold_b = 0.7059968943 + det_threshold_c = 0.5506904359 + + @cached_property + def _face_detection_nets(self): + with tf.Graph().as_default(): + sess = tf.Session() + return _FaceDetectionNets(*detect_face.create_mtcnn(sess, None)) + + def crop_face(self, img: Array3D, box: BoundingBoxDTO) -> Array3D: + return squish_img(crop_img(img, box), (self.IMAGE_SIZE, self.IMAGE_SIZE)) + + def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[BoundingBoxDTO]: + if det_prob_threshold is None: + det_prob_threshold = self.det_prob_threshold + assert 0 <= det_prob_threshold <= 1 + scaler = ImgScaler(self.IMG_LENGTH_LIMIT) + img = scaler.downscale_img(img) + + fdn = self._face_detection_nets + detect_face_result = detect_face.detect_face( + img, self.FACE_MIN_SIZE, fdn.pnet, fdn.rnet, fdn.onet, + [self.det_threshold_a, self.det_threshold_b, self.det_threshold_c], self.SCALE_FACTOR) + img_size = np.asarray(img.shape)[0:2] + bounding_boxes = [] + for result_item in detect_face_result[0]: + result_item = np.squeeze(result_item) + margin = self.BOX_MARGIN / 2 + box = BoundingBoxDTO( + x_min=int(np.maximum(result_item[0] - margin, 0)), + y_min=int(np.maximum(result_item[1] - margin, 0)), + x_max=int(np.minimum(result_item[2] + margin, img_size[1])), + y_max=int(np.minimum(result_item[3] + margin, img_size[0])), + probability=result_item[4] + ) + logger.debug(f"Found: {box}") + bounding_boxes.append(box) + + filtered_bounding_boxes = [] + for box in bounding_boxes: + box = box.scaled(scaler.upscale_coefficient) + if box.probability <= det_prob_threshold: + logger.debug(f'Box filtered out because below threshold ({det_prob_threshold}): {box}') + continue + filtered_bounding_boxes.append(box) + return filtered_bounding_boxes + + +class Calculator(core.BaseCalculator): + BATCH_SIZE = 25 + EMBEDDING_MODEL_PATH = CURRENT_DIR / '..' / 'scanner' / 'facenet' / 'model' / 'embedding_calc_model_20180402.pb' + + def calc_embedding(self, face_img: Array3D) -> Array3D: + return self._calculate_embeddings([face_img])[0] + + @cached_property + def _embedding_calculator(self): + with tf.Graph().as_default() as graph: + graph_def = tf.GraphDef() + with gfile.FastGFile(str(self.EMBEDDING_MODEL_PATH), 'rb') as f: + model = f.read() + graph_def.ParseFromString(model) + tf.import_graph_def(graph_def, name='') + return _EmbeddingCalculator(graph=graph, sess=tf.Session(graph=graph)) + + def _calculate_embeddings(self, cropped_images): + """Run forward pass to calculate embeddings""" + calc_model = self._embedding_calculator + graph_images_placeholder = calc_model.graph.get_tensor_by_name("input:0") + graph_embeddings = calc_model.graph.get_tensor_by_name("embeddings:0") + graph_phase_train_placeholder = calc_model.graph.get_tensor_by_name("phase_train:0") + embedding_size = graph_embeddings.get_shape()[1] + image_count = len(cropped_images) + batches_per_epoch = int(math.ceil(1.0 * image_count / self.BATCH_SIZE)) + embeddings = np.zeros((image_count, embedding_size)) + for i in range(batches_per_epoch): + start_index = i * self.BATCH_SIZE + end_index = min((i + 1) * self.BATCH_SIZE, image_count) + feed_dict = {graph_images_placeholder: cropped_images, graph_phase_train_placeholder: False} + embeddings[start_index:end_index, :] = calc_model.sess.run( + graph_embeddings, feed_dict=feed_dict) + return embeddings diff --git a/embedding-calculator/src/services/facescan/plugins/insightface.py b/embedding-calculator/src/services/facescan/plugins/insightface.py new file mode 100644 index 0000000000..915d4f51da --- /dev/null +++ b/embedding-calculator/src/services/facescan/plugins/insightface.py @@ -0,0 +1,164 @@ +# Copyright (c) 2020 the original author or authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. + +import os +import logging +from typing import List, Tuple + +import attr +import numpy as np +from cached_property import cached_property +from insightface.app import FaceAnalysis +from insightface.model_zoo import model_zoo, model_store +from insightface.model_zoo import face_recognition, face_detection, face_genderage +from insightface.utils import face_align + +from src.constants import ENV +from src.services.dto.bounding_box import BoundingBoxDTO +from src.services.facescan.imgscaler.imgscaler import ImgScaler +from src.services.facescan import core +from src.services.dto import plugin_result +from src.services.imgtools.types import Array3D + +logger = logging.getLogger(__name__) + +BACKEND = 'InsightFace' + + +def _get_model_file(name): + """ Return location for the pretrained on local file system. + InsightFace `get_model_file` works only with build in models. + """ + root = os.path.expanduser(os.path.join('~', '.insightface', 'models')) + dir_path = os.path.join(root, name) + return model_store.find_params_file(dir_path) + + +@attr.s(auto_attribs=True, frozen=True) +class InsightFaceBoundingBox(BoundingBoxDTO): + landmark: Tuple[int, ...] + + @property + def dto(self): + return BoundingBoxDTO(x_min=self.x_min, x_max=self.x_max, + y_min=self.y_min, y_max=self.y_max, + probability=self.probability) + + def scaled(self, coefficient: float) -> 'InsightFaceBoundingBox': + # noinspection PyTypeChecker + return InsightFaceBoundingBox(x_min=self.x_min * coefficient, x_max=self.x_max * coefficient, + y_min=self.y_min * coefficient, y_max=self.y_max * coefficient, + probability=self.probability, + landmark=self.landmark * coefficient) + + +class DetectionOnlyFaceAnalysis(FaceAnalysis): + rec_model = None + ga_model = None + + def __init__(self, det_name): + try: + self.det_model = model_zoo.get_model(det_name) + except ValueError: + file = _get_model_file(det_name) + self.det_model = face_detection.FaceDetector(file, 'net3') + + +class InsightFaceConfig: + _CTX_ID = ENV.GPU_ID + + +class FaceDetector(core.BaseFaceDetector, InsightFaceConfig): + DETECTION_MODEL_NAME = ENV.DETECTION_MODEL + IMG_LENGTH_LIMIT = ENV.IMG_LENGTH_LIMIT + _NMS = 0.4 + det_prob_threshold = 0.8 + + @cached_property + def _detection_model(self): + model = DetectionOnlyFaceAnalysis(self.DETECTION_MODEL_NAME) + model.prepare(ctx_id=self._CTX_ID, nms=self._NMS) + return model + + def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[InsightFaceBoundingBox]: + if det_prob_threshold is None: + det_prob_threshold = self.det_prob_threshold + assert 0 <= det_prob_threshold <= 1 + scaler = ImgScaler(self.IMG_LENGTH_LIMIT) + img = scaler.downscale_img(img) + results = self._detection_model.get(img, det_thresh=det_prob_threshold) + boxes = [] + for result in results: + downscaled_box_array = result.bbox.astype(np.int).flatten() + downscaled_box = InsightFaceBoundingBox(x_min=downscaled_box_array[0], + y_min=downscaled_box_array[1], + x_max=downscaled_box_array[2], + y_max=downscaled_box_array[3], + probability=result.det_score, + landmark=result.landmark) + box = downscaled_box.scaled(scaler.upscale_coefficient) + if box.probability <= det_prob_threshold: + logger.debug(f'Box Filtered out because below threshold ({det_prob_threshold}: {box})') + continue + logger.debug(f"Found: {box.dto}") + boxes.append(box) + return boxes + + def crop_face(self, img: Array3D, box: InsightFaceBoundingBox) -> Array3D: + return face_align.norm_crop(img, landmark=box.landmark) + + +class Calculator(core.BaseCalculator, InsightFaceConfig): + CALCULATION_MODEL_NAME = ENV.CALCULATION_MODEL + + def calc_embedding(self, face_img: Array3D) -> Array3D: + return self._calculation_model.get_embedding(face_img).flatten() + + @cached_property + def _calculation_model(self): + name = self.CALCULATION_MODEL_NAME + try: + model = model_zoo.get_model(name) + except ValueError: + model = face_recognition.FaceRecognition( + name, True, _get_model_file(name)) + model.prepare(ctx_id=self._CTX_ID) + return model + + +@attr.s(auto_attribs=True, frozen=True) +class GenderAgeDTO(plugin_result.PluginResultDTO): + gender: str + age: Tuple[int, int] + + +class GenderAgeDetector(core.BasePlugin, InsightFaceConfig): + name = 'gender_age' + GENDERAGE_MODEL_NAME = ENV.GENDERAGE_MODEL + GENDERS = ('female', 'male') + + def __call__(self, face_img: Array3D): + gender, age = self._genderage_model.get(face_img) + return GenderAgeDTO(gender=self.GENDERS[int(gender)], age=(age, age)) + + @cached_property + def _genderage_model(self): + name = self.GENDERAGE_MODEL_NAME + try: + model = model_zoo.get_model(name) + except ValueError: + model = face_genderage.FaceGenderage( + name, True, _get_model_file(name)) + model.prepare(ctx_id=self._CTX_ID) + return model diff --git a/embedding-calculator/src/services/facescan/plugins/rude_carnie.py b/embedding-calculator/src/services/facescan/plugins/rude_carnie.py new file mode 100644 index 0000000000..7623412af4 --- /dev/null +++ b/embedding-calculator/src/services/facescan/plugins/rude_carnie.py @@ -0,0 +1,82 @@ +# Copyright (c) 2020 the original author or authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. + +from functools import lru_cache +from typing import Tuple, Union + +import numpy as np +import tensorflow as tf + +from srcext.rude_carnie.model import inception_v3, get_checkpoint +from srcext.facenet.facenet import prewhiten + +from src.services.imgtools.types import Array3D +from src.services.facescan import core +from src.services.dto import plugin_result + +IMAGE_SIZE = 160 # TODO: change to ENV + + +@lru_cache(maxsize=2) +def _get_rude_carnie_model(type: str, labels: Tuple): + g = tf.Graph() + with g.as_default(): + sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) + + images = tf.placeholder(tf.float32, [None, IMAGE_SIZE, IMAGE_SIZE, 3]) + logits = inception_v3(len(labels), images, 1, False) + init = tf.global_variables_initializer() + + model_dir = f'/app/ml/srcext/rude_carnie/models/{type}' + model_checkpoint_path, global_step = get_checkpoint(model_dir, None, 'checkpoint') + + saver = tf.train.Saver() + saver.restore(sess, model_checkpoint_path) + softmax_output = tf.nn.softmax(logits) + + def get_value(img: Array3D) -> Tuple[Union[str, Tuple], float]: + img = np.expand_dims(prewhiten(img), 0) + output = sess.run(softmax_output, feed_dict={images:img})[0] + best_i = int(np.argmax(output)) + return labels[best_i], output[best_i] + return get_value + + +class BaseGADetector(core.BasePlugin): + LABELS: Tuple[str] + dto_class: plugin_result.GenderDTO + + def __call__(self, face_img: Array3D) -> dict: + model = _get_rude_carnie_model(self.name, self.LABELS) + value, probability = model(face_img) + return {self.name: value, f'{self.name}_probability': probability} + + +class AgeDetector(BaseGADetector): + name = 'age' + LABELS = ((0, 2), (4, 6), (8, 12), (15, 20), (25, 32), (38, 43), (48, 53), (60, 100)) + + def __call__(self, face_img: Array3D): + model = _get_rude_carnie_model(self.name, self.LABELS) + value, probability = model(face_img) + return plugin_result.AgeDTO(age=value, age_probability=probability) + +class GenderDetector(BaseGADetector): + name = 'gender' + LABELS = ('male', 'female') + + def __call__(self, face_img: Array3D): + model = _get_rude_carnie_model(self.name, self.LABELS) + value, probability = model(face_img) + return plugin_result.GenderDTO(gender=value, gender_probability=probability) diff --git a/embedding-calculator/src/services/facescan/scanner/facescanner.py b/embedding-calculator/src/services/facescan/scanner/facescanner.py index 4ed8f49b89..afbade476a 100644 --- a/embedding-calculator/src/services/facescan/scanner/facescanner.py +++ b/embedding-calculator/src/services/facescan/scanner/facescanner.py @@ -17,7 +17,6 @@ import numpy as np -from src.exceptions import MoreThanOneFaceFoundError, NoFaceFoundError from src.services.dto.bounding_box import BoundingBoxDTO from src.services.dto.scanned_face import ScannedFace from src.services.imgtools.types import Array3D @@ -44,15 +43,6 @@ def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[Bou """ Find face bounding boxes, without calculating embeddings""" raise NotImplementedError - def scan_one(self, img: Array3D, - det_prob_threshold: float = None) -> ScannedFace: - results = self.scan(img=img, det_prob_threshold=det_prob_threshold) - if len(results) > 1: - raise MoreThanOneFaceFoundError - if len(results) == 0: - raise NoFaceFoundError - return results[0] - class MockScanner(FaceScanner): ID = 'MockScanner' diff --git a/embedding-calculator/src/services/facescan/scanner/facescanners.py b/embedding-calculator/src/services/facescan/scanner/facescanners.py index 0fcf661661..49356e2b4d 100644 --- a/embedding-calculator/src/services/facescan/scanner/facescanners.py +++ b/embedding-calculator/src/services/facescan/scanner/facescanners.py @@ -27,4 +27,7 @@ id_2_face_scanner_cls = {backend.ID: backend for backend in _ALL_SCANNERS} TESTED_SCANNERS = [id_2_face_scanner_cls[k] for k in ENV_MAIN.SCANNERS] -scanner = id_2_face_scanner_cls[ENV_MAIN.SCANNER]() +# scanner = id_2_face_scanner_cls[ENV_MAIN.SCANNER]() + +from src.services.facescan.helpers import Scanner +scanner = Scanner() diff --git a/embedding-calculator/src/services/facescan/scanner/test/_scanner_cache.py b/embedding-calculator/src/services/facescan/scanner/test/_cache.py similarity index 66% rename from embedding-calculator/src/services/facescan/scanner/test/_scanner_cache.py rename to embedding-calculator/src/services/facescan/scanner/test/_cache.py index 2740a5835f..750839f98e 100644 --- a/embedding-calculator/src/services/facescan/scanner/test/_scanner_cache.py +++ b/embedding-calculator/src/services/facescan/scanner/test/_cache.py @@ -13,18 +13,9 @@ # permissions and limitations under the License. from functools import lru_cache -from src.services.imgtools.read_img import read_img +from src.services.imgtools.read_img import read_img as org_read_img @lru_cache(maxsize=None) -def get_scanner(scanner_cls): - scanner = scanner_cls() - - @lru_cache(maxsize=None) - def scan(img_path, *args, **kwargs): - img = read_img(img_path) - return scanner.scan_(img, *args, **kwargs) - - scanner.scan_ = scanner.scan - scanner.scan = scan - return scanner +def read_img(img_path): + return org_read_img(img_path) diff --git a/embedding-calculator/src/services/facescan/scanner/test/test_detector.py b/embedding-calculator/src/services/facescan/scanner/test/test_detector.py index 80e2b02a65..030611e821 100644 --- a/embedding-calculator/src/services/facescan/scanner/test/test_detector.py +++ b/embedding-calculator/src/services/facescan/scanner/test/test_detector.py @@ -13,13 +13,14 @@ # permissions and limitations under the License. import pytest +from typing import Type, Union from sample_images import IMG_DIR from sample_images.annotations import SAMPLE_IMAGES from src.services.dto.bounding_box import BoundingBoxDTO from src.services.facescan.scanner.facescanner import FaceScanner from src.services.facescan.scanner.facescanners import TESTED_SCANNERS -from src.services.facescan.scanner.test._scanner_cache import get_scanner +from src.services.facescan.scanner.test._cache import read_img from src.services.facescan.scanner.test.calculate_errors import calculate_errors from src.services.utils.pytestutils import is_sorted @@ -27,8 +28,8 @@ @pytest.mark.integration @pytest.mark.parametrize('scanner_cls', TESTED_SCANNERS) def test__given_no_faces_img__when_scanned__then_returns_no_faces(scanner_cls): - scanner: FaceScanner = get_scanner(scanner_cls) - img = IMG_DIR / '017_0.jpg' + scanner: FaceScanner = scanner_cls() + img = read_img(IMG_DIR / '017_0.jpg') result = scanner.scan(img) @@ -43,8 +44,8 @@ def test__given_5face_img__when_scanned__then_returns_5_correct_bounding_boxes_s BoundingBoxDTO(161, 36, 266, 160, 1), BoundingBoxDTO(342, 160, 437, 268, 1), BoundingBoxDTO(243, 174, 352, 309, 1)] - scanner: FaceScanner = get_scanner(scanner_cls) - img = IMG_DIR / '000_5.jpg' + scanner: FaceScanner = scanner_cls() + img = read_img(IMG_DIR / '000_5.jpg') faces = scanner.scan(img) @@ -56,8 +57,8 @@ def test__given_5face_img__when_scanned__then_returns_5_correct_bounding_boxes_s @pytest.mark.integration @pytest.mark.parametrize('scanner_cls', TESTED_SCANNERS) def test__given_threshold_set_to_1__when_scanned__then_returns_no_faces(scanner_cls): - scanner: FaceScanner = get_scanner(scanner_cls) - img = IMG_DIR / '000_5.jpg' + scanner: FaceScanner = scanner_cls() + img = read_img(IMG_DIR / '000_5.jpg') result = scanner.scan(img, det_prob_threshold=1) @@ -68,8 +69,8 @@ def test__given_threshold_set_to_1__when_scanned__then_returns_no_faces(scanner_ @pytest.mark.parametrize('scanner_cls', TESTED_SCANNERS) @pytest.mark.parametrize('row', (k for k in SAMPLE_IMAGES if k.include_to_tests)) def test__given_img__when_scanned__then_1_to_1_relationship_between_all_returned_boxes_and_faces(scanner_cls, row): - scanner: FaceScanner = get_scanner(scanner_cls) - img = IMG_DIR / row.img_name + scanner: FaceScanner = scanner_cls() + img = read_img(IMG_DIR / row.img_name) scanned_faces = scanner.scan(img) diff --git a/embedding-calculator/src/services/facescan/scanner/test/test_embedder.py b/embedding-calculator/src/services/facescan/scanner/test/test_embedder.py index e22573d6da..6ab150279b 100644 --- a/embedding-calculator/src/services/facescan/scanner/test/test_embedder.py +++ b/embedding-calculator/src/services/facescan/scanner/test/test_embedder.py @@ -17,7 +17,7 @@ from sample_images import IMG_DIR, PERSON_B, PERSON_C from src.services.facescan.scanner.facescanner import FaceScanner from src.services.facescan.scanner.facescanners import TESTED_SCANNERS -from src.services.facescan.scanner.test._scanner_cache import get_scanner +from src.services.facescan.scanner.test._cache import read_img from src.services.utils.pyutils import first_and_only DIFFERENCE_THRESHOLD = { @@ -35,9 +35,9 @@ def embeddings_are_equal(embedding1, embedding2, difference_threshold): @pytest.mark.integration @pytest.mark.parametrize('scanner_cls', TESTED_SCANNERS) def test__given_same_face_images__when_scanned__then_returns_same_embeddings(scanner_cls): - scanner: FaceScanner = get_scanner(scanner_cls) - img1 = IMG_DIR / PERSON_B[0] - img2 = IMG_DIR / PERSON_B[1] + scanner: FaceScanner = scanner_cls() + img1 = read_img(IMG_DIR / PERSON_B[0]) + img2 = read_img(IMG_DIR / PERSON_B[1]) emb1 = first_and_only(scanner.scan(img1)).embedding emb2 = first_and_only(scanner.scan(img2)).embedding @@ -48,9 +48,9 @@ def test__given_same_face_images__when_scanned__then_returns_same_embeddings(sca @pytest.mark.integration @pytest.mark.parametrize('scanner_cls', TESTED_SCANNERS) def test__given_diff_face_images__when_scanned__then_returns_diff_embeddings(scanner_cls): - scanner: FaceScanner = get_scanner(scanner_cls) - img1 = IMG_DIR / PERSON_B[0] - img2 = IMG_DIR / PERSON_C[0] + scanner: FaceScanner = scanner_cls() + img1 = read_img(IMG_DIR / PERSON_B[0]) + img2 = read_img(IMG_DIR / PERSON_C[0]) emb1 = first_and_only(scanner.scan(img1)).embedding emb2 = first_and_only(scanner.scan(img2)).embedding @@ -60,6 +60,7 @@ def test__given_diff_face_images__when_scanned__then_returns_diff_embeddings(sca @pytest.mark.integration @pytest.mark.parametrize('scanner_cls', TESTED_SCANNERS) def test__size_of_embeddings(scanner_cls): - scanner: FaceScanner = get_scanner(scanner_cls) - emb = first_and_only(scanner.scan(IMG_DIR / '007_B.jpg')).embedding + scanner: FaceScanner = scanner_cls() + img = read_img(IMG_DIR / PERSON_B[0]) + emb = first_and_only(scanner.scan(img)).embedding assert len(emb) == 512 diff --git a/embedding-calculator/src/services/flask_/constants.py b/embedding-calculator/src/services/flask_/constants.py index 2c217d6f98..049c1a1649 100644 --- a/embedding-calculator/src/services/flask_/constants.py +++ b/embedding-calculator/src/services/flask_/constants.py @@ -18,3 +18,5 @@ class ARG: LIMIT = 'limit' DET_PROB_THRESHOLD = 'det_prob_threshold' + DET_PLUGIN = 'det_plugin' + FACE_PLUGINS = 'face_plugins' diff --git a/embedding-calculator/srcext/rude_carnie b/embedding-calculator/srcext/rude_carnie new file mode 160000 index 0000000000..d70127f67e --- /dev/null +++ b/embedding-calculator/srcext/rude_carnie @@ -0,0 +1 @@ +Subproject commit d70127f67ef53c35a5588cf4ea06adcc35c390ba