From 834513b595eb0cec92b02ac4f0aff04bb648d54e Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Wed, 30 Oct 2024 07:46:42 -0600 Subject: [PATCH 1/8] WIP --- child_lab_framework/demo_sequential.py | 162 +++++++------ child_lab_framework/task/emotions/__init__.py | 3 + child_lab_framework/task/emotions/emotions.py | 95 ++++++++ poetry.lock | 219 +++++++++++++++++- pyproject.toml | 1 + 5 files changed, 405 insertions(+), 75 deletions(-) create mode 100644 child_lab_framework/task/emotions/__init__.py create mode 100644 child_lab_framework/task/emotions/emotions.py diff --git a/child_lab_framework/demo_sequential.py b/child_lab_framework/demo_sequential.py index 3f855c7..577d621 100644 --- a/child_lab_framework/demo_sequential.py +++ b/child_lab_framework/demo_sequential.py @@ -3,6 +3,8 @@ import torch +from child_lab_framework.task import emotions + from .core.video import Format, Perspective, Reader, Writer from .logging import Logger from .task import depth, face, gaze, pose @@ -20,33 +22,33 @@ def main() -> None: gpu = torch.device('mps') ceiling_reader = Reader( - 'dev/data/aruco_cubic_ultra_short/ceiling.mp4', + '/Users/wojciechbarczynski/child-lab-framework/dev/data/aruco_cubic_ultra_short/ceiling.mp4', perspective=Perspective.CEILING, batch_size=BATCH_SIZE, ) window_left_reader = Reader( - 'dev/data/aruco_cubic_ultra_short/window_left.mp4', + '/Users/wojciechbarczynski/child-lab-framework/dev/data/aruco_cubic_ultra_short/window_left.mp4', perspective=Perspective.WINDOW_LEFT, batch_size=BATCH_SIZE, like=ceiling_reader.properties, ) window_right_reader = Reader( - 'dev/data/aruco_cubic_ultra_short/window_right.mp4', + '/Users/wojciechbarczynski/child-lab-framework/dev/data/aruco_cubic_ultra_short/window_right.mp4', perspective=Perspective.WINDOW_RIGHT, batch_size=BATCH_SIZE, like=ceiling_reader.properties, ) - depth_estimator = depth.Estimator(executor, gpu, input=ceiling_reader.properties) + # depth_estimator = depth.Estimator(executor, gpu, input=ceiling_reader.properties) - transformation_estimator = transformation.heuristic.Estimator( - executor, - window_left_reader.properties, - ceiling_reader.properties, - keypoint_threshold=0.35, - ) + # transformation_estimator = transformation.heuristic.Estimator( + # executor, + # window_left_reader.properties, + # ceiling_reader.properties, + # keypoint_threshold=0.35, + # ) pose_estimator = pose.Estimator( executor, @@ -62,22 +64,25 @@ def main() -> None: threshold=0.1, ) - window_left_gaze_estimator = gaze.Estimator( - executor, - input=window_left_reader.properties, - ) + emotions_estimator_left = emotions.Estimator(executor) + emotions_estimator_right = emotions.Estimator(executor) - window_right_gaze_estimator = gaze.Estimator( - executor, - input=window_right_reader.properties, - ) + # window_left_gaze_estimator = gaze.Estimator( + # executor, + # input=window_left_reader.properties, + # ) - ceiling_gaze_estimator = gaze.ceiling_projection.Estimator( - executor, - ceiling_reader.properties, - window_left_reader.properties, - window_right_reader.properties, - ) + # window_right_gaze_estimator = gaze.Estimator( + # executor, + # input=window_right_reader.properties, + # ) + + # ceiling_gaze_estimator = gaze.ceiling_projection.Estimator( + # executor, + # ceiling_reader.properties, + # window_left_reader.properties, + # window_right_reader.properties, + # ) # social_distance_estimator = social_distance.Estimator(executor) # social_distance_logger = social_distance.FileLogger('dev/output/distance.csv') @@ -89,23 +94,25 @@ def main() -> None: ) ceiling_writer = Writer( - 'dev/output/sequential/ceiling.mp4', + 'dev/output/ceiling.mp4', ceiling_reader.properties, output_format=Format.MP4, ) window_left_writer = Writer( - 'dev/output/sequential/window_left.mp4', + 'dev/output/window_left.mp4', window_left_reader.properties, output_format=Format.MP4, ) window_right_writer = Writer( - 'dev/output/sequential/window_right.mp4', + 'dev/output/window_right.mp4', window_right_reader.properties, output_format=Format.MP4, ) + print('Starting sequential processing') + while True: ceiling_frames = ceiling_reader.read_batch() if ceiling_frames is None: @@ -119,36 +126,36 @@ def main() -> None: if window_right_frames is None: break - n_frames = len(ceiling_frames) + # n_frames = len(ceiling_frames) ceiling_poses = pose_estimator.predict_batch(ceiling_frames) window_left_poses = pose_estimator.predict_batch(window_left_frames) window_right_poses = pose_estimator.predict_batch(window_right_frames) - ceiling_depth = depth_estimator.predict(ceiling_frames[0]) - ceiling_depths = [ceiling_depth for _ in range(n_frames)] - - window_left_to_ceiling = ( - transformation_estimator.predict_batch( - ceiling_poses, - window_left_poses, - ceiling_depths, - [None for _ in range(n_frames)], # type: ignore # safe to pass - ) - if ceiling_poses is not None and window_left_poses is not None - else None - ) - - window_right_to_ceiling = ( - transformation_estimator.predict_batch( - ceiling_poses, - window_right_poses, - ceiling_depths, - [None for _ in range(n_frames)], # type: ignore # safe to pass - ) - if ceiling_poses is not None and window_right_poses is not None - else None - ) + # ceiling_depth = depth_estimator.predict(ceiling_frames[0]) + # ceiling_depths = [ceiling_depth for _ in range(n_frames)] + + # window_left_to_ceiling = ( + # transformation_estimator.predict_batch( + # ceiling_poses, + # window_left_poses, + # ceiling_depths, + # [None for _ in range(n_frames)], # type: ignore # safe to pass + # ) + # if ceiling_poses is not None and window_left_poses is not None + # else None + # ) + + # window_right_to_ceiling = ( + # transformation_estimator.predict_batch( + # ceiling_poses, + # window_right_poses, + # ceiling_depths, + # [None for _ in range(n_frames)], # type: ignore # safe to pass + # ) + # if ceiling_poses is not None and window_right_poses is not None + # else None + # ) if ceiling_poses is None: Logger.error('ceiling_poses == None') @@ -177,34 +184,41 @@ def main() -> None: if window_right_faces is None: Logger.error('window_right_faces == None') - window_left_gazes = ( - window_left_gaze_estimator.predict_batch( - window_left_frames, window_left_faces - ) - if window_left_faces is not None - else None - ) - - window_right_gazes = ( - window_right_gaze_estimator.predict_batch( - window_right_frames, window_right_faces - ) - if window_right_faces is not None - else None - ) + # window_left_gazes = ( + # # window_left_gaze_estimator.predict_batch( + # # window_left_frames, window_left_faces + # # ) + # None + # if window_left_faces is not None + # else None + # ) + + # window_right_gazes = ( + # # window_right_gaze_estimator.predict_batch( + # # window_right_frames, window_right_faces + # # ) + # None + # if window_right_faces is not None + # else None + # ) ceiling_gazes = ( - ceiling_gaze_estimator.predict_batch( - ceiling_poses, - window_left_gazes, - window_right_gazes, - window_left_to_ceiling, - window_right_to_ceiling, - ) + # ceiling_gaze_estimator.predict_batch( + # ceiling_poses, + # window_left_gazes, + # window_right_gazes, + # window_left_to_ceiling, + # window_right_to_ceiling, + # ) + None if ceiling_poses is not None else None ) + window_left_emotions = emotions_estimator_left.predict_batch(window_left_frames, window_left_faces) + window_right_emotions = emotions_estimator_right.predict_batch(window_right_frames, window_right_faces) + + ceiling_annotated_frames = visualizer.annotate_batch( ceiling_frames, ceiling_poses, diff --git a/child_lab_framework/task/emotions/__init__.py b/child_lab_framework/task/emotions/__init__.py new file mode 100644 index 0000000..d8ced8b --- /dev/null +++ b/child_lab_framework/task/emotions/__init__.py @@ -0,0 +1,3 @@ +from .emotions import Estimator, Result + +__all__ = ['Estimator', 'Result'] \ No newline at end of file diff --git a/child_lab_framework/task/emotions/emotions.py b/child_lab_framework/task/emotions/emotions.py new file mode 100644 index 0000000..f4445b4 --- /dev/null +++ b/child_lab_framework/task/emotions/emotions.py @@ -0,0 +1,95 @@ +import asyncio +from deepface import DeepFace +from concurrent.futures import ThreadPoolExecutor +from itertools import repeat, starmap + +from child_lab_framework.core.sequence import imputed_with_reference_inplace +from child_lab_framework.task import face +from ...core.video import Frame +from ...typing.stream import Fiber + +type Input = tuple[ + list[Frame | None] | None, + list[face.Result | None] | None, +] + +class Result: + n_detections: int + emotions: list[float] + + def __init__(self, n_detections: int, emotions: list[float]) -> None: + self.n_detections = n_detections + self.emotions = emotions + +class Estimator: + executor: ThreadPoolExecutor + + def __init__(self, executor: ThreadPoolExecutor) -> None: + self.executor = executor + + def predict(self, frame: Frame, faces: face.Result | None) -> Result: + n_detections = 0 + face_emotions = [] + for face in faces.boxes: + analysis = DeepFace.analyze(frame, actions=['emotion'], enforce_detection=False) + emotion = score_emotions(analysis[0]) + n_detections += 1 + face_emotions.append(emotion) + + return Result(n_detections, face_emotions) + + def predict_batch( + self, + frames: list[Frame], + faces: list[face.Result | None], + ) -> list[Result] | None: + return imputed_with_reference_inplace( + list(starmap(self.predict, zip(frames, faces))) + ) + + async def stream( + self, + ) -> Fiber[list[Frame | None] | None, list[Result | None] | None]: + loop = asyncio.get_running_loop() + executor = self.executor + + results: list[Result | None] | None = None + + while True: + match (yield results): + case ( + list(frames), + faces, + ): + results = await loop.run_in_executor( + executor, + lambda: list( + starmap( + self.__predict, + zip( + frames, + faces or repeat(None) + ), + ) + ), + ) + + case _: + results = None + +def score_emotions(emotions): + # Most of the time, "angry" and "fear" are similar to "neutral" in the reality + scores = { + 'angry': -0.05, + 'disgust': 0, + 'fear': -0.07, + 'happy': 1, + 'sad': -1, + 'surprise': 0, + 'neutral': 0, + } + val = 0 + for emotion, score in scores.items(): + val += emotions['emotion'][emotion] * score + + return val \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index ad60eaa..f9155e4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -63,6 +63,38 @@ docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphi tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +description = "Screen-scraping library" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + +[package.dependencies] +soupsieve = ">1.2" + +[package.extras] +cchardet = ["cchardet"] +chardet = ["chardet"] +charset-normalizer = ["charset-normalizer"] +html5lib = ["html5lib"] +lxml = ["lxml"] + +[[package]] +name = "blinker" +version = "1.8.2" +description = "Fast, simple object-to-object and broadcast signaling" +optional = false +python-versions = ">=3.8" +files = [ + {file = "blinker-1.8.2-py3-none-any.whl", hash = "sha256:1779309f71bf239144b9399d06ae925637cf6634cf6bd131104184531bf67c01"}, + {file = "blinker-1.8.2.tar.gz", hash = "sha256:8f77b09d3bf7c795e969e9486f39c2c5e9c39d4ee07424be2bc594ece9642d83"}, +] + [[package]] name = "certifi" version = "2024.8.30" @@ -267,6 +299,20 @@ files = [ {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, ] +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "colorama" version = "0.4.6" @@ -377,6 +423,34 @@ files = [ docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] tests = ["pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "deepface" +version = "0.0.93" +description = "A Lightweight Face Recognition and Facial Attribute Analysis Framework (Age, Gender, Emotion, Race) for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "deepface-0.0.93-py3-none-any.whl", hash = "sha256:27043e1aa5df05a060bcfe1743409075c66f6f1de86a592ba0cdac79ac9e7987"}, + {file = "deepface-0.0.93.tar.gz", hash = "sha256:7f5fc6306a3a07ee6c529b03571e64fe53d9f259e1d4091f5e28386264962b92"}, +] + +[package.dependencies] +fire = ">=0.4.0" +Flask = ">=1.1.2" +flask-cors = ">=4.0.1" +gdown = ">=3.10.1" +gunicorn = ">=20.1.0" +keras = ">=2.2.0" +mtcnn = ">=0.1.0" +numpy = ">=1.14.0" +opencv-python = ">=4.5.5.64" +pandas = ">=0.23.4" +Pillow = ">=5.2.0" +requests = ">=2.27.1" +retina-face = ">=0.0.1" +tensorflow = ">=1.9.0" +tqdm = ">=4.30.0" + [[package]] name = "depth-pro" version = "0.1.0" @@ -450,6 +524,55 @@ docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2. testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] typing = ["typing-extensions (>=4.12.2)"] +[[package]] +name = "fire" +version = "0.7.0" +description = "A library for automatically generating command line interfaces." +optional = false +python-versions = "*" +files = [ + {file = "fire-0.7.0.tar.gz", hash = "sha256:961550f07936eaf65ad1dc8360f2b2bf8408fad46abbfa4d2a3794f8d2a95cdf"}, +] + +[package.dependencies] +termcolor = "*" + +[[package]] +name = "flask" +version = "3.0.3" +description = "A simple framework for building complex web applications." +optional = false +python-versions = ">=3.8" +files = [ + {file = "flask-3.0.3-py3-none-any.whl", hash = "sha256:34e815dfaa43340d1d15a5c3a02b8476004037eb4840b34910c6e21679d288f3"}, + {file = "flask-3.0.3.tar.gz", hash = "sha256:ceb27b0af3823ea2737928a4d99d125a06175b8512c445cbd9a9ce200ef76842"}, +] + +[package.dependencies] +blinker = ">=1.6.2" +click = ">=8.1.3" +itsdangerous = ">=2.1.2" +Jinja2 = ">=3.1.2" +Werkzeug = ">=3.0.0" + +[package.extras] +async = ["asgiref (>=3.2)"] +dotenv = ["python-dotenv"] + +[[package]] +name = "flask-cors" +version = "5.0.0" +description = "A Flask extension adding a decorator for CORS support" +optional = false +python-versions = "*" +files = [ + {file = "Flask_Cors-5.0.0-py2.py3-none-any.whl", hash = "sha256:b9e307d082a9261c100d8fb0ba909eec6a228ed1b60a8315fd85f783d61910bc"}, + {file = "flask_cors-5.0.0.tar.gz", hash = "sha256:5aadb4b950c4e93745034594d9f3ea6591f734bb3662e16e255ffbf5e89c88ef"}, +] + +[package.dependencies] +Flask = ">=0.9" + [[package]] name = "flatbuffers" version = "24.3.25" @@ -582,6 +705,26 @@ files = [ {file = "gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb"}, ] +[[package]] +name = "gdown" +version = "5.2.0" +description = "Google Drive Public File/Folder Downloader" +optional = false +python-versions = ">=3.8" +files = [ + {file = "gdown-5.2.0-py3-none-any.whl", hash = "sha256:33083832d82b1101bdd0e9df3edd0fbc0e1c5f14c9d8c38d2a35bf1683b526d6"}, + {file = "gdown-5.2.0.tar.gz", hash = "sha256:2145165062d85520a3cd98b356c9ed522c5e7984d408535409fd46f94defc787"}, +] + +[package.dependencies] +beautifulsoup4 = "*" +filelock = "*" +requests = {version = "*", extras = ["socks"]} +tqdm = "*" + +[package.extras] +test = ["build", "mypy", "pytest", "pytest-xdist", "ruff", "twine", "types-requests", "types-setuptools"] + [[package]] name = "google-pasta" version = "0.2.0" @@ -664,6 +807,27 @@ files = [ [package.extras] protobuf = ["grpcio-tools (>=1.67.0)"] +[[package]] +name = "gunicorn" +version = "23.0.0" +description = "WSGI HTTP Server for UNIX" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d"}, + {file = "gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec"}, +] + +[package.dependencies] +packaging = "*" + +[package.extras] +eventlet = ["eventlet (>=0.24.1,!=0.36.0)"] +gevent = ["gevent (>=1.4.0)"] +setproctitle = ["setproctitle"] +testing = ["coverage", "eventlet", "gevent", "pytest", "pytest-cov"] +tornado = ["tornado (>=0.2)"] + [[package]] name = "h5py" version = "3.12.1" @@ -767,6 +931,17 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +[[package]] +name = "itsdangerous" +version = "2.2.0" +description = "Safely pass data to untrusted environments and back." +optional = false +python-versions = ">=3.8" +files = [ + {file = "itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef"}, + {file = "itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173"}, +] + [[package]] name = "jax" version = "0.4.35" @@ -2138,6 +2313,18 @@ files = [ [package.extras] diagrams = ["jinja2", "railroad-diagrams"] +[[package]] +name = "pysocks" +version = "1.7.1" +description = "A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "PySocks-1.7.1-py27-none-any.whl", hash = "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299"}, + {file = "PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5"}, + {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"}, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2267,12 +2454,31 @@ files = [ certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" +PySocks = {version = ">=1.5.6,<1.5.7 || >1.5.7", optional = true, markers = "extra == \"socks\""} urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "retina-face" +version = "0.0.17" +description = "RetinaFace: Deep Face Detection Framework in TensorFlow for Python" +optional = false +python-versions = ">=3.5.5" +files = [ + {file = "retina-face-0.0.17.tar.gz", hash = "sha256:7532b136ed01fe9a8cba8dfbc5a046dd6fb1214b1a83e57f3210bd145a91cd73"}, + {file = "retina_face-0.0.17-py3-none-any.whl", hash = "sha256:b43fdac4078678b9d8bc45b88a7090f05d81c44e1e10710e6c16d703bb7add41"}, +] + +[package.dependencies] +gdown = ">=3.10.1" +numpy = ">=1.14.0" +opencv-python = ">=3.4.4" +Pillow = ">=5.2.0" +tensorflow = ">=1.9.0" + [[package]] name = "rich" version = "13.9.3" @@ -2572,6 +2778,17 @@ CFFI = ">=1.0" [package.extras] numpy = ["NumPy"] +[[package]] +name = "soupsieve" +version = "2.6" +description = "A modern CSS selector implementation for Beautiful Soup." +optional = false +python-versions = ">=3.8" +files = [ + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, +] + [[package]] name = "sympy" version = "1.13.1" @@ -3135,4 +3352,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "502cd6985714a46f2941ca82be24b94ef7b9ec7df3d3fa0f0b0d038385b65701" +content-hash = "746631e1ebed35c316b6840e25da0c1ce48b43da19fa63361392fccc7ae5e56b" diff --git a/pyproject.toml b/pyproject.toml index 8dcf456..3dfd891 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ ml-dtypes = "0.4.0" # installi tensorflow = "^2.17.0" mini-face = ">=0.1.0" depth-pro = { git = "https://github.com/child-lab-uj/depth-pro.git" } +deepface = "^0.0.93" [tool.poetry.group.dev.dependencies] poethepoet = "^0.26.1" From b972f3c7a5ecd88730aba40dd73186ed6e0cdd30 Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Wed, 6 Nov 2024 18:41:44 -0500 Subject: [PATCH 2/8] Uncomment --- child_lab_framework/demo_sequential.py | 143 +++++++++--------- child_lab_framework/task/emotions/emotions.py | 26 ++-- .../task/visualization/visualization.py | 15 +- 3 files changed, 102 insertions(+), 82 deletions(-) diff --git a/child_lab_framework/demo_sequential.py b/child_lab_framework/demo_sequential.py index 577d621..4639a02 100644 --- a/child_lab_framework/demo_sequential.py +++ b/child_lab_framework/demo_sequential.py @@ -41,14 +41,14 @@ def main() -> None: like=ceiling_reader.properties, ) - # depth_estimator = depth.Estimator(executor, gpu, input=ceiling_reader.properties) + depth_estimator = depth.Estimator(executor, gpu, input=ceiling_reader.properties) - # transformation_estimator = transformation.heuristic.Estimator( - # executor, - # window_left_reader.properties, - # ceiling_reader.properties, - # keypoint_threshold=0.35, - # ) + transformation_estimator = transformation.heuristic.Estimator( + executor, + window_left_reader.properties, + ceiling_reader.properties, + keypoint_threshold=0.35, + ) pose_estimator = pose.Estimator( executor, @@ -67,22 +67,22 @@ def main() -> None: emotions_estimator_left = emotions.Estimator(executor) emotions_estimator_right = emotions.Estimator(executor) - # window_left_gaze_estimator = gaze.Estimator( - # executor, - # input=window_left_reader.properties, - # ) + window_left_gaze_estimator = gaze.Estimator( + executor, + input=window_left_reader.properties, + ) - # window_right_gaze_estimator = gaze.Estimator( - # executor, - # input=window_right_reader.properties, - # ) + window_right_gaze_estimator = gaze.Estimator( + executor, + input=window_right_reader.properties, + ) - # ceiling_gaze_estimator = gaze.ceiling_projection.Estimator( - # executor, - # ceiling_reader.properties, - # window_left_reader.properties, - # window_right_reader.properties, - # ) + ceiling_gaze_estimator = gaze.ceiling_projection.Estimator( + executor, + ceiling_reader.properties, + window_left_reader.properties, + window_right_reader.properties, + ) # social_distance_estimator = social_distance.Estimator(executor) # social_distance_logger = social_distance.FileLogger('dev/output/distance.csv') @@ -126,36 +126,36 @@ def main() -> None: if window_right_frames is None: break - # n_frames = len(ceiling_frames) + n_frames = len(ceiling_frames) ceiling_poses = pose_estimator.predict_batch(ceiling_frames) window_left_poses = pose_estimator.predict_batch(window_left_frames) window_right_poses = pose_estimator.predict_batch(window_right_frames) - # ceiling_depth = depth_estimator.predict(ceiling_frames[0]) - # ceiling_depths = [ceiling_depth for _ in range(n_frames)] - - # window_left_to_ceiling = ( - # transformation_estimator.predict_batch( - # ceiling_poses, - # window_left_poses, - # ceiling_depths, - # [None for _ in range(n_frames)], # type: ignore # safe to pass - # ) - # if ceiling_poses is not None and window_left_poses is not None - # else None - # ) - - # window_right_to_ceiling = ( - # transformation_estimator.predict_batch( - # ceiling_poses, - # window_right_poses, - # ceiling_depths, - # [None for _ in range(n_frames)], # type: ignore # safe to pass - # ) - # if ceiling_poses is not None and window_right_poses is not None - # else None - # ) + ceiling_depth = depth_estimator.predict(ceiling_frames[0]) + ceiling_depths = [ceiling_depth for _ in range(n_frames)] + + window_left_to_ceiling = ( + transformation_estimator.predict_batch( + ceiling_poses, + window_left_poses, + ceiling_depths, + [None for _ in range(n_frames)], # type: ignore # safe to pass + ) + if ceiling_poses is not None and window_left_poses is not None + else None + ) + + window_right_to_ceiling = ( + transformation_estimator.predict_batch( + ceiling_poses, + window_right_poses, + ceiling_depths, + [None for _ in range(n_frames)], # type: ignore # safe to pass + ) + if ceiling_poses is not None and window_right_poses is not None + else None + ) if ceiling_poses is None: Logger.error('ceiling_poses == None') @@ -184,33 +184,30 @@ def main() -> None: if window_right_faces is None: Logger.error('window_right_faces == None') - # window_left_gazes = ( - # # window_left_gaze_estimator.predict_batch( - # # window_left_frames, window_left_faces - # # ) - # None - # if window_left_faces is not None - # else None - # ) - - # window_right_gazes = ( - # # window_right_gaze_estimator.predict_batch( - # # window_right_frames, window_right_faces - # # ) - # None - # if window_right_faces is not None - # else None - # ) + window_left_gazes = ( + window_left_gaze_estimator.predict_batch( + window_left_frames, window_left_faces + ) + if window_left_faces is not None + else None + ) + + window_right_gazes = ( + window_right_gaze_estimator.predict_batch( + window_right_frames, window_right_faces + ) + if window_right_faces is not None + else None + ) ceiling_gazes = ( - # ceiling_gaze_estimator.predict_batch( - # ceiling_poses, - # window_left_gazes, - # window_right_gazes, - # window_left_to_ceiling, - # window_right_to_ceiling, - # ) - None + ceiling_gaze_estimator.predict_batch( + ceiling_poses, + window_left_gazes, + window_right_gazes, + window_left_to_ceiling, + window_right_to_ceiling, + ) if ceiling_poses is not None else None ) @@ -218,12 +215,12 @@ def main() -> None: window_left_emotions = emotions_estimator_left.predict_batch(window_left_frames, window_left_faces) window_right_emotions = emotions_estimator_right.predict_batch(window_right_frames, window_right_faces) - ceiling_annotated_frames = visualizer.annotate_batch( ceiling_frames, ceiling_poses, None, ceiling_gazes, + None, ) window_left_annotated_frames = visualizer.annotate_batch( @@ -231,6 +228,7 @@ def main() -> None: window_left_poses, window_left_faces, None, + window_left_emotions ) window_right_annotated_frames = visualizer.annotate_batch( @@ -238,6 +236,7 @@ def main() -> None: window_right_poses, window_right_faces, None, + window_right_emotions ) ceiling_writer.write_batch(ceiling_annotated_frames) diff --git a/child_lab_framework/task/emotions/emotions.py b/child_lab_framework/task/emotions/emotions.py index f4445b4..10fd682 100644 --- a/child_lab_framework/task/emotions/emotions.py +++ b/child_lab_framework/task/emotions/emotions.py @@ -7,6 +7,7 @@ from child_lab_framework.task import face from ...core.video import Frame from ...typing.stream import Fiber +from ...typing.array import FloatArray2 type Input = tuple[ list[Frame | None] | None, @@ -14,12 +15,12 @@ ] class Result: - n_detections: int emotions: list[float] + boxes: list[FloatArray2] - def __init__(self, n_detections: int, emotions: list[float]) -> None: - self.n_detections = n_detections + def __init__(self, emotions: list[float], boxes: list[FloatArray2]) -> None: self.emotions = emotions + self.boxes = boxes class Estimator: executor: ThreadPoolExecutor @@ -28,15 +29,22 @@ def __init__(self, executor: ThreadPoolExecutor) -> None: self.executor = executor def predict(self, frame: Frame, faces: face.Result | None) -> Result: - n_detections = 0 face_emotions = [] - for face in faces.boxes: - analysis = DeepFace.analyze(frame, actions=['emotion'], enforce_detection=False) + boxes = [] + frame_height, frame_width, _ = frame.shape + for face_box in faces.boxes: + x_min, y_min, x_max, y_max = face_box + x_min = max(x_min - 50, 0) + x_max = min(x_max + 50, frame_width) + y_min = max(y_min - 50, 0) + y_max = min(y_max + 50, frame_height) + cropped_frame = frame[y_min:y_max, x_min:x_max] + analysis = DeepFace.analyze(cropped_frame, actions=['emotion'], enforce_detection=False) emotion = score_emotions(analysis[0]) - n_detections += 1 face_emotions.append(emotion) - - return Result(n_detections, face_emotions) + boxes.append(face_box) + + return Result(face_emotions, boxes) def predict_batch( self, diff --git a/child_lab_framework/task/visualization/visualization.py b/child_lab_framework/task/visualization/visualization.py index 094090e..3a3be89 100644 --- a/child_lab_framework/task/visualization/visualization.py +++ b/child_lab_framework/task/visualization/visualization.py @@ -3,12 +3,13 @@ from itertools import repeat, starmap import cv2 +import cv2.text import numpy as np from ...core.video import Frame, Properties from ...typing.array import FloatArray1, FloatArray2, IntArray1 from ...typing.stream import Fiber -from .. import face, pose +from .. import face, pose, emotions from ..gaze import ceiling_projection from ..pose.keypoint import YOLO_SKELETON @@ -122,6 +123,12 @@ def __draw_face_box(self, frame: Frame, result: face.Result) -> Frame: cv2.rectangle(frame, (x1, y1), (x2, y2), color, thickness) return frame + + def __draw_emotions_text(self, frame: Frame, result: emotions.Result) -> Frame: + color = self.FACE_BOUNDING_BOX_COLOR + for (value, box) in zip(result.emotions, result.boxes): + cv2.putText(frame, str(value), [box[0], box[3]], cv2.FONT_HERSHEY_SIMPLEX, 0.9, color, 2) + return frame def __annotate_safe( self, @@ -129,6 +136,7 @@ def __annotate_safe( poses: pose.Result | None, faces: face.Result | None, gazes: ceiling_projection.Result | None, + emotions: emotions.Result | None ) -> Frame: out = frame.copy() out.flags.writeable = True @@ -142,6 +150,9 @@ def __annotate_safe( if gazes is not None: out = self.__draw_gaze_estimation(out, gazes) + + if emotions is not None: + out = self.__draw_emotions_text(out, emotions) return out @@ -151,6 +162,7 @@ def annotate_batch( poses: list[pose.Result] | None, faces: list[face.Result] | None, gazes: list[ceiling_projection.Result] | None, + emotions: list[emotions.Result] | None ) -> list[Frame]: return list( starmap( @@ -160,6 +172,7 @@ def annotate_batch( poses or repeat(None), faces or repeat(None), gazes or repeat(None), + emotions or repeat(None) ), ) ) From 25fac352ba59aaca4e7641fc49d00a431b9707f3 Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Wed, 6 Nov 2024 18:45:04 -0500 Subject: [PATCH 3/8] Fix inconsistencies --- child_lab_framework/demo_sequential.py | 16 +++++++--------- child_lab_framework/task/emotions/__init__.py | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/child_lab_framework/demo_sequential.py b/child_lab_framework/demo_sequential.py index 4639a02..87471c9 100644 --- a/child_lab_framework/demo_sequential.py +++ b/child_lab_framework/demo_sequential.py @@ -3,11 +3,9 @@ import torch -from child_lab_framework.task import emotions - from .core.video import Format, Perspective, Reader, Writer from .logging import Logger -from .task import depth, face, gaze, pose +from .task import depth, face, gaze, pose, emotions from .task.camera import transformation from .task.visualization import Visualizer @@ -22,20 +20,20 @@ def main() -> None: gpu = torch.device('mps') ceiling_reader = Reader( - '/Users/wojciechbarczynski/child-lab-framework/dev/data/aruco_cubic_ultra_short/ceiling.mp4', + 'dev/data/aruco_cubic_ultra_short/ceiling.mp4', perspective=Perspective.CEILING, batch_size=BATCH_SIZE, ) window_left_reader = Reader( - '/Users/wojciechbarczynski/child-lab-framework/dev/data/aruco_cubic_ultra_short/window_left.mp4', + 'dev/data/aruco_cubic_ultra_short/window_left.mp4', perspective=Perspective.WINDOW_LEFT, batch_size=BATCH_SIZE, like=ceiling_reader.properties, ) window_right_reader = Reader( - '/Users/wojciechbarczynski/child-lab-framework/dev/data/aruco_cubic_ultra_short/window_right.mp4', + 'dev/data/aruco_cubic_ultra_short/window_right.mp4', perspective=Perspective.WINDOW_RIGHT, batch_size=BATCH_SIZE, like=ceiling_reader.properties, @@ -94,19 +92,19 @@ def main() -> None: ) ceiling_writer = Writer( - 'dev/output/ceiling.mp4', + 'dev/output/sequential/ceiling.mp4', ceiling_reader.properties, output_format=Format.MP4, ) window_left_writer = Writer( - 'dev/output/window_left.mp4', + 'dev/output/sequential/window_left.mp4', window_left_reader.properties, output_format=Format.MP4, ) window_right_writer = Writer( - 'dev/output/window_right.mp4', + 'dev/output/sequential/window_right.mp4', window_right_reader.properties, output_format=Format.MP4, ) diff --git a/child_lab_framework/task/emotions/__init__.py b/child_lab_framework/task/emotions/__init__.py index d8ced8b..8057aae 100644 --- a/child_lab_framework/task/emotions/__init__.py +++ b/child_lab_framework/task/emotions/__init__.py @@ -1,3 +1,3 @@ from .emotions import Estimator, Result -__all__ = ['Estimator', 'Result'] \ No newline at end of file +__all__ = ['Estimator', 'Result'] From 2ed9ce3b1a46a50421253ac1fb35e50b8d42fe5b Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Wed, 6 Nov 2024 18:52:52 -0500 Subject: [PATCH 4/8] Typing --- child_lab_framework/task/emotions/emotions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/child_lab_framework/task/emotions/emotions.py b/child_lab_framework/task/emotions/emotions.py index 10fd682..3a47256 100644 --- a/child_lab_framework/task/emotions/emotions.py +++ b/child_lab_framework/task/emotions/emotions.py @@ -1,4 +1,5 @@ import asyncio +from typing import Dict, List from deepface import DeepFace from concurrent.futures import ThreadPoolExecutor from itertools import repeat, starmap @@ -85,7 +86,7 @@ async def stream( case _: results = None -def score_emotions(emotions): +def score_emotions(emotions: List[Dict[str, float]]) -> float: # Most of the time, "angry" and "fear" are similar to "neutral" in the reality scores = { 'angry': -0.05, From f035b66e4401c48b3d7ab54db318628cf2504a85 Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Wed, 6 Nov 2024 18:54:08 -0500 Subject: [PATCH 5/8] Rename emotions -> emotion --- child_lab_framework/demo_sequential.py | 6 +++--- child_lab_framework/task/emotion/__init__.py | 3 +++ .../task/{emotions/emotions.py => emotion/emotion.py} | 0 child_lab_framework/task/emotions/__init__.py | 3 --- child_lab_framework/task/visualization/visualization.py | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 child_lab_framework/task/emotion/__init__.py rename child_lab_framework/task/{emotions/emotions.py => emotion/emotion.py} (100%) delete mode 100644 child_lab_framework/task/emotions/__init__.py diff --git a/child_lab_framework/demo_sequential.py b/child_lab_framework/demo_sequential.py index 87471c9..c5d337b 100644 --- a/child_lab_framework/demo_sequential.py +++ b/child_lab_framework/demo_sequential.py @@ -5,7 +5,7 @@ from .core.video import Format, Perspective, Reader, Writer from .logging import Logger -from .task import depth, face, gaze, pose, emotions +from .task import depth, face, gaze, pose, emotion from .task.camera import transformation from .task.visualization import Visualizer @@ -62,8 +62,8 @@ def main() -> None: threshold=0.1, ) - emotions_estimator_left = emotions.Estimator(executor) - emotions_estimator_right = emotions.Estimator(executor) + emotions_estimator_left = emotion.Estimator(executor) + emotions_estimator_right = emotion.Estimator(executor) window_left_gaze_estimator = gaze.Estimator( executor, diff --git a/child_lab_framework/task/emotion/__init__.py b/child_lab_framework/task/emotion/__init__.py new file mode 100644 index 0000000..3c8fd40 --- /dev/null +++ b/child_lab_framework/task/emotion/__init__.py @@ -0,0 +1,3 @@ +from .emotion import Estimator, Result + +__all__ = ['Estimator', 'Result'] diff --git a/child_lab_framework/task/emotions/emotions.py b/child_lab_framework/task/emotion/emotion.py similarity index 100% rename from child_lab_framework/task/emotions/emotions.py rename to child_lab_framework/task/emotion/emotion.py diff --git a/child_lab_framework/task/emotions/__init__.py b/child_lab_framework/task/emotions/__init__.py deleted file mode 100644 index 8057aae..0000000 --- a/child_lab_framework/task/emotions/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .emotions import Estimator, Result - -__all__ = ['Estimator', 'Result'] diff --git a/child_lab_framework/task/visualization/visualization.py b/child_lab_framework/task/visualization/visualization.py index 3a3be89..f540e06 100644 --- a/child_lab_framework/task/visualization/visualization.py +++ b/child_lab_framework/task/visualization/visualization.py @@ -9,7 +9,7 @@ from ...core.video import Frame, Properties from ...typing.array import FloatArray1, FloatArray2, IntArray1 from ...typing.stream import Fiber -from .. import face, pose, emotions +from .. import face, pose, emotion from ..gaze import ceiling_projection from ..pose.keypoint import YOLO_SKELETON @@ -124,7 +124,7 @@ def __draw_face_box(self, frame: Frame, result: face.Result) -> Frame: return frame - def __draw_emotions_text(self, frame: Frame, result: emotions.Result) -> Frame: + def __draw_emotions_text(self, frame: Frame, result: emotion.Result) -> Frame: color = self.FACE_BOUNDING_BOX_COLOR for (value, box) in zip(result.emotions, result.boxes): cv2.putText(frame, str(value), [box[0], box[3]], cv2.FONT_HERSHEY_SIMPLEX, 0.9, color, 2) @@ -136,7 +136,7 @@ def __annotate_safe( poses: pose.Result | None, faces: face.Result | None, gazes: ceiling_projection.Result | None, - emotions: emotions.Result | None + emotions: emotion.Result | None ) -> Frame: out = frame.copy() out.flags.writeable = True @@ -162,7 +162,7 @@ def annotate_batch( poses: list[pose.Result] | None, faces: list[face.Result] | None, gazes: list[ceiling_projection.Result] | None, - emotions: list[emotions.Result] | None + emotions: list[emotion.Result] | None ) -> list[Frame]: return list( starmap( From b5e7c33ef8c8c421de01d3e58a45130b4a529303 Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Wed, 6 Nov 2024 18:57:07 -0500 Subject: [PATCH 6/8] Reformat --- child_lab_framework/demo_sequential.py | 14 +++++++----- child_lab_framework/task/emotion/emotion.py | 16 ++++++++------ .../task/visualization/visualization.py | 22 +++++++++++++------ 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/child_lab_framework/demo_sequential.py b/child_lab_framework/demo_sequential.py index c5d337b..9646308 100644 --- a/child_lab_framework/demo_sequential.py +++ b/child_lab_framework/demo_sequential.py @@ -110,7 +110,7 @@ def main() -> None: ) print('Starting sequential processing') - + while True: ceiling_frames = ceiling_reader.read_batch() if ceiling_frames is None: @@ -210,8 +210,12 @@ def main() -> None: else None ) - window_left_emotions = emotions_estimator_left.predict_batch(window_left_frames, window_left_faces) - window_right_emotions = emotions_estimator_right.predict_batch(window_right_frames, window_right_faces) + window_left_emotions = emotions_estimator_left.predict_batch( + window_left_frames, window_left_faces + ) + window_right_emotions = emotions_estimator_right.predict_batch( + window_right_frames, window_right_faces + ) ceiling_annotated_frames = visualizer.annotate_batch( ceiling_frames, @@ -226,7 +230,7 @@ def main() -> None: window_left_poses, window_left_faces, None, - window_left_emotions + window_left_emotions, ) window_right_annotated_frames = visualizer.annotate_batch( @@ -234,7 +238,7 @@ def main() -> None: window_right_poses, window_right_faces, None, - window_right_emotions + window_right_emotions, ) ceiling_writer.write_batch(ceiling_annotated_frames) diff --git a/child_lab_framework/task/emotion/emotion.py b/child_lab_framework/task/emotion/emotion.py index 3a47256..b1479c7 100644 --- a/child_lab_framework/task/emotion/emotion.py +++ b/child_lab_framework/task/emotion/emotion.py @@ -15,6 +15,7 @@ list[face.Result | None] | None, ] + class Result: emotions: list[float] boxes: list[FloatArray2] @@ -23,6 +24,7 @@ def __init__(self, emotions: list[float], boxes: list[FloatArray2]) -> None: self.emotions = emotions self.boxes = boxes + class Estimator: executor: ThreadPoolExecutor @@ -40,13 +42,15 @@ def predict(self, frame: Frame, faces: face.Result | None) -> Result: y_min = max(y_min - 50, 0) y_max = min(y_max + 50, frame_height) cropped_frame = frame[y_min:y_max, x_min:x_max] - analysis = DeepFace.analyze(cropped_frame, actions=['emotion'], enforce_detection=False) + analysis = DeepFace.analyze( + cropped_frame, actions=['emotion'], enforce_detection=False + ) emotion = score_emotions(analysis[0]) face_emotions.append(emotion) boxes.append(face_box) return Result(face_emotions, boxes) - + def predict_batch( self, frames: list[Frame], @@ -75,10 +79,7 @@ async def stream( lambda: list( starmap( self.__predict, - zip( - frames, - faces or repeat(None) - ), + zip(frames, faces or repeat(None)), ) ), ) @@ -86,6 +87,7 @@ async def stream( case _: results = None + def score_emotions(emotions: List[Dict[str, float]]) -> float: # Most of the time, "angry" and "fear" are similar to "neutral" in the reality scores = { @@ -101,4 +103,4 @@ def score_emotions(emotions: List[Dict[str, float]]) -> float: for emotion, score in scores.items(): val += emotions['emotion'][emotion] * score - return val \ No newline at end of file + return val diff --git a/child_lab_framework/task/visualization/visualization.py b/child_lab_framework/task/visualization/visualization.py index f540e06..e9c6874 100644 --- a/child_lab_framework/task/visualization/visualization.py +++ b/child_lab_framework/task/visualization/visualization.py @@ -123,11 +123,19 @@ def __draw_face_box(self, frame: Frame, result: face.Result) -> Frame: cv2.rectangle(frame, (x1, y1), (x2, y2), color, thickness) return frame - + def __draw_emotions_text(self, frame: Frame, result: emotion.Result) -> Frame: color = self.FACE_BOUNDING_BOX_COLOR - for (value, box) in zip(result.emotions, result.boxes): - cv2.putText(frame, str(value), [box[0], box[3]], cv2.FONT_HERSHEY_SIMPLEX, 0.9, color, 2) + for value, box in zip(result.emotions, result.boxes): + cv2.putText( + frame, + str(value), + [box[0], box[3]], + cv2.FONT_HERSHEY_SIMPLEX, + 0.9, + color, + 2, + ) return frame def __annotate_safe( @@ -136,7 +144,7 @@ def __annotate_safe( poses: pose.Result | None, faces: face.Result | None, gazes: ceiling_projection.Result | None, - emotions: emotion.Result | None + emotions: emotion.Result | None, ) -> Frame: out = frame.copy() out.flags.writeable = True @@ -150,7 +158,7 @@ def __annotate_safe( if gazes is not None: out = self.__draw_gaze_estimation(out, gazes) - + if emotions is not None: out = self.__draw_emotions_text(out, emotions) @@ -162,7 +170,7 @@ def annotate_batch( poses: list[pose.Result] | None, faces: list[face.Result] | None, gazes: list[ceiling_projection.Result] | None, - emotions: list[emotion.Result] | None + emotions: list[emotion.Result] | None, ) -> list[Frame]: return list( starmap( @@ -172,7 +180,7 @@ def annotate_batch( poses or repeat(None), faces or repeat(None), gazes or repeat(None), - emotions or repeat(None) + emotions or repeat(None), ), ) ) From a0c9daa71362bbbe86e70adb53e13635954e7466 Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Thu, 14 Nov 2024 09:05:26 +0100 Subject: [PATCH 7/8] Review --- child_lab_framework/task/emotion/emotion.py | 49 +++++++++++---------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/child_lab_framework/task/emotion/emotion.py b/child_lab_framework/task/emotion/emotion.py index b1479c7..6082f7e 100644 --- a/child_lab_framework/task/emotion/emotion.py +++ b/child_lab_framework/task/emotion/emotion.py @@ -1,11 +1,10 @@ import asyncio -from typing import Dict, List from deepface import DeepFace from concurrent.futures import ThreadPoolExecutor from itertools import repeat, starmap -from child_lab_framework.core.sequence import imputed_with_reference_inplace -from child_lab_framework.task import face +from ...task import face +from ...core.sequence import imputed_with_reference_inplace from ...core.video import Frame from ...typing.stream import Fiber from ...typing.array import FloatArray2 @@ -31,7 +30,7 @@ class Estimator: def __init__(self, executor: ThreadPoolExecutor) -> None: self.executor = executor - def predict(self, frame: Frame, faces: face.Result | None) -> Result: + def predict(self, frame: Frame, faces: face.Result) -> Result: face_emotions = [] boxes = [] frame_height, frame_width, _ = frame.shape @@ -45,11 +44,16 @@ def predict(self, frame: Frame, faces: face.Result | None) -> Result: analysis = DeepFace.analyze( cropped_frame, actions=['emotion'], enforce_detection=False ) - emotion = score_emotions(analysis[0]) + emotion = self.__score(analysis[0]) face_emotions.append(emotion) boxes.append(face_box) return Result(face_emotions, boxes) + + def __predict_safe(self, frame: Frame, faces: face.Result | None) -> Result: + if faces is None: + return Result([], []) + return self.predict(frame, faces) def predict_batch( self, @@ -62,7 +66,7 @@ def predict_batch( async def stream( self, - ) -> Fiber[list[Frame | None] | None, list[Result | None] | None]: + ) -> Fiber[list[Input] | None, list[Result | None] | None]: loop = asyncio.get_running_loop() executor = self.executor @@ -87,20 +91,19 @@ async def stream( case _: results = None - -def score_emotions(emotions: List[Dict[str, float]]) -> float: - # Most of the time, "angry" and "fear" are similar to "neutral" in the reality - scores = { - 'angry': -0.05, - 'disgust': 0, - 'fear': -0.07, - 'happy': 1, - 'sad': -1, - 'surprise': 0, - 'neutral': 0, - } - val = 0 - for emotion, score in scores.items(): - val += emotions['emotion'][emotion] * score - - return val + def __score(emotions: list[dict[str, float]]) -> list[float]: + # Most of the time, "angry" and "fear" are similar to "neutral" in the reality + scores = { + 'angry': -0.05, + 'disgust': 0, + 'fear': -0.07, + 'happy': 1, + 'sad': -1, + 'surprise': 0, + 'neutral': 0, + } + val = 0 + for emotion, score in scores.items(): + val += emotions['emotion'][emotion] * score + + return val \ No newline at end of file From 1af7bc73a9cf59d62b5acf3f06984cc3bb9c9b3b Mon Sep 17 00:00:00 2001 From: WojciechBarczynski Date: Thu, 14 Nov 2024 09:10:39 +0100 Subject: [PATCH 8/8] Fix predict method --- child_lab_framework/task/emotion/emotion.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/child_lab_framework/task/emotion/emotion.py b/child_lab_framework/task/emotion/emotion.py index 6082f7e..6391b08 100644 --- a/child_lab_framework/task/emotion/emotion.py +++ b/child_lab_framework/task/emotion/emotion.py @@ -30,7 +30,7 @@ class Estimator: def __init__(self, executor: ThreadPoolExecutor) -> None: self.executor = executor - def predict(self, frame: Frame, faces: face.Result) -> Result: + def __predict(self, frame: Frame, faces: face.Result) -> Result: face_emotions = [] boxes = [] frame_height, frame_width, _ = frame.shape @@ -53,7 +53,7 @@ def predict(self, frame: Frame, faces: face.Result) -> Result: def __predict_safe(self, frame: Frame, faces: face.Result | None) -> Result: if faces is None: return Result([], []) - return self.predict(frame, faces) + return self.__predict(frame, faces) def predict_batch( self, @@ -61,7 +61,7 @@ def predict_batch( faces: list[face.Result | None], ) -> list[Result] | None: return imputed_with_reference_inplace( - list(starmap(self.predict, zip(frames, faces))) + list(starmap(self.__predict, zip(frames, faces))) ) async def stream(