From e3c778cc7fcc68cca3813e8c417af14dac2d0c71 Mon Sep 17 00:00:00 2001
From: shackerman <jec.alferez02@gmail.com>
Date: Thu, 6 Jun 2024 00:19:19 -0700
Subject: [PATCH] feat: added face landmark to photo

---
 examples/facelandmarkdetection/src/Photo.tsx | 119 ++++++++++++++++++-
 1 file changed, 115 insertions(+), 4 deletions(-)

diff --git a/examples/facelandmarkdetection/src/Photo.tsx b/examples/facelandmarkdetection/src/Photo.tsx
index a56d440..62d4697 100644
--- a/examples/facelandmarkdetection/src/Photo.tsx
+++ b/examples/facelandmarkdetection/src/Photo.tsx
@@ -1,21 +1,122 @@
 import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs";
-import React from "react";
+import React, { useState, useCallback } from "react";
 import { View, Text, Pressable, StyleSheet, Image } from "react-native";
 import type { RootTabParamList } from "./navigation";
 import ImagePicker from "react-native-image-crop-picker";
 import { objectDetectionOnImage, type Dims } from "react-native-mediapipe";
 import { useSettings } from "./app-settings";
+import {
+  faceLandmarkDetectionModuleConstants,
+  useFaceLandmarkDetection,
+  type FaceLandmarksModuleConstants,
+  RunningMode,
+  type FaceLandmarkDetectionResultBundle,
+} from "react-native-mediapipe";
+import { FaceDrawFrame, convertLandmarksToSegments, type FaceSegment } from "./Drawing";
 
 type Props = BottomTabScreenProps<RootTabParamList, "Photo">;
 
 const PHOTO_SIZE: Dims = { width: 300, height: 400 };
 
 export const Photo: React.FC<Props> = () => {
-  const [screenState, setScreenState] = React.useState<
+  const [screenState, setScreenState] = useState<
     "initial" | "selecting" | "inferring" | "completed" | "error"
   >("initial");
   const { settings } = useSettings();
-  const [imagePath, setImagePath] = React.useState<string>();
+  const [imagePath, setImagePath] = useState<string>();
+  const [faceLandmarks] = useState<FaceLandmarksModuleConstants["knownLandmarks"]>(
+    faceLandmarkDetectionModuleConstants().knownLandmarks
+  );
+  const [faceSegments, setFaceSegments] = useState<FaceSegment[]>([]);
+
+  const onFaceDetectionResults = useCallback((
+    results: FaceLandmarkDetectionResultBundle,
+    viewSize: Dims,
+    mirrored: boolean
+  ) => {
+    if (results.results.length === 0) {
+      setFaceSegments([]);
+      return;
+    }
+    const firstResult = results.results[0];
+    const segments = firstResult.faceLandmarks.length > 0
+      ? [
+          ...convertLandmarksToSegments(
+            firstResult.faceLandmarks[0],
+            faceLandmarks.lips,
+            "FireBrick",
+            {
+              width: results.inputImageWidth,
+              height: results.inputImageHeight,
+            },
+            { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
+            mirrored
+          ),
+          ...convertLandmarksToSegments(
+            firstResult.faceLandmarks[0],
+            faceLandmarks.leftEye,
+            "ForestGreen",
+            {
+              width: results.inputImageWidth,
+              height: results.inputImageHeight,
+            },
+            { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
+            mirrored
+          ),
+          ...convertLandmarksToSegments(
+            firstResult.faceLandmarks[0],
+            faceLandmarks.rightEye,
+            "ForestGreen",
+            {
+              width: results.inputImageWidth,
+              height: results.inputImageHeight,
+            },
+            { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
+            mirrored
+          ),
+          ...convertLandmarksToSegments(
+            firstResult.faceLandmarks[0],
+            faceLandmarks.leftEyebrow,
+            "Coral",
+            {
+              width: results.inputImageWidth,
+              height: results.inputImageHeight,
+            },
+            { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
+            mirrored
+          ),
+          ...convertLandmarksToSegments(
+            firstResult.faceLandmarks[0],
+            faceLandmarks.rightEyebrow,
+            "Coral",
+            {
+              width: results.inputImageWidth,
+              height: results.inputImageHeight,
+            },
+            { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
+            mirrored
+          ),
+        ]
+      : [];
+
+    console.log(JSON.stringify({ infTime: results.inferenceTime }));
+    setFaceSegments(segments);
+  }, [faceLandmarks]);
+
+  const onFaceDetectionError = useCallback((error: unknown) => {
+    console.error(`onError: ${error}`);
+  }, []);
+
+  const faceDetection = useFaceLandmarkDetection(
+    onFaceDetectionResults,
+    onFaceDetectionError,
+    RunningMode.IMAGE,
+    "face_landmarker.task",
+    {
+      delegate: settings.processor,
+    }
+  );
+
   const onClickSelectPhoto = async () => {
     setScreenState("selecting");
     try {
@@ -24,6 +125,7 @@ export const Photo: React.FC<Props> = () => {
         width: PHOTO_SIZE.width,
         height: PHOTO_SIZE.height,
       });
+
       const results = await objectDetectionOnImage(
         image.path,
         `${settings.model}.tflite`
@@ -39,6 +141,10 @@ export const Photo: React.FC<Props> = () => {
           })),
         })
       );
+
+      // Perform face landmark detection
+      faceDetection.detect(image.path); // Ensure detect is a method of faceDetection
+
       setImagePath(image.path);
       setScreenState("completed");
     } catch (e) {
@@ -47,7 +153,6 @@ export const Photo: React.FC<Props> = () => {
     }
   };
 
-  // TODO : implement face landmark rendering
   return (
     <View style={styles.root}>
       {screenState === "initial" && (
@@ -59,6 +164,11 @@ export const Photo: React.FC<Props> = () => {
         <>
           <View style={styles.photoContainer}>
             <Image source={{ uri: imagePath }} style={styles.photo} />
+            <FaceDrawFrame
+              style={StyleSheet.absoluteFill}
+              facePoints={[]}
+              faceSegments={faceSegments}
+            />
           </View>
           <Pressable style={styles.selectButton} onPress={onClickSelectPhoto}>
             <Text style={styles.selectButtonText}>Select a new photo</Text>
@@ -85,3 +195,4 @@ const styles = StyleSheet.create({
   photo: { position: "absolute", top: 0, left: 0, right: 0, bottom: 0 },
   errorText: { fontSize: 30, color: "red" },
 });
+