diff --git a/examples/facelandmarkdetection/src/Photo.tsx b/examples/facelandmarkdetection/src/Photo.tsx index 83ff98d..8410296 100644 --- a/examples/facelandmarkdetection/src/Photo.tsx +++ b/examples/facelandmarkdetection/src/Photo.tsx @@ -1,18 +1,21 @@ import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs"; -import React, { useState, useCallback } from "react"; +import React, { useState } from "react"; import { View, Text, Pressable, StyleSheet, Image } from "react-native"; import type { RootTabParamList } from "./navigation"; import ImagePicker from "react-native-image-crop-picker"; -import { objectDetectionOnImage, type Dims } from "react-native-mediapipe"; import { useSettings } from "./app-settings"; +import { CustomColors } from "./colors"; import { faceLandmarkDetectionModuleConstants, - useFaceLandmarkDetection, type FaceLandmarksModuleConstants, - RunningMode, - type FaceLandmarkDetectionResultBundle, + type Dims, + faceLandmarkDetectionOnImage, } from "react-native-mediapipe"; -import { FaceDrawFrame, convertLandmarksToSegments, type FaceSegment } from "./Drawing"; +import { + FaceDrawFrame, + convertLandmarksToSegments, + type FaceSegment, +} from "./Drawing"; type Props = BottomTabScreenProps; @@ -24,98 +27,11 @@ export const Photo: React.FC = () => { >("initial"); const { settings } = useSettings(); const [imagePath, setImagePath] = useState(); - const [faceLandmarks] = useState( - faceLandmarkDetectionModuleConstants().knownLandmarks - ); + const [faceLandmarks] = useState< + FaceLandmarksModuleConstants["knownLandmarks"] + >(faceLandmarkDetectionModuleConstants().knownLandmarks); const [faceSegments, setFaceSegments] = useState([]); - - const onFaceDetectionResults = useCallback(( - results: FaceLandmarkDetectionResultBundle, - viewSize: Dims, - mirrored: boolean - ) => { - if (results.results.length === 0) { - setFaceSegments([]); - return; - } - const firstResult = results.results[0]; - const segments = firstResult.faceLandmarks.length > 0 - ? [ - ...convertLandmarksToSegments( - firstResult.faceLandmarks[0], - faceLandmarks.lips, - "FireBrick", - { - width: results.inputImageWidth, - height: results.inputImageHeight, - }, - { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height }, - mirrored - ), - ...convertLandmarksToSegments( - firstResult.faceLandmarks[0], - faceLandmarks.leftEye, - "ForestGreen", - { - width: results.inputImageWidth, - height: results.inputImageHeight, - }, - { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height }, - mirrored - ), - ...convertLandmarksToSegments( - firstResult.faceLandmarks[0], - faceLandmarks.rightEye, - "ForestGreen", - { - width: results.inputImageWidth, - height: results.inputImageHeight, - }, - { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height }, - mirrored - ), - ...convertLandmarksToSegments( - firstResult.faceLandmarks[0], - faceLandmarks.leftEyebrow, - "Coral", - { - width: results.inputImageWidth, - height: results.inputImageHeight, - }, - { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height }, - mirrored - ), - ...convertLandmarksToSegments( - firstResult.faceLandmarks[0], - faceLandmarks.rightEyebrow, - "Coral", - { - width: results.inputImageWidth, - height: results.inputImageHeight, - }, - { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height }, - mirrored - ), - ] - : []; - - console.log(JSON.stringify({ infTime: results.inferenceTime })); - setFaceSegments(segments); - }, [faceLandmarks]); - - const onFaceDetectionError = useCallback((error: unknown) => { - console.error(`onError: ${error}`); - }, []); - - const faceDetection = useFaceLandmarkDetection( - onFaceDetectionResults, - onFaceDetectionError, - RunningMode.IMAGE, - "face_landmarker.task", - { - delegate: settings.processor, - } - ); + const [errorMessage, setErrorMessage] = React.useState(""); const onClickSelectPhoto = async () => { setScreenState("selecting"); @@ -125,29 +41,93 @@ export const Photo: React.FC = () => { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height, }); - - const results = await objectDetectionOnImage( + const results = await faceLandmarkDetectionOnImage( image.path, - `${settings.model}.tflite` - ); - const detections = results.results[0]?.detections ?? []; - console.log( - JSON.stringify({ - width: image.width, - height: image.height, - detections: detections.map((d) => ({ - bb: d.boundingBox, - cat: d.categories[0]?.categoryName, - })), - }) + "face_landmarker.task" ); + const frameSize = { + width: results.inputImageWidth, + height: results.inputImageHeight, + }; + if (results.results.length === 0) { + setFaceSegments([]); + return; + } + const firstResult = results.results[0]; + const segments = + firstResult.faceLandmarks.length > 0 + ? [ + ...convertLandmarksToSegments( + firstResult.faceLandmarks[0], + faceLandmarks.lips, + "FireBrick", + { + width: results.inputImageWidth, + height: results.inputImageHeight, + }, + PHOTO_SIZE + ), + ...convertLandmarksToSegments( + firstResult.faceLandmarks[0], + faceLandmarks.leftEye, + "ForestGreen", + { + width: results.inputImageWidth, + height: results.inputImageHeight, + }, + PHOTO_SIZE + ), + ...convertLandmarksToSegments( + firstResult.faceLandmarks[0], + faceLandmarks.rightEye, + "ForestGreen", + { + width: results.inputImageWidth, + height: results.inputImageHeight, + }, + PHOTO_SIZE + ), + ...convertLandmarksToSegments( + firstResult.faceLandmarks[0], + faceLandmarks.leftEyebrow, + "Coral", + { + width: results.inputImageWidth, + height: results.inputImageHeight, + }, + PHOTO_SIZE + ), + ...convertLandmarksToSegments( + firstResult.faceLandmarks[0], + faceLandmarks.rightEyebrow, + "Coral", + { + width: results.inputImageWidth, + height: results.inputImageHeight, + }, + PHOTO_SIZE + ), + ] + : []; - faceDetection.detect(image.path); + console.log(JSON.stringify({ infTime: results.inferenceTime })); + setFaceSegments(segments); setImagePath(image.path); setScreenState("completed"); } catch (e) { console.error(e); + if (e instanceof Error) { + if (e.message.includes("User cancelled image selection")) { + setErrorMessage("User cancelled image selection."); + } else if (e.message.includes("Permissions")) { + setErrorMessage("Permission denied. Please allow access to photos."); + } else { + setErrorMessage("An unexpected error occurred."); + } + } else { + setErrorMessage("An unexpected error occurred."); + } setScreenState("error"); } }; @@ -164,7 +144,7 @@ export const Photo: React.FC = () => { @@ -176,7 +156,10 @@ export const Photo: React.FC = () => { )} {screenState === "error" && ( <> - Error! Please try again. + + Error! + {errorMessage} + Select a photo @@ -187,11 +170,60 @@ export const Photo: React.FC = () => { }; const styles = StyleSheet.create({ - root: { flex: 1, alignItems: "center", justifyContent: "center" }, - selectButton: { backgroundColor: "blue", padding: 10, borderRadius: 5 }, - selectButtonText: { fontSize: 20, color: "white" }, - photoContainer: { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height }, - photo: { position: "absolute", top: 0, left: 0, right: 0, bottom: 0 }, - errorText: { fontSize: 30, color: "red" }, + root: { + flex: 1, + alignItems: "center", + justifyContent: "center", + backgroundColor: CustomColors.backgroundGrayBlue, + }, + selectButton: { + backgroundColor: CustomColors.elecBlue, + padding: 15.5, + paddingRight: 25, + paddingLeft: 25, + borderRadius: 5, + }, + selectButtonText: { + fontSize: 20, + color: "black", + fontWeight: "bold", + }, + photoContainer: { + position: "relative", + width: PHOTO_SIZE.width, + height: PHOTO_SIZE.height, + }, + photo: { + position: "absolute", + top: 0, + left: 0, + right: 0, + bottom: 0, + }, + objectsOverlay: { + position: "absolute", + top: 0, + left: 0, + right: 0, + bottom: 0, + }, + errorText: { + fontSize: 25, + color: "black", + bottom: 10, + fontWeight: "bold", + textAlign: "center", + }, + errorInfoText: { + fontSize: 15.5, + color: CustomColors.teal, + }, + errorBox: { + backgroundColor: CustomColors.lightGray, + padding: 20, + borderRadius: 12, + borderWidth: 1, + borderColor: CustomColors.teal, + bottom: 25, + }, }); - diff --git a/examples/facelandmarkdetection/src/colors.ts b/examples/facelandmarkdetection/src/colors.ts new file mode 100644 index 0000000..9fdca7d --- /dev/null +++ b/examples/facelandmarkdetection/src/colors.ts @@ -0,0 +1,7 @@ +export const CustomColors = { + teal: "#2C8395", + elecBlue: "#5FCCF5", + backgroundGrayBlue: "#EAF5F9", + deepSkyBlue: "#00BFFF", + lightGray: "#F3F3F3", + }; \ No newline at end of file