Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: make so facelandmarker photo passes typecheck #133

Merged
merged 1 commit into from
Jun 22, 2024
Merged

fix: make so facelandmarker photo passes typecheck #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
274 changes: 153 additions & 121 deletions examples/facelandmarkdetection/src/Photo.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
import type { BottomTabScreenProps } from "@react-navigation/bottom-tabs";
import React, { useState, useCallback } from "react";
import React, { useState } from "react";
import { View, Text, Pressable, StyleSheet, Image } from "react-native";
import type { RootTabParamList } from "./navigation";
import ImagePicker from "react-native-image-crop-picker";
import { objectDetectionOnImage, type Dims } from "react-native-mediapipe";
import { useSettings } from "./app-settings";
import { CustomColors } from "./colors";
import {
faceLandmarkDetectionModuleConstants,
useFaceLandmarkDetection,
type FaceLandmarksModuleConstants,
RunningMode,
type FaceLandmarkDetectionResultBundle,
type Dims,
faceLandmarkDetectionOnImage,
} from "react-native-mediapipe";
import { FaceDrawFrame, convertLandmarksToSegments, type FaceSegment } from "./Drawing";
import {
FaceDrawFrame,
convertLandmarksToSegments,
type FaceSegment,
} from "./Drawing";

type Props = BottomTabScreenProps<RootTabParamList, "Photo">;

Expand All @@ -22,100 +25,13 @@
const [screenState, setScreenState] = useState<
"initial" | "selecting" | "inferring" | "completed" | "error"
>("initial");
const { settings } = useSettings();

Check warning on line 28 in examples/facelandmarkdetection/src/Photo.tsx

View workflow job for this annotation

GitHub Actions / lint 

'settings' is assigned a value but never used. Allowed unused vars must match /^_/u
const [imagePath, setImagePath] = useState<string>();
const [faceLandmarks] = useState<FaceLandmarksModuleConstants["knownLandmarks"]>(
faceLandmarkDetectionModuleConstants().knownLandmarks
);
const [faceLandmarks] = useState<
FaceLandmarksModuleConstants["knownLandmarks"]
>(faceLandmarkDetectionModuleConstants().knownLandmarks);
const [faceSegments, setFaceSegments] = useState<FaceSegment[]>([]);

const onFaceDetectionResults = useCallback((
results: FaceLandmarkDetectionResultBundle,
viewSize: Dims,
mirrored: boolean
) => {
if (results.results.length === 0) {
setFaceSegments([]);
return;
}
const firstResult = results.results[0];
const segments = firstResult.faceLandmarks.length > 0
? [
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.lips,
"FireBrick",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
{ width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
mirrored
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.leftEye,
"ForestGreen",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
{ width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
mirrored
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.rightEye,
"ForestGreen",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
{ width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
mirrored
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.leftEyebrow,
"Coral",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
{ width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
mirrored
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.rightEyebrow,
"Coral",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
{ width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
mirrored
),
]
: [];

console.log(JSON.stringify({ infTime: results.inferenceTime }));
setFaceSegments(segments);
}, [faceLandmarks]);

const onFaceDetectionError = useCallback((error: unknown) => {
console.error(`onError: ${error}`);
}, []);

const faceDetection = useFaceLandmarkDetection(
onFaceDetectionResults,
onFaceDetectionError,
RunningMode.IMAGE,
"face_landmarker.task",
{
delegate: settings.processor,
}
);
const [errorMessage, setErrorMessage] = React.useState<string>("");

const onClickSelectPhoto = async () => {
setScreenState("selecting");
Expand All @@ -125,29 +41,93 @@
width: PHOTO_SIZE.width,
height: PHOTO_SIZE.height,
});

const results = await objectDetectionOnImage(
const results = await faceLandmarkDetectionOnImage(
image.path,
`${settings.model}.tflite`
);
const detections = results.results[0]?.detections ?? [];
console.log(
JSON.stringify({
width: image.width,
height: image.height,
detections: detections.map((d) => ({
bb: d.boundingBox,
cat: d.categories[0]?.categoryName,
})),
})
"face_landmarker.task"
);
const frameSize = {

Check warning on line 48 in examples/facelandmarkdetection/src/Photo.tsx

View workflow job for this annotation

GitHub Actions / lint 

'frameSize' is assigned a value but never used. Allowed unused vars must match /^_/u
width: results.inputImageWidth,
height: results.inputImageHeight,
};
if (results.results.length === 0) {
setFaceSegments([]);
return;
}
const firstResult = results.results[0];
const segments =
firstResult.faceLandmarks.length > 0
? [
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.lips,
"FireBrick",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
PHOTO_SIZE
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.leftEye,
"ForestGreen",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
PHOTO_SIZE
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.rightEye,
"ForestGreen",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
PHOTO_SIZE
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.leftEyebrow,
"Coral",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
PHOTO_SIZE
),
...convertLandmarksToSegments(
firstResult.faceLandmarks[0],
faceLandmarks.rightEyebrow,
"Coral",
{
width: results.inputImageWidth,
height: results.inputImageHeight,
},
PHOTO_SIZE
),
]
: [];

faceDetection.detect(image.path);
console.log(JSON.stringify({ infTime: results.inferenceTime }));

setFaceSegments(segments);
setImagePath(image.path);
setScreenState("completed");
} catch (e) {
console.error(e);
if (e instanceof Error) {
if (e.message.includes("User cancelled image selection")) {
setErrorMessage("User cancelled image selection.");
} else if (e.message.includes("Permissions")) {
setErrorMessage("Permission denied. Please allow access to photos.");
} else {
setErrorMessage("An unexpected error occurred.");
}
} else {
setErrorMessage("An unexpected error occurred.");
}
setScreenState("error");
}
};
Expand All @@ -164,7 +144,7 @@
<View style={styles.photoContainer}>
<Image source={{ uri: imagePath }} style={styles.photo} />
<FaceDrawFrame
style={StyleSheet.absoluteFill}
style={styles.objectsOverlay}
facePoints={[]}
faceSegments={faceSegments}
/>
Expand All @@ -176,7 +156,10 @@
)}
{screenState === "error" && (
<>
<Text style={styles.errorText}>Error! Please try again.</Text>
<View style={styles.errorBox}>
<Text style={styles.errorText}>Error!</Text>
<Text style={styles.errorInfoText}>{errorMessage}</Text>
</View>
<Pressable style={styles.selectButton} onPress={onClickSelectPhoto}>
<Text style={styles.selectButtonText}>Select a photo</Text>
</Pressable>
Expand All @@ -187,11 +170,60 @@
};

const styles = StyleSheet.create({
root: { flex: 1, alignItems: "center", justifyContent: "center" },
selectButton: { backgroundColor: "blue", padding: 10, borderRadius: 5 },
selectButtonText: { fontSize: 20, color: "white" },
photoContainer: { width: PHOTO_SIZE.width, height: PHOTO_SIZE.height },
photo: { position: "absolute", top: 0, left: 0, right: 0, bottom: 0 },
errorText: { fontSize: 30, color: "red" },
root: {
flex: 1,
alignItems: "center",
justifyContent: "center",
backgroundColor: CustomColors.backgroundGrayBlue,
},
selectButton: {
backgroundColor: CustomColors.elecBlue,
padding: 15.5,
paddingRight: 25,
paddingLeft: 25,
borderRadius: 5,
},
selectButtonText: {
fontSize: 20,
color: "black",
fontWeight: "bold",
},
photoContainer: {
position: "relative",
width: PHOTO_SIZE.width,
height: PHOTO_SIZE.height,
},
photo: {
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
},
objectsOverlay: {
position: "absolute",
top: 0,
left: 0,
right: 0,
bottom: 0,
},
errorText: {
fontSize: 25,
color: "black",
bottom: 10,
fontWeight: "bold",
textAlign: "center",
},
errorInfoText: {
fontSize: 15.5,
color: CustomColors.teal,
},
errorBox: {
backgroundColor: CustomColors.lightGray,
padding: 20,
borderRadius: 12,
borderWidth: 1,
borderColor: CustomColors.teal,
bottom: 25,
},
});

7 changes: 7 additions & 0 deletions examples/facelandmarkdetection/src/colors.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export const CustomColors = {
teal: "#2C8395",
elecBlue: "#5FCCF5",
backgroundGrayBlue: "#EAF5F9",
deepSkyBlue: "#00BFFF",
lightGray: "#F3F3F3",
};
Loading