feat: added mouth movement using visemes based on the text provided

memori-ai · Oct 7, 2024 · af55d34 · af55d34
1 parent 82a1fae
commit af55d34
Show file tree

Hide file tree

Showing 5 changed files with 295 additions and 89 deletions.
diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx
@@ -49,7 +49,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   chatEmission,
   showControls,
   animation,
-  loading,
+ // loading,
   url,
   sex,
   eyeBlink,
@@ -75,21 +75,29 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   const { createVisemeSequence, currentVisemes, clearVisemes } = useViseme();
 
   // Set the morph target influences for the given emotions
-  const setEmotion = useCallback(
-    (action: string) => {
-      const emotionMap = {
-        Gioia: { eyesClosed: 0.5, mouthSmile: 1 },
-        Rabbia: { eyesClosed: 1, mouthSmile: -0.5 },
-        Sorpresa: { mouthSmile: 0.5, eyesClosed: -0.2 },
-        Tristezza: { mouthSmile: -0.6, eyesClosed: 0.5 },
-        Timore: { mouthSmile: -0.5, eyesClosed: 1 },
-        default: { mouthSmile: 0, eyesClosed: 0 }
-      };
-      const emotion = Object.keys(emotionMap).find(key => action.startsWith(key)) || 'default';
-      setMorphTargetInfluences(emotionMap[emotion as keyof typeof emotionMap]);
-    },
-    []
-  );
+  const setEmotion = useCallback((action: string) => {
+    const emotionMap: Record<string, Record<string, number>> = {
+      Gioia: { Gioria: 1 },
+      Rabbia: { Rabbia: 1 },
+      Sorpresa: { Sorpresa: 1 },
+      Tristezza: { Tristezza: 1 },
+      Timore: { Timore: 1 },
+    };
+
+    const defaultEmotions = Object.keys(emotionMap).reduce((acc, key) => {
+      acc[key] = 0;
+      return acc;
+    }, {} as Record<string, number>);
+
+    const emotion = Object.keys(emotionMap).find(key => action.startsWith(key)) || 'default';
+    const emotionValues = emotion === 'default' ? defaultEmotions : emotionMap[emotion];
+
+    setMorphTargetInfluences(prevInfluences => ({
+      ...prevInfluences,
+      ...defaultEmotions,
+      ...emotionValues
+    }));
+  }, []);
 
   const onBaseActionChange = useCallback((action: string) => {
     setEmotion(action);
@@ -122,8 +130,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
 
   // Set the emotion based on the chatEmission
   useEffect(() => {
-
-    if(chatEmission){
+    if (chatEmission) {
       createVisemeSequence(chatEmission);
     }
 
@@ -144,19 +151,27 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
       //Choose a random number between 1 and 3
       const randomNumber = Math.floor(Math.random() * 3) + 1;
       const emotion = `${outputContent}${randomNumber}`;
-      setEmotion(emotion);
+
+      onBaseActionChange(emotion);
     }
   }, [chatEmission]);
 
+  const resetToIdle = useCallback(() => {
+    const randomIdle = Math.floor(Math.random() * 5) + 1;
+    setCurrentBaseAction({
+      action: `Idle${randomIdle}`,
+      weight: 1,
+    });
+    setMorphTargetInfluences({ mouthSmile: 0, eyesClosed: 0 });
+  }, []);
+
+
   //Set a loading state to true if the avatar is loading
-  useEffect(() => {
-    if (loading) {
-      setCurrentBaseAction({
-        action: 'Idle1',
-        weight: 1,
-      });
-    }
-  }, [loading]);
+  // useEffect(() => {
+  //   if (loading) {
+  //     resetToIdle();
+  //   }
+  // }, [loading]);
 
   return (
     <>

diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/controls.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/controls.tsx
@@ -91,7 +91,7 @@ const AnimationControlPanel: React.FC<AnimationControlPanelProps> = ({
     modifyTimeScale,
     baseActions,
     morphTargetDictionary,
-    timeScale,
+    timeScale
   ]);
 
   return null; // This component doesn't render anything itself

diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx
@@ -1,16 +1,22 @@
-// Import necessary dependencies
-import React, { useEffect, useRef, useState } from 'react';
-import { Vector3, Euler, AnimationMixer, SkinnedMesh, Object3D } from 'three';
+import React, { useEffect, useRef, useState, useCallback } from 'react';
+import {
+  Vector3,
+  Euler,
+  AnimationMixer,
+  SkinnedMesh,
+  Object3D,
+  AnimationAction,
+} from 'three';
 import { useAnimations, useGLTF } from '@react-three/drei';
 import { useGraph, dispose, useFrame } from '@react-three/fiber';
 import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
 import { useAvatarBlink } from '../../utils/useEyeBlink';
+import { useMouthAnimation } from '../../utils/useMouthAnimation';
 
 const lerp = (start: number, end: number, alpha: number): number => {
   return start * (1 - alpha) + end * alpha;
 };
 
-// Define the props interface for the FullbodyAvatar component
 interface FullbodyAvatarProps {
   url: string;
   sex: 'MALE' | 'FEMALE';
@@ -28,21 +34,25 @@ interface FullbodyAvatarProps {
   morphTargetInfluences: { [key: string]: number };
   morphTargetDictionary: { [key: string]: number };
   eyeBlink?: boolean;
+  currentVisemes: {
+    name: string;
+    duration: number;
+    weight: number;
+    startTime: number;
+  }[];
 }
 
-// Define constants for avatar positioning
 const AVATAR_POSITION = new Vector3(0, -1, 0);
 const AVATAR_ROTATION = new Euler(0.175, 0, 0);
 const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
 
-// Define URLs for male and female animation assets
 const ANIMATION_URLS = {
   MALE: 'https://assets.memori.ai/api/v2/asset/1c350a21-97d8-4add-82cc-9dc10767a26b.glb',
   FEMALE:
-    'https://assets.memori.ai/api/v2/asset/a1908dbf-8ce8-438d-90df-acf9dc2604ad.glb',
+    'https://assets.memori.ai/api/v2/asset/c2b07166-de10-4c66-918b-7b7cd380cca7.glb',
 };
+const ANIMATION_DURATION = 3000; // Duration in milliseconds for non-idle animations
 
-// Define the FullbodyAvatar component
 export default function FullbodyAvatar({
   url,
   sex,
@@ -54,16 +64,21 @@ export default function FullbodyAvatar({
   setMorphTargetDictionary,
   morphTargetInfluences,
   eyeBlink,
+  currentVisemes,
 }: FullbodyAvatarProps) {
-  // Load the 3D model and animations
   const { scene } = useGLTF(url);
   const { animations } = useGLTF(ANIMATION_URLS[sex]);
   const { nodes, materials } = useGraph(scene);
   const { actions } = useAnimations(animations, scene);
   const [mixer] = useState(() => new AnimationMixer(scene));
 
-  // Create a ref for the SkinnedMesh
   const avatarMeshRef = useRef<SkinnedMesh>();
+  const currentActionRef = useRef<AnimationAction | null>(null);
+  const isTransitioningRef = useRef(false);
+  const { handleMouthMovement } = useMouthAnimation({
+    currentVisemes,
+    avatarMeshRef: avatarMeshRef as React.RefObject<SkinnedMesh>,
+  });
 
   useAvatarBlink({
     enabled: eyeBlink || false,
@@ -75,21 +90,99 @@ export default function FullbodyAvatar({
     },
   });
 
-  // Effect to setup morphTargets and cleanup
+  const transitionToIdle = useCallback(() => {
+    if (!actions || isTransitioningRef.current) return;
+
+    isTransitioningRef.current = true;
+
+    const finishCurrentAnimation = () => {
+      if (currentActionRef.current && !currentActionRef.current.paused) {
+        const remainingTime = (currentActionRef.current.getClip().duration - currentActionRef.current.time) * 1000;
+        setTimeout(() => {
+          startIdleAnimation();
+        }, remainingTime);
+      } else {
+        startIdleAnimation();
+      }
+    };
+
+    const startIdleAnimation = () => {
+      // Choose a random Idle animation
+      const idleAnimations = Object.keys(actions).filter(key =>
+        key.startsWith('Idle')
+      );
+      const randomIdle =
+        idleAnimations[Math.floor(Math.random() * idleAnimations.length)];
+
+      const idleAction = actions[randomIdle];
+      const fadeOutDuration = 0.5;
+      const fadeInDuration = 0.5;
+
+      if (currentActionRef.current) {
+        currentActionRef.current.fadeOut(fadeOutDuration);
+      }
+
+      idleAction?.reset().fadeIn(fadeInDuration).play();
+      currentActionRef.current = idleAction;
+
+      setTimeout(() => {
+        isTransitioningRef.current = false;
+      }, (fadeOutDuration + fadeInDuration) * 1000);
+    };
+
+    if (currentActionRef.current && !currentActionRef.current.getClip().name.startsWith('Idle')) {
+      finishCurrentAnimation();
+    } else {
+      startIdleAnimation();
+    }
+  }, [actions]);
+
+  // Handle base animation
+  useEffect(() => {
+    if (!actions || !currentBaseAction.action || isTransitioningRef.current)
+      return;
+
+    const newAction = actions[currentBaseAction.action];
+    if (!newAction) {
+      console.warn(
+        `Animation "${currentBaseAction.action}" not found in actions.`
+      );
+      return;
+    }
+
+    const fadeOutDuration = 0.8;
+    const fadeInDuration = 0.8;
+
+    // If the new action is not an Idle animation, set up the transition back to idle
+    if (!currentBaseAction.action.startsWith('Idle')) {
+      setTimeout(() => {
+        transitionToIdle();
+      }, ANIMATION_DURATION);
+    }
+
+    if (currentActionRef.current) {
+      currentActionRef.current.fadeOut(fadeOutDuration);
+    }
+
+    newAction.timeScale = timeScale;
+    newAction.reset().fadeIn(fadeInDuration).play();
+    currentActionRef.current = newAction;
+  }, [currentBaseAction, timeScale, actions, transitionToIdle]);
+
+  // Handle avatar blend shape animation
   useEffect(() => {
-    // Correct materials for the avatar
     correctMaterials(materials);
 
-    // Find the avatar mesh
     scene.traverse((object: Object3D) => {
-      if (object instanceof SkinnedMesh && object.name === 'Wolf3D_Avatar020') {
+      if (
+        object instanceof SkinnedMesh &&
+        (object.name === 'Wolf3D_Avatar020' || object.name === 'Wolf3D_Avatar')
+      ) {
         avatarMeshRef.current = object;
 
-        // Set up morph target dictionary and influences
         if (object.morphTargetDictionary && object.morphTargetInfluences) {
           setMorphTargetDictionary(object.morphTargetDictionary);
 
-          // Create an object with all morph target influences set to 0
           const initialInfluences = Object.keys(
             object.morphTargetDictionary
           ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
@@ -98,10 +191,8 @@ export default function FullbodyAvatar({
       }
     });
 
-    // Call onLoaded callback if provided
     onLoaded?.();
 
-    // Cleanup function
     return () => {
       Object.values(materials).forEach(dispose);
       Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
@@ -115,53 +206,29 @@ export default function FullbodyAvatar({
     setMorphTargetInfluences,
   ]);
 
-  // Effect to handle animation changes
-  useEffect(() => {
-    if (!actions || !currentBaseAction.action) return;
-
-    const newAction = actions[currentBaseAction.action];
-    if (!newAction) {
-      console.warn(
-        `Animation "${currentBaseAction.action}" not found in actions.`
-      );
-      return;
-    }
-
-    const fadeOutDuration = 0.8;
-    const fadeInDuration = 0.8;
-
-    // Set the timeScale and play the new action
-    newAction.timeScale = timeScale;
-    newAction.reset().fadeIn(fadeInDuration).play();
-
-    // Cleanup function to fade out the action
-    return () => {
-      newAction.fadeOut(fadeOutDuration);
-    };
-  }, [currentBaseAction, timeScale, actions]);
-
   // Frame update for morph target influences and animation mixer
-  useFrame((_, delta) => {
+  useFrame((state, delta) => {
     // Update morph target influences
     if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
+      handleMouthMovement(state.clock.elapsedTime);
+      updateMorphTargetInfluences();
+    }
+    // Update the animation mixer
+    mixer.update(delta * 0.001);
+
+    function updateMorphTargetInfluences() {
       Object.entries(morphTargetInfluences).forEach(([key, value]) => {
         const index = avatarMeshRef.current!.morphTargetDictionary![key];
-        if (
-          typeof index === 'number' &&
-          avatarMeshRef.current!.morphTargetInfluences
-        ) {
-          const currentValue =
-            avatarMeshRef.current!.morphTargetInfluences[index];
+        if (typeof index === 'number' &&
+          avatarMeshRef.current!.morphTargetInfluences) {
+          const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
           const smoothValue = lerp(currentValue, value, 0.1);
           avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
         }
       });
     }
-    // Update the animation mixer
-    mixer.update(delta * 0.001);
   });
 
-  // Render the avatar
   return (
     <group
       position={isZoomed ? AVATAR_POSITION_ZOOMED : AVATAR_POSITION}