diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx index f56a92e2..feaefb0c 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx @@ -49,7 +49,7 @@ export const AvatarView: React.FC = ({ chatEmission, showControls, animation, - loading, + // loading, url, sex, eyeBlink, @@ -75,21 +75,29 @@ export const AvatarView: React.FC = ({ const { createVisemeSequence, currentVisemes, clearVisemes } = useViseme(); // Set the morph target influences for the given emotions - const setEmotion = useCallback( - (action: string) => { - const emotionMap = { - Gioia: { eyesClosed: 0.5, mouthSmile: 1 }, - Rabbia: { eyesClosed: 1, mouthSmile: -0.5 }, - Sorpresa: { mouthSmile: 0.5, eyesClosed: -0.2 }, - Tristezza: { mouthSmile: -0.6, eyesClosed: 0.5 }, - Timore: { mouthSmile: -0.5, eyesClosed: 1 }, - default: { mouthSmile: 0, eyesClosed: 0 } - }; - const emotion = Object.keys(emotionMap).find(key => action.startsWith(key)) || 'default'; - setMorphTargetInfluences(emotionMap[emotion as keyof typeof emotionMap]); - }, - [] - ); + const setEmotion = useCallback((action: string) => { + const emotionMap: Record> = { + Gioia: { Gioria: 1 }, + Rabbia: { Rabbia: 1 }, + Sorpresa: { Sorpresa: 1 }, + Tristezza: { Tristezza: 1 }, + Timore: { Timore: 1 }, + }; + + const defaultEmotions = Object.keys(emotionMap).reduce((acc, key) => { + acc[key] = 0; + return acc; + }, {} as Record); + + const emotion = Object.keys(emotionMap).find(key => action.startsWith(key)) || 'default'; + const emotionValues = emotion === 'default' ? defaultEmotions : emotionMap[emotion]; + + setMorphTargetInfluences(prevInfluences => ({ + ...prevInfluences, + ...defaultEmotions, + ...emotionValues + })); + }, []); const onBaseActionChange = useCallback((action: string) => { setEmotion(action); @@ -122,8 +130,7 @@ export const AvatarView: React.FC = ({ // Set the emotion based on the chatEmission useEffect(() => { - - if(chatEmission){ + if (chatEmission) { createVisemeSequence(chatEmission); } @@ -144,19 +151,27 @@ export const AvatarView: React.FC = ({ //Choose a random number between 1 and 3 const randomNumber = Math.floor(Math.random() * 3) + 1; const emotion = `${outputContent}${randomNumber}`; - setEmotion(emotion); + + onBaseActionChange(emotion); } }, [chatEmission]); + const resetToIdle = useCallback(() => { + const randomIdle = Math.floor(Math.random() * 5) + 1; + setCurrentBaseAction({ + action: `Idle${randomIdle}`, + weight: 1, + }); + setMorphTargetInfluences({ mouthSmile: 0, eyesClosed: 0 }); + }, []); + + //Set a loading state to true if the avatar is loading - useEffect(() => { - if (loading) { - setCurrentBaseAction({ - action: 'Idle1', - weight: 1, - }); - } - }, [loading]); + // useEffect(() => { + // if (loading) { + // resetToIdle(); + // } + // }, [loading]); return ( <> diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/controls.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/controls.tsx index f74c8042..ebd543c9 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/controls.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/controls.tsx @@ -91,7 +91,7 @@ const AnimationControlPanel: React.FC = ({ modifyTimeScale, baseActions, morphTargetDictionary, - timeScale, + timeScale ]); return null; // This component doesn't render anything itself diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx index 6a9c413b..c8ef4af4 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx @@ -1,16 +1,22 @@ -// Import necessary dependencies -import React, { useEffect, useRef, useState } from 'react'; -import { Vector3, Euler, AnimationMixer, SkinnedMesh, Object3D } from 'three'; +import React, { useEffect, useRef, useState, useCallback } from 'react'; +import { + Vector3, + Euler, + AnimationMixer, + SkinnedMesh, + Object3D, + AnimationAction, +} from 'three'; import { useAnimations, useGLTF } from '@react-three/drei'; import { useGraph, dispose, useFrame } from '@react-three/fiber'; import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils'; import { useAvatarBlink } from '../../utils/useEyeBlink'; +import { useMouthAnimation } from '../../utils/useMouthAnimation'; const lerp = (start: number, end: number, alpha: number): number => { return start * (1 - alpha) + end * alpha; }; -// Define the props interface for the FullbodyAvatar component interface FullbodyAvatarProps { url: string; sex: 'MALE' | 'FEMALE'; @@ -28,21 +34,25 @@ interface FullbodyAvatarProps { morphTargetInfluences: { [key: string]: number }; morphTargetDictionary: { [key: string]: number }; eyeBlink?: boolean; + currentVisemes: { + name: string; + duration: number; + weight: number; + startTime: number; + }[]; } -// Define constants for avatar positioning const AVATAR_POSITION = new Vector3(0, -1, 0); const AVATAR_ROTATION = new Euler(0.175, 0, 0); const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0); -// Define URLs for male and female animation assets const ANIMATION_URLS = { MALE: 'https://assets.memori.ai/api/v2/asset/1c350a21-97d8-4add-82cc-9dc10767a26b.glb', FEMALE: - 'https://assets.memori.ai/api/v2/asset/a1908dbf-8ce8-438d-90df-acf9dc2604ad.glb', + 'https://assets.memori.ai/api/v2/asset/c2b07166-de10-4c66-918b-7b7cd380cca7.glb', }; +const ANIMATION_DURATION = 3000; // Duration in milliseconds for non-idle animations -// Define the FullbodyAvatar component export default function FullbodyAvatar({ url, sex, @@ -54,16 +64,21 @@ export default function FullbodyAvatar({ setMorphTargetDictionary, morphTargetInfluences, eyeBlink, + currentVisemes, }: FullbodyAvatarProps) { - // Load the 3D model and animations const { scene } = useGLTF(url); const { animations } = useGLTF(ANIMATION_URLS[sex]); const { nodes, materials } = useGraph(scene); const { actions } = useAnimations(animations, scene); const [mixer] = useState(() => new AnimationMixer(scene)); - // Create a ref for the SkinnedMesh const avatarMeshRef = useRef(); + const currentActionRef = useRef(null); + const isTransitioningRef = useRef(false); + const { handleMouthMovement } = useMouthAnimation({ + currentVisemes, + avatarMeshRef: avatarMeshRef as React.RefObject, + }); useAvatarBlink({ enabled: eyeBlink || false, @@ -75,21 +90,99 @@ export default function FullbodyAvatar({ }, }); - // Effect to setup morphTargets and cleanup + const transitionToIdle = useCallback(() => { + if (!actions || isTransitioningRef.current) return; + + isTransitioningRef.current = true; + + const finishCurrentAnimation = () => { + if (currentActionRef.current && !currentActionRef.current.paused) { + const remainingTime = (currentActionRef.current.getClip().duration - currentActionRef.current.time) * 1000; + setTimeout(() => { + startIdleAnimation(); + }, remainingTime); + } else { + startIdleAnimation(); + } + }; + + const startIdleAnimation = () => { + // Choose a random Idle animation + const idleAnimations = Object.keys(actions).filter(key => + key.startsWith('Idle') + ); + const randomIdle = + idleAnimations[Math.floor(Math.random() * idleAnimations.length)]; + + const idleAction = actions[randomIdle]; + const fadeOutDuration = 0.5; + const fadeInDuration = 0.5; + + if (currentActionRef.current) { + currentActionRef.current.fadeOut(fadeOutDuration); + } + + idleAction?.reset().fadeIn(fadeInDuration).play(); + currentActionRef.current = idleAction; + + setTimeout(() => { + isTransitioningRef.current = false; + }, (fadeOutDuration + fadeInDuration) * 1000); + }; + + if (currentActionRef.current && !currentActionRef.current.getClip().name.startsWith('Idle')) { + finishCurrentAnimation(); + } else { + startIdleAnimation(); + } + }, [actions]); + + // Handle base animation + useEffect(() => { + if (!actions || !currentBaseAction.action || isTransitioningRef.current) + return; + + const newAction = actions[currentBaseAction.action]; + if (!newAction) { + console.warn( + `Animation "${currentBaseAction.action}" not found in actions.` + ); + return; + } + + const fadeOutDuration = 0.8; + const fadeInDuration = 0.8; + + // If the new action is not an Idle animation, set up the transition back to idle + if (!currentBaseAction.action.startsWith('Idle')) { + setTimeout(() => { + transitionToIdle(); + }, ANIMATION_DURATION); + } + + if (currentActionRef.current) { + currentActionRef.current.fadeOut(fadeOutDuration); + } + + newAction.timeScale = timeScale; + newAction.reset().fadeIn(fadeInDuration).play(); + currentActionRef.current = newAction; + }, [currentBaseAction, timeScale, actions, transitionToIdle]); + + // Handle avatar blend shape animation useEffect(() => { - // Correct materials for the avatar correctMaterials(materials); - // Find the avatar mesh scene.traverse((object: Object3D) => { - if (object instanceof SkinnedMesh && object.name === 'Wolf3D_Avatar020') { + if ( + object instanceof SkinnedMesh && + (object.name === 'Wolf3D_Avatar020' || object.name === 'Wolf3D_Avatar') + ) { avatarMeshRef.current = object; - // Set up morph target dictionary and influences if (object.morphTargetDictionary && object.morphTargetInfluences) { setMorphTargetDictionary(object.morphTargetDictionary); - // Create an object with all morph target influences set to 0 const initialInfluences = Object.keys( object.morphTargetDictionary ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); @@ -98,10 +191,8 @@ export default function FullbodyAvatar({ } }); - // Call onLoaded callback if provided onLoaded?.(); - // Cleanup function return () => { Object.values(materials).forEach(dispose); Object.values(nodes).filter(isSkinnedMesh).forEach(dispose); @@ -115,53 +206,29 @@ export default function FullbodyAvatar({ setMorphTargetInfluences, ]); - // Effect to handle animation changes - useEffect(() => { - if (!actions || !currentBaseAction.action) return; - - const newAction = actions[currentBaseAction.action]; - if (!newAction) { - console.warn( - `Animation "${currentBaseAction.action}" not found in actions.` - ); - return; - } - - const fadeOutDuration = 0.8; - const fadeInDuration = 0.8; - - // Set the timeScale and play the new action - newAction.timeScale = timeScale; - newAction.reset().fadeIn(fadeInDuration).play(); - - // Cleanup function to fade out the action - return () => { - newAction.fadeOut(fadeOutDuration); - }; - }, [currentBaseAction, timeScale, actions]); - // Frame update for morph target influences and animation mixer - useFrame((_, delta) => { + useFrame((state, delta) => { // Update morph target influences if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) { + handleMouthMovement(state.clock.elapsedTime); + updateMorphTargetInfluences(); + } + // Update the animation mixer + mixer.update(delta * 0.001); + + function updateMorphTargetInfluences() { Object.entries(morphTargetInfluences).forEach(([key, value]) => { const index = avatarMeshRef.current!.morphTargetDictionary![key]; - if ( - typeof index === 'number' && - avatarMeshRef.current!.morphTargetInfluences - ) { - const currentValue = - avatarMeshRef.current!.morphTargetInfluences[index]; + if (typeof index === 'number' && + avatarMeshRef.current!.morphTargetInfluences) { + const currentValue = avatarMeshRef.current!.morphTargetInfluences[index]; const smoothValue = lerp(currentValue, value, 0.1); avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue; } }); } - // Update the animation mixer - mixer.update(delta * 0.001); }); - // Render the avatar return ( ; +} + +const lerp = (start: number, end: number, alpha: number): number => { + return start * (1 - alpha) + end * alpha; +}; + +export function useMouthAnimation({ currentVisemes, avatarMeshRef }: UseMouthAnimationProps) { + const visemeStartTimeRef = useRef(0); + const currentVisemeWeightRef = useRef<{ [key: string]: number }>({}); + + // Helper function to get current viseme information + const getCurrentVisemeInfo = (elapsedTime: number) => { + let currentVisemeIndex = 0; + let accumulatedDuration = 0; + + while ( + currentVisemeIndex < currentVisemes.length && + accumulatedDuration <= elapsedTime + ) { + accumulatedDuration += currentVisemes[currentVisemeIndex].duration; + currentVisemeIndex++; + } + + return { currentVisemeIndex, accumulatedDuration }; + }; + + // Helper function to apply the current viseme + const applyCurrentViseme = (index: number, elapsedTime: number, accumulatedDuration: number) => { + const currentViseme = currentVisemes[index - 1]; + const visemeProgress = (elapsedTime - (accumulatedDuration - currentViseme.duration)) / currentViseme.duration; + const targetWeight = Math.sin(visemeProgress * Math.PI) * currentViseme.weight; + + // Smooth the transition between visemes + if (!currentVisemeWeightRef.current[currentViseme.name]) { + currentVisemeWeightRef.current[currentViseme.name] = 0; + } + currentVisemeWeightRef.current[currentViseme.name] = lerp( + currentVisemeWeightRef.current[currentViseme.name], + targetWeight, + VISEME_SMOOTHING + ); + + const visemeIndex = avatarMeshRef.current?.morphTargetDictionary?.[currentViseme.name]; + if (typeof visemeIndex === 'number' && avatarMeshRef.current?.morphTargetInfluences) { + avatarMeshRef.current.morphTargetInfluences[visemeIndex] = currentVisemeWeightRef.current[currentViseme.name]; + } + }; + + // Helper function to reset viseme animation + const resetVisemeAnimation = (currentTime: number) => { + visemeStartTimeRef.current = currentTime; + currentVisemeWeightRef.current = {}; + }; + + // Main function to handle viseme-based mouth movement + const handleMouthMovement = (elapsedTime: number) => { + if (currentVisemes.length === 0) return; + + const currentTime = elapsedTime * VISEME_SPEED_FACTOR; + const visemeElapsedTime = currentTime - visemeStartTimeRef.current; + + const { currentVisemeIndex, accumulatedDuration } = getCurrentVisemeInfo(visemeElapsedTime); + + if (currentVisemeIndex > 0) { + applyCurrentViseme(currentVisemeIndex, visemeElapsedTime, accumulatedDuration); + } + + // Reset viseme animation if we've reached the end + if (visemeElapsedTime > accumulatedDuration) { + resetVisemeAnimation(currentTime); + } + }; + + return { + handleMouthMovement, + }; +} \ No newline at end of file diff --git a/src/components/Avatar/AvatarView/utils/useViseme.ts b/src/components/Avatar/AvatarView/utils/useViseme.ts index 7d922c30..6928ae5b 100644 --- a/src/components/Avatar/AvatarView/utils/useViseme.ts +++ b/src/components/Avatar/AvatarView/utils/useViseme.ts @@ -4,6 +4,7 @@ type Viseme = { name: string; duration: number; weight: number; + startTime: number; }; export const useViseme = () => { @@ -28,7 +29,7 @@ export const useViseme = () => { }; const createVisemeSequence = useCallback((text: string, durationPerPhoneme: number = 0.1) => { - // Simple mapping of Italian phonemes to visemes + // Improved mapping of Italian phonemes to visemes const phonemeToViseme = (char: string): string => { char = char.toUpperCase(); if ('AEIOU'.includes(char)) return visemeMap[char]; @@ -36,33 +37,61 @@ export const useViseme = () => { if ('FV'.includes(char)) return visemeMap['FF']; if ('TD'.includes(char)) return visemeMap['DD']; if ('KG'.includes(char)) return visemeMap['kk']; - if ('CS'.includes(char)) return visemeMap['SS']; + if ('CSZ'.includes(char)) return visemeMap['SS']; if ('NM'.includes(char)) return visemeMap['nn']; - if (char === 'R') return visemeMap['RR']; - if (char === 'L') return visemeMap['TH']; + if ('RL'.includes(char)) return visemeMap['RR']; + if ('GN'.includes(char)) return visemeMap['nn']; + if ('GL'.includes(char)) return visemeMap['TH']; return visemeMap['sil']; }; + let startTime = 0; const sequence: Viseme[] = []; const chars = text.split(''); - chars.forEach((char) => { + chars.forEach((char, index) => { if (char.trim() === '') { // Add a brief silence for spaces sequence.push({ name: visemeMap['sil'], duration: durationPerPhoneme / 2, - weight: 1 + weight: 1, + startTime, }); + startTime += durationPerPhoneme / 2; return; } const visemeName = phonemeToViseme(char); + const nextChar = chars[index + 1]; + const isLastChar = index === chars.length - 1; + + // Adjust duration and weight based on surrounding characters + let duration = durationPerPhoneme; + let weight = 0.8; + + if (!isLastChar && nextChar && phonemeToViseme(nextChar) === visemeName) { + duration *= 1.5; // Lengthen duration for consecutive same visemes + } else if (visemeName === visemeMap['sil']) { + duration *= 0.5; // Shorten duration for silence + weight = 0.5; + } + sequence.push({ name: visemeName, - duration: durationPerPhoneme, - weight: 1 + duration, + weight, + startTime, }); + startTime += duration; + }); + + // Add a final silence + sequence.push({ + name: visemeMap['sil'], + duration: durationPerPhoneme / 2, + weight: 0.5, + startTime, }); setCurrentVisemes(sequence); @@ -73,6 +102,8 @@ export const useViseme = () => { setCurrentVisemes([]); }, []); + + return { currentVisemes, createVisemeSequence,