From dea374098259671f4117ec6b77747f989088d7e1 Mon Sep 17 00:00:00 2001 From: andrepat0 Date: Tue, 8 Oct 2024 12:41:46 +0200 Subject: [PATCH] feat: removed unused hooks and linked context function for lips animation --- src/components/Avatar/Avatar.test.tsx | 48 ++++---- src/components/Avatar/Avatar.tsx | 6 + .../AvatarComponent/avatarComponent.tsx | 35 ++---- .../components/fullbodyAvatar.tsx | 40 +++---- .../components/halfbodyAvatar.tsx | 23 +++- src/components/Avatar/AvatarView/index.tsx | 8 +- .../AvatarView/utils/useMouthSpeaking.ts | 87 -------------- .../Avatar/AvatarView/utils/useViseme.ts | 112 ------------------ 8 files changed, 86 insertions(+), 273 deletions(-) delete mode 100644 src/components/Avatar/AvatarView/utils/useMouthSpeaking.ts delete mode 100644 src/components/Avatar/AvatarView/utils/useViseme.ts diff --git a/src/components/Avatar/Avatar.test.tsx b/src/components/Avatar/Avatar.test.tsx index b119daf2..603ed77c 100644 --- a/src/components/Avatar/Avatar.test.tsx +++ b/src/components/Avatar/Avatar.test.tsx @@ -2,29 +2,32 @@ import React from 'react'; import { render } from '@testing-library/react'; import { memori, tenant, integration } from '../../mocks/data'; import Avatar from './Avatar'; - +import { VisemeProvider } from '../../context/visemeContext'; const integrationConfig = JSON.parse(integration.customData ?? '{}'); it('renders defualt Avatar (blob) unchanged', () => { const { container } = render( - + {}} hasUserActivatedSpeak={false} - isPlayingAudio={false} - /> + isPlayingAudio={false} + /> + ); expect(container).toMatchSnapshot(); }); it('renders Avatar with blob and avatar in blob unchanged', () => { const { container } = render( - + { setAvatar3dVisible={() => {}} hasUserActivatedSpeak={false} isPlayingAudio={false} - /> + /> + ); expect(container).toMatchSnapshot(); }); it('renders Avatar with custom glb model unchanged', () => { const { container } = render( - + { setAvatar3dVisible={() => {}} hasUserActivatedSpeak={false} isPlayingAudio={false} - /> + /> + ); expect(container).toMatchSnapshot(); }); it('renders Avatar with rpm 3d avatar unchanged', () => { - const { container } = render( - + { instruct={false} avatar3dVisible={true} setAvatar3dVisible={() => {}} - hasUserActivatedSpeak={false} - isPlayingAudio={false} - /> + hasUserActivatedSpeak={false} + isPlayingAudio={false} + /> + ); expect(container).toMatchSnapshot(); }); diff --git a/src/components/Avatar/Avatar.tsx b/src/components/Avatar/Avatar.tsx index 6d7590d2..85552154 100644 --- a/src/components/Avatar/Avatar.tsx +++ b/src/components/Avatar/Avatar.tsx @@ -16,6 +16,7 @@ import EyeInvisible from '../icons/EyeInvisible'; import Edit from '../icons/Edit'; import cx from 'classnames'; import ContainerAvatarView from './AvatarView'; +import { useViseme } from '../../context/visemeContext'; export interface Props { memori: Memori; @@ -52,9 +53,12 @@ const Avatar: React.FC = ({ isZoomed = false, chatProps, }) => { + const { t } = useTranslation(); const [isClient, setIsClient] = useState(false); + const { setMeshRef, clearVisemes } = useViseme(); + useEffect(() => { setIsClient(true); }, []); @@ -137,6 +141,8 @@ const Avatar: React.FC = ({ speaking={isPlayingAudio} loading={loading} style={getAvatarStyle()} + clearVisemes={clearVisemes} + setMeshRef={setMeshRef} isZoomed={isZoomed} chatEmission={chatProps?.dialogState?.emission} /> diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx index feaefb0c..4a2221d3 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx @@ -2,7 +2,6 @@ import React, { useState, useEffect, useCallback } from 'react'; import AnimationControlPanel from './components/controls'; import FullbodyAvatar from './components/fullbodyAvatar'; import HalfBodyAvatar from './components/halfbodyAvatar'; -import { useViseme } from '../utils/useViseme'; interface Props { showControls: boolean; @@ -15,6 +14,8 @@ interface Props { speaking: boolean; isZoomed: boolean; chatEmission: any; + setMeshRef: any; + clearVisemes: () => void; } interface BaseAction { @@ -46,10 +47,11 @@ const baseActions: Record = { }; export const AvatarView: React.FC = ({ + setMeshRef, + clearVisemes, chatEmission, showControls, animation, - // loading, url, sex, eyeBlink, @@ -63,6 +65,7 @@ export const AvatarView: React.FC = ({ weight: 1, }); + const [morphTargetInfluences, setMorphTargetInfluences] = useState<{ [key: string]: number; }>({}); @@ -72,8 +75,6 @@ export const AvatarView: React.FC = ({ const [timeScale, setTimeScale] = useState(0.8); - const { createVisemeSequence, currentVisemes, clearVisemes } = useViseme(); - // Set the morph target influences for the given emotions const setEmotion = useCallback((action: string) => { const emotionMap: Record> = { @@ -107,6 +108,7 @@ export const AvatarView: React.FC = ({ }); }, []); + const onMorphTargetInfluencesChange = useCallback( (influences: { [key: string]: number }) => { setMorphTargetInfluences(prevInfluences => ({ @@ -130,9 +132,6 @@ export const AvatarView: React.FC = ({ // Set the emotion based on the chatEmission useEffect(() => { - if (chatEmission) { - createVisemeSequence(chatEmission); - } //Check if chatEmission has a tag const hasOutputTag = chatEmission?.includes( @@ -156,23 +155,6 @@ export const AvatarView: React.FC = ({ } }, [chatEmission]); - const resetToIdle = useCallback(() => { - const randomIdle = Math.floor(Math.random() * 5) + 1; - setCurrentBaseAction({ - action: `Idle${randomIdle}`, - weight: 1, - }); - setMorphTargetInfluences({ mouthSmile: 0, eyesClosed: 0 }); - }, []); - - - //Set a loading state to true if the avatar is loading - // useEffect(() => { - // if (loading) { - // resetToIdle(); - // } - // }, [loading]); - return ( <> {showControls && ( @@ -190,9 +172,11 @@ export const AvatarView: React.FC = ({ {halfBody ? ( ) : ( = ({ morphTargetInfluences={morphTargetInfluences} morphTargetDictionary={morphTargetDictionary} isZoomed={isZoomed} - currentVisemes={currentVisemes} + setMeshRef={setMeshRef} + clearVisemes={clearVisemes} /> )} diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx index c8ef4af4..2dedbbfd 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx @@ -11,7 +11,7 @@ import { useAnimations, useGLTF } from '@react-three/drei'; import { useGraph, dispose, useFrame } from '@react-three/fiber'; import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils'; import { useAvatarBlink } from '../../utils/useEyeBlink'; -import { useMouthAnimation } from '../../utils/useMouthAnimation'; +import { useViseme } from '../../../../../context/visemeContext'; const lerp = (start: number, end: number, alpha: number): number => { return start * (1 - alpha) + end * alpha; @@ -33,13 +33,9 @@ interface FullbodyAvatarProps { setMorphTargetDictionary: (dictionary: { [key: string]: number }) => void; morphTargetInfluences: { [key: string]: number }; morphTargetDictionary: { [key: string]: number }; + setMeshRef: any; eyeBlink?: boolean; - currentVisemes: { - name: string; - duration: number; - weight: number; - startTime: number; - }[]; + clearVisemes: () => void; } const AVATAR_POSITION = new Vector3(0, -1, 0); @@ -64,7 +60,8 @@ export default function FullbodyAvatar({ setMorphTargetDictionary, morphTargetInfluences, eyeBlink, - currentVisemes, + setMeshRef, + clearVisemes, }: FullbodyAvatarProps) { const { scene } = useGLTF(url); const { animations } = useGLTF(ANIMATION_URLS[sex]); @@ -75,11 +72,8 @@ export default function FullbodyAvatar({ const avatarMeshRef = useRef(); const currentActionRef = useRef(null); const isTransitioningRef = useRef(false); - const { handleMouthMovement } = useMouthAnimation({ - currentVisemes, - avatarMeshRef: avatarMeshRef as React.RefObject, - }); - + + // Blink animation useAvatarBlink({ enabled: eyeBlink || false, setMorphTargetInfluences, @@ -90,6 +84,7 @@ export default function FullbodyAvatar({ }, }); + // Idle animation when emotion animation is finished const transitionToIdle = useCallback(() => { if (!actions || isTransitioningRef.current) return; @@ -107,7 +102,6 @@ export default function FullbodyAvatar({ }; const startIdleAnimation = () => { - // Choose a random Idle animation const idleAnimations = Object.keys(actions).filter(key => key.startsWith('Idle') ); @@ -137,7 +131,7 @@ export default function FullbodyAvatar({ } }, [actions]); - // Handle base animation + // Base animation useEffect(() => { if (!actions || !currentBaseAction.action || isTransitioningRef.current) return; @@ -153,7 +147,6 @@ export default function FullbodyAvatar({ const fadeOutDuration = 0.8; const fadeInDuration = 0.8; - // If the new action is not an Idle animation, set up the transition back to idle if (!currentBaseAction.action.startsWith('Idle')) { setTimeout(() => { transitionToIdle(); @@ -169,7 +162,7 @@ export default function FullbodyAvatar({ currentActionRef.current = newAction; }, [currentBaseAction, timeScale, actions, transitionToIdle]); - // Handle avatar blend shape animation + // Set up the mesh reference and morph target influences useEffect(() => { correctMaterials(materials); @@ -179,6 +172,7 @@ export default function FullbodyAvatar({ (object.name === 'Wolf3D_Avatar020' || object.name === 'Wolf3D_Avatar') ) { avatarMeshRef.current = object; + setMeshRef(object); if (object.morphTargetDictionary && object.morphTargetInfluences) { setMorphTargetDictionary(object.morphTargetDictionary); @@ -196,6 +190,7 @@ export default function FullbodyAvatar({ return () => { Object.values(materials).forEach(dispose); Object.values(nodes).filter(isSkinnedMesh).forEach(dispose); + clearVisemes(); }; }, [ materials, @@ -204,16 +199,15 @@ export default function FullbodyAvatar({ onLoaded, setMorphTargetDictionary, setMorphTargetInfluences, + setMeshRef, + clearVisemes, ]); - // Frame update for morph target influences and animation mixer - useFrame((state, delta) => { - // Update morph target influences + // Update morph target influences + useFrame((_, delta) => { if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) { - handleMouthMovement(state.clock.elapsedTime); updateMorphTargetInfluences(); } - // Update the animation mixer mixer.update(delta * 0.001); function updateMorphTargetInfluences() { @@ -237,4 +231,4 @@ export default function FullbodyAvatar({ ); -} +} \ No newline at end of file diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx index 983ea2dd..e80ef799 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx @@ -1,11 +1,10 @@ import React, { useEffect, useMemo } from 'react'; -import { Object3D, Vector3 } from 'three'; +import { Object3D, SkinnedMesh, Vector3 } from 'three'; import { useGLTF } from '@react-three/drei'; import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils'; import { useGraph, dispose } from '@react-three/fiber'; import { useAvatarBlink } from '../../utils/useEyeBlink'; import useHeadMovement from '../../utils/useHeadMovement'; -import useMouthSpeaking from '../../utils/useMouthSpeaking'; import { hideHands } from '../../utils/utils'; @@ -15,6 +14,8 @@ interface HalfBodyAvatarProps { headMovement?: boolean; speaking?: boolean; onLoaded?: () => void; + setMeshRef: (mesh: Object3D) => void; + clearVisemes: () => void; } const AVATAR_POSITION = new Vector3(0, -0.6, 0); @@ -23,8 +24,9 @@ export default function HalfBodyAvatar({ url, setMorphTargetInfluences, headMovement, - speaking, + setMeshRef, onLoaded, + clearVisemes, }: HalfBodyAvatarProps) { const { scene } = useGLTF(url); const { nodes, materials } = useGraph(scene); @@ -39,12 +41,22 @@ export default function HalfBodyAvatar({ } }); useHeadMovement(headMovement, nodes); - useMouthSpeaking(!!speaking, nodes); useEffect(() => { const setupAvatar = () => { hideHands(nodes); correctMaterials(materials); + // Set mesh reference for the first SkinnedMesh found + const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh; + if (firstSkinnedMesh) { + setMeshRef(firstSkinnedMesh); + if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) { + const initialInfluences = Object.keys( + firstSkinnedMesh.morphTargetDictionary + ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); + setMorphTargetInfluences(initialInfluences); + } + } onLoaded?.(); }; @@ -54,11 +66,12 @@ export default function HalfBodyAvatar({ const disposeObjects = () => { Object.values(materials).forEach(dispose); Object.values(nodes).filter(isSkinnedMesh).forEach(dispose); + clearVisemes(); }; disposeObjects(); }; - }, [materials, nodes, url, onLoaded]); + }, [materials, nodes, url, onLoaded, clearVisemes]); const skinnedMeshes = useMemo( () => Object.values(nodes).filter(isSkinnedMesh), diff --git a/src/components/Avatar/AvatarView/index.tsx b/src/components/Avatar/AvatarView/index.tsx index 790c3e0c..b1c6e706 100644 --- a/src/components/Avatar/AvatarView/index.tsx +++ b/src/components/Avatar/AvatarView/index.tsx @@ -22,6 +22,8 @@ export interface Props { showControls?: boolean; isZoomed?: boolean; chatEmission?: any; + setMeshRef?: any; + clearVisemes: () => void; } const defaultStyles = { @@ -85,6 +87,8 @@ export default function ContainerAvatarView({ showControls = true, isZoomed, chatEmission, + setMeshRef, + clearVisemes, }: Props) { return ( + setMeshRef={setMeshRef} + clearVisemes={clearVisemes} + /> ); diff --git a/src/components/Avatar/AvatarView/utils/useMouthSpeaking.ts b/src/components/Avatar/AvatarView/utils/useMouthSpeaking.ts deleted file mode 100644 index f0335047..00000000 --- a/src/components/Avatar/AvatarView/utils/useMouthSpeaking.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { Nodes } from './utils'; -import { SkinnedMesh } from 'three'; -import { useEffect, useRef, useCallback } from 'react'; -import { useFrame } from '@react-three/fiber'; - -interface MouthState { - moveTime: number; - mesh: SkinnedMesh | null; - morphIndices: { - open: number; - smile: number; - funner: number; - pucker: number; - }; -} - -const MOUTH_MOVE_DURATION = 2; -const MOUTH_MOVE_INTERVAL_MIN = 100; -const MOUTH_MOVE_INTERVAL_MAX = 500; - -export default function useMouthSpeaking(speaking: boolean | undefined, nodes: Nodes) { - const mouthStateRef = useRef({ - moveTime: 999, - mesh: null, - morphIndices: { open: 0, smile: 0, funner: 0, pucker: 0 }, - }); - - const setNextMouthMove = useCallback(() => { - mouthStateRef.current.moveTime = 0; - const nextMoveDelay = Math.random() * (MOUTH_MOVE_INTERVAL_MAX - MOUTH_MOVE_INTERVAL_MIN) + MOUTH_MOVE_INTERVAL_MIN; - setTimeout(setNextMouthMove, nextMoveDelay); - }, []); - - useEffect(() => { - if (!speaking) return; - - const mouthMesh = (nodes.Wolf3D_Head || nodes.Wolf3D_Avatar020 || nodes.Wolf3D_Avatar001) as SkinnedMesh; - mouthStateRef.current.mesh = mouthMesh; - - if (mouthMesh?.morphTargetDictionary && mouthMesh?.morphTargetInfluences) { - mouthStateRef.current.morphIndices = { - open: mouthMesh.morphTargetDictionary.mouthOpen, - smile: mouthMesh.morphTargetDictionary.mouthSmile, - funner: mouthMesh.morphTargetDictionary.mouthFunner, - pucker: mouthMesh.morphTargetDictionary.mouthPucker, - }; - } - - const initialMoveDelay = setTimeout(setNextMouthMove, 200); - - return () => { - clearTimeout(initialMoveDelay); - }; - }, [nodes, speaking, setNextMouthMove]); - - useFrame((_, delta) => { - const { moveTime, mesh, morphIndices } = mouthStateRef.current; - - if (!speaking || !mesh?.morphTargetInfluences) { - resetMouthShape(mesh, morphIndices); - return; - } - - if (moveTime < MOUTH_MOVE_DURATION) { - const value = Math.abs(Math.sin((moveTime * Math.PI) / 2)); - mouthStateRef.current.moveTime += delta * 10; - updateMouthShape(mesh, morphIndices, value); - } else { - resetMouthShape(mesh, morphIndices); - } - }); -} - -function updateMouthShape(mesh: SkinnedMesh, morphIndices: MouthState['morphIndices'], value: number) { - mesh.morphTargetInfluences![morphIndices.open] = value / 3; - mesh.morphTargetInfluences![morphIndices.smile] = value / 10; - mesh.morphTargetInfluences![morphIndices.funner] = value / 7; - mesh.morphTargetInfluences![morphIndices.pucker] = value / 5; -} - -function resetMouthShape(mesh: SkinnedMesh | null, morphIndices: MouthState['morphIndices']) { - if (!mesh?.morphTargetInfluences) return; - mesh.morphTargetInfluences[morphIndices.open] = 0; - mesh.morphTargetInfluences[morphIndices.smile] = 0; - mesh.morphTargetInfluences[morphIndices.funner] = 0; - mesh.morphTargetInfluences[morphIndices.pucker] = 0; -} diff --git a/src/components/Avatar/AvatarView/utils/useViseme.ts b/src/components/Avatar/AvatarView/utils/useViseme.ts deleted file mode 100644 index 6928ae5b..00000000 --- a/src/components/Avatar/AvatarView/utils/useViseme.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { useState, useCallback } from 'react'; - -type Viseme = { - name: string; - duration: number; - weight: number; - startTime: number; -}; - -export const useViseme = () => { - const [currentVisemes, setCurrentVisemes] = useState([]); - - const visemeMap: { [key: string]: string } = { - 'A': 'viseme_aa', - 'E': 'viseme_E', - 'I': 'viseme_I', - 'O': 'viseme_O', - 'U': 'viseme_U', - 'PP': 'viseme_PP', - 'FF': 'viseme_FF', - 'TH': 'viseme_TH', - 'DD': 'viseme_DD', - 'kk': 'viseme_kk', - 'CH': 'viseme_CH', - 'SS': 'viseme_SS', - 'nn': 'viseme_nn', - 'RR': 'viseme_RR', - 'sil': 'viseme_sil', - }; - - const createVisemeSequence = useCallback((text: string, durationPerPhoneme: number = 0.1) => { - // Improved mapping of Italian phonemes to visemes - const phonemeToViseme = (char: string): string => { - char = char.toUpperCase(); - if ('AEIOU'.includes(char)) return visemeMap[char]; - if ('BP'.includes(char)) return visemeMap['PP']; - if ('FV'.includes(char)) return visemeMap['FF']; - if ('TD'.includes(char)) return visemeMap['DD']; - if ('KG'.includes(char)) return visemeMap['kk']; - if ('CSZ'.includes(char)) return visemeMap['SS']; - if ('NM'.includes(char)) return visemeMap['nn']; - if ('RL'.includes(char)) return visemeMap['RR']; - if ('GN'.includes(char)) return visemeMap['nn']; - if ('GL'.includes(char)) return visemeMap['TH']; - return visemeMap['sil']; - }; - - let startTime = 0; - const sequence: Viseme[] = []; - const chars = text.split(''); - - chars.forEach((char, index) => { - if (char.trim() === '') { - // Add a brief silence for spaces - sequence.push({ - name: visemeMap['sil'], - duration: durationPerPhoneme / 2, - weight: 1, - startTime, - }); - startTime += durationPerPhoneme / 2; - return; - } - - const visemeName = phonemeToViseme(char); - const nextChar = chars[index + 1]; - const isLastChar = index === chars.length - 1; - - // Adjust duration and weight based on surrounding characters - let duration = durationPerPhoneme; - let weight = 0.8; - - if (!isLastChar && nextChar && phonemeToViseme(nextChar) === visemeName) { - duration *= 1.5; // Lengthen duration for consecutive same visemes - } else if (visemeName === visemeMap['sil']) { - duration *= 0.5; // Shorten duration for silence - weight = 0.5; - } - - sequence.push({ - name: visemeName, - duration, - weight, - startTime, - }); - startTime += duration; - }); - - // Add a final silence - sequence.push({ - name: visemeMap['sil'], - duration: durationPerPhoneme / 2, - weight: 0.5, - startTime, - }); - - setCurrentVisemes(sequence); - return sequence; - }, []); - - const clearVisemes = useCallback(() => { - setCurrentVisemes([]); - }, []); - - - - return { - currentVisemes, - createVisemeSequence, - clearVisemes - }; -}; \ No newline at end of file