From c188673485078a6f001bc09f99c46a9349e5c184 Mon Sep 17 00:00:00 2001 From: andrepat0 Date: Tue, 8 Oct 2024 16:42:33 +0200 Subject: [PATCH] feat: added emotion to XML audio tag --- src/components/Avatar/Avatar.tsx | 3 +- .../AvatarComponent/avatarComponent.tsx | 10 +- .../components/halfbodyAvatar.tsx | 2 +- src/components/Avatar/AvatarView/index.tsx | 3 + src/components/MemoriWidget/MemoriWidget.tsx | 16 +- src/context/visemeContext.tsx | 222 ++++++++++++------ 6 files changed, 170 insertions(+), 86 deletions(-) diff --git a/src/components/Avatar/Avatar.tsx b/src/components/Avatar/Avatar.tsx index 85552154..13956bca 100644 --- a/src/components/Avatar/Avatar.tsx +++ b/src/components/Avatar/Avatar.tsx @@ -57,7 +57,7 @@ const Avatar: React.FC = ({ const { t } = useTranslation(); const [isClient, setIsClient] = useState(false); - const { setMeshRef, clearVisemes } = useViseme(); + const { setMeshRef, clearVisemes, setEmotion } = useViseme(); useEffect(() => { setIsClient(true); @@ -145,6 +145,7 @@ const Avatar: React.FC = ({ setMeshRef={setMeshRef} isZoomed={isZoomed} chatEmission={chatProps?.dialogState?.emission} + setEmotion={setEmotion} /> ); diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx index 2326dcf8..7e06af85 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx @@ -16,6 +16,7 @@ interface Props { chatEmission: any; setMeshRef: any; clearVisemes: () => void; + setEmotion: (emotion: string) => void; } interface BaseAction { @@ -59,6 +60,7 @@ export const AvatarView: React.FC = ({ speaking, halfBody, isZoomed, + setEmotion, }) => { const [currentBaseAction, setCurrentBaseAction] = useState({ action: animation || 'Idle1', @@ -76,7 +78,7 @@ export const AvatarView: React.FC = ({ const [timeScale, setTimeScale] = useState(0.8); // Set the morph target influences for the given emotions - const setEmotion = useCallback((action: string) => { + const setEmotionMorphTargetInfluences = useCallback((action: string) => { const emotionMap: Record> = { Gioia: { Gioria: 1 }, Rabbia: { Rabbia: 1 }, @@ -85,6 +87,10 @@ export const AvatarView: React.FC = ({ Timore: { Timore: 1 }, }; + //remove the last character from the action + const newEmotion = action.slice(0, -1); + setEmotion(newEmotion); + const defaultEmotions = Object.keys(emotionMap).reduce((acc, key) => { acc[key] = 0; return acc; @@ -101,7 +107,7 @@ export const AvatarView: React.FC = ({ }, []); const onBaseActionChange = useCallback((action: string) => { - setEmotion(action); + setEmotionMorphTargetInfluences(action); setCurrentBaseAction({ action, weight: 1, diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx index 31065fe8..4f0ad93f 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx @@ -90,7 +90,7 @@ export default function HalfBodyAvatar({ ); // Update morph target influences - useFrame((_, delta) => { + useFrame((_) => { if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) { updateMorphTargetInfluences(); } diff --git a/src/components/Avatar/AvatarView/index.tsx b/src/components/Avatar/AvatarView/index.tsx index b1c6e706..95327c3c 100644 --- a/src/components/Avatar/AvatarView/index.tsx +++ b/src/components/Avatar/AvatarView/index.tsx @@ -24,6 +24,7 @@ export interface Props { chatEmission?: any; setMeshRef?: any; clearVisemes: () => void; + setEmotion: (emotion: string) => void; } const defaultStyles = { @@ -89,6 +90,7 @@ export default function ContainerAvatarView({ chatEmission, setMeshRef, clearVisemes, + setEmotion, }: Props) { return ( diff --git a/src/components/MemoriWidget/MemoriWidget.tsx b/src/components/MemoriWidget/MemoriWidget.tsx index e3727eea..9e4a60df 100644 --- a/src/components/MemoriWidget/MemoriWidget.tsx +++ b/src/components/MemoriWidget/MemoriWidget.tsx @@ -538,7 +538,13 @@ const MemoriWidget = ({ ); const [hideEmissions, setHideEmissions] = useState(false); - const { addVisemeToQueue, processVisemeQueue, clearVisemes } = useViseme(); + const { + addVisemeToQueue, + processVisemeQueue, + clearVisemes, + emotion, + getAzureStyleForEmotion, + } = useViseme(); useEffect(() => { setIsPlayingAudio(!!speechSynthesizer); @@ -1962,13 +1968,13 @@ const MemoriWidget = ({ }); }; - speechSynthesizer.speakSsmlAsync( - `${replaceTextWithPhonemes( + )}"> ${replaceTextWithPhonemes( escapeHTML(stripMarkdown(stripEmojis(stripOutputTags(text)))), userLang.toLowerCase() - )}`, + )}`, result => { if (result) { setIsPlayingAudio(true); diff --git a/src/context/visemeContext.tsx b/src/context/visemeContext.tsx index ed3bf2b6..78ac37aa 100644 --- a/src/context/visemeContext.tsx +++ b/src/context/visemeContext.tsx @@ -1,4 +1,11 @@ -import React, { createContext, useContext, useState, useCallback, useRef, useEffect } from 'react'; +import React, { + createContext, + useContext, + useState, + useCallback, + useRef, + useEffect, +} from 'react'; import { SkinnedMesh } from 'three'; type AzureViseme = { visemeId: number; audioOffset: number }; @@ -16,6 +23,9 @@ interface VisemeContextType { processVisemeQueue: () => ProcessedViseme[]; clearVisemes: () => void; isMeshSet: boolean; + setEmotion: (emotion: string) => void; + emotion: string; + getAzureStyleForEmotion: (emotion: string) => string; } const VisemeContext = createContext(undefined); @@ -53,8 +63,11 @@ const VISEME_MAP: { [key: number]: string } = { 21: 'viseme_PP', // y (closest match, could be debated) }; -export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { +export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ + children, +}) => { const [isMeshSet, setIsMeshSet] = useState(false); + const [emotion, setEmotion] = useState('Neutral'); const isAnimatingRef = useRef(false); const currentVisemesRef = useRef([]); const visemeQueueRef = useRef([]); @@ -71,15 +84,18 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr return x < 0.5 ? 4 * x * x * x : 1 - Math.pow(-2 * x + 2, 3) / 2; }; - const setMeshRef = useCallback((mesh: SkinnedMesh | null) => { - if (mesh && mesh.morphTargetDictionary && mesh.morphTargetInfluences) { - meshRef.current = mesh; - setIsMeshSet(true); - // console.log('Mesh set successfully:', mesh); - } else { - console.error('Invalid mesh provided:', mesh); - } - }, [meshRef]); + const setMeshRef = useCallback( + (mesh: SkinnedMesh | null) => { + if (mesh && mesh.morphTargetDictionary && mesh.morphTargetInfluences) { + meshRef.current = mesh; + setIsMeshSet(true); + // console.log('Mesh set successfully:', mesh); + } else { + console.error('Invalid mesh provided:', mesh); + } + }, + [meshRef] + ); const addVisemeToQueue = useCallback((viseme: AzureViseme) => { visemeQueueRef.current.push(viseme); @@ -100,84 +116,100 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr const getDynamicSpeedFactor = (visemeDuration: number): number => { const baseDuration = 0.1; // Average expected viseme duration - return VISEME_BASE_SPEED * (baseDuration / visemeDuration) * AUDIO_PLAYBACK_RATE; - }; - - const applyViseme = useCallback((viseme: ProcessedViseme, elapsedTime: number) => { - if (!meshRef.current) { - console.error('Mesh not set'); - return; - } - - const visemeProgress = Math.min( - (elapsedTime - viseme.startTime) / viseme.duration, - 1 - ); - - const dynamicSpeedFactor = getDynamicSpeedFactor(viseme.duration); - const adjustedProgress = visemeProgress * dynamicSpeedFactor; - - // Use a cubic easing function for smoother transitions - const easedProgress = easeInOutCubic(adjustedProgress); - const targetWeight = Math.sin(easedProgress * Math.PI) * viseme.weight; - - currentVisemeWeightRef.current[viseme.name] = lerp( - currentVisemeWeightRef.current[viseme.name] || 0, - targetWeight, - VISEME_SMOOTHING + return ( + VISEME_BASE_SPEED * (baseDuration / visemeDuration) * AUDIO_PLAYBACK_RATE ); - - const visemeIndex = meshRef.current.morphTargetDictionary?.[viseme.name]; - if (typeof visemeIndex === 'number' && meshRef.current.morphTargetInfluences) { - meshRef.current.morphTargetInfluences[visemeIndex] = currentVisemeWeightRef.current[viseme.name]; - // console.log(`Applied viseme: ${viseme.name}, weight: ${currentVisemeWeightRef.current[viseme.name]}`); - } else { - console.error(`Viseme not found in morph target dictionary: ${viseme.name}`); - } - }, []); + }; - const animate = useCallback((time: number) => { - if (startTimeRef.current === null) { - startTimeRef.current = time; - } + const applyViseme = useCallback( + (viseme: ProcessedViseme, elapsedTime: number) => { + if (!meshRef.current) { + console.error('Mesh not set'); + return; + } - const elapsedTime = (time - startTimeRef.current) / 1000 * VISEME_SPEED_FACTOR; + const visemeProgress = Math.min( + (elapsedTime - viseme.startTime) / viseme.duration, + 1 + ); - const currentViseme = getCurrentViseme(elapsedTime); + const dynamicSpeedFactor = getDynamicSpeedFactor(viseme.duration); + const adjustedProgress = visemeProgress * dynamicSpeedFactor; - if (currentViseme) { - applyViseme(currentViseme, elapsedTime); - } + // Use a cubic easing function for smoother transitions + const easedProgress = easeInOutCubic(adjustedProgress); + const targetWeight = Math.sin(easedProgress * Math.PI) * viseme.weight; - if ( - currentVisemesRef.current.length > 0 && - elapsedTime < - currentVisemesRef.current[currentVisemesRef.current.length - 1].startTime + - currentVisemesRef.current[currentVisemesRef.current.length - 1].duration - ) { - animationFrameRef.current = requestAnimationFrame(animate); - } else { - clearVisemes(); - } - }, [getCurrentViseme, applyViseme]); + currentVisemeWeightRef.current[viseme.name] = lerp( + currentVisemeWeightRef.current[viseme.name] || 0, + targetWeight, + VISEME_SMOOTHING + ); + + const visemeIndex = meshRef.current.morphTargetDictionary?.[viseme.name]; + if ( + typeof visemeIndex === 'number' && + meshRef.current.morphTargetInfluences + ) { + meshRef.current.morphTargetInfluences[visemeIndex] = + currentVisemeWeightRef.current[viseme.name]; + // console.log(`Applied viseme: ${viseme.name}, weight: ${currentVisemeWeightRef.current[viseme.name]}`); + } else { + console.error( + `Viseme not found in morph target dictionary: ${viseme.name}` + ); + } + }, + [] + ); + + const animate = useCallback( + (time: number) => { + if (startTimeRef.current === null) { + startTimeRef.current = time; + } + + const elapsedTime = + ((time - startTimeRef.current) / 1000) * VISEME_SPEED_FACTOR; + + const currentViseme = getCurrentViseme(elapsedTime); + + if (currentViseme) { + applyViseme(currentViseme, elapsedTime); + } + + if ( + currentVisemesRef.current.length > 0 && + elapsedTime < + currentVisemesRef.current[currentVisemesRef.current.length - 1] + .startTime + + currentVisemesRef.current[currentVisemesRef.current.length - 1] + .duration + ) { + animationFrameRef.current = requestAnimationFrame(animate); + } else { + clearVisemes(); + } + }, + [getCurrentViseme, applyViseme] + ); const processVisemeQueue = useCallback(() => { - const azureVisemes = [...visemeQueueRef.current]; visemeQueueRef.current = []; - + if (azureVisemes.length === 0) { // console.log('No visemes to process'); return []; } - + const processedVisemes: ProcessedViseme[] = azureVisemes.map( (currentViseme, i) => { const nextViseme = azureVisemes[i + 1]; const duration = nextViseme ? (nextViseme.audioOffset - currentViseme.audioOffset) / 10000000 : DEFAULT_VISEME_DURATION; - + const processedViseme = { name: VISEME_MAP[currentViseme.visemeId] || 'viseme_sil', duration, @@ -188,9 +220,9 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr return processedViseme; } ); - + currentVisemesRef.current = processedVisemes; - + // Start animation immediately if not already animating if (!isAnimatingRef.current) { isAnimatingRef.current = true; @@ -201,11 +233,13 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr // If already animating, adjust the start time for the new visemes if (startTimeRef.current !== null) { const currentTime = performance.now(); - const elapsedTime = (currentTime - startTimeRef.current) / 1000 * VISEME_SPEED_FACTOR; - startTimeRef.current = currentTime - (elapsedTime / VISEME_SPEED_FACTOR) * 1000; + const elapsedTime = + ((currentTime - startTimeRef.current) / 1000) * VISEME_SPEED_FACTOR; + startTimeRef.current = + currentTime - (elapsedTime / VISEME_SPEED_FACTOR) * 1000; } } - + return processedVisemes; }, [isMeshSet, animate]); @@ -218,7 +252,10 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr animationFrameRef.current = null; } - if (meshRef.current?.morphTargetDictionary && meshRef.current?.morphTargetInfluences) { + if ( + meshRef.current?.morphTargetDictionary && + meshRef.current?.morphTargetInfluences + ) { Object.values(meshRef.current.morphTargetDictionary).forEach(index => { if (typeof index === 'number') { meshRef.current!.morphTargetInfluences![index] = 0; @@ -232,6 +269,30 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr // console.log('Visemes cleared'); }, []); + // Your existing emotion map + const emotionMap: Record> = { + Gioia: { Gioria: 1 }, + Rabbia: { Rabbia: 1 }, + Sorpresa: { Sorpresa: 1 }, + Tristezza: { Tristezza: 1 }, + Timore: { Timore: 1 }, + }; + + // Mapping from your emotions to Azure styles + const emotionToAzureStyleMap: Record = { + Gioia: 'cheerful', + Rabbia: 'angry', + Sorpresa: 'excited', + Tristezza: 'sad', + Timore: 'terrified', + }; + + // Function to get Azure style from emotion + function getAzureStyleForEmotion(emotion: string): string { + return emotionToAzureStyleMap[emotion] || 'neutral'; + } + + useEffect(() => { return () => { if (animationFrameRef.current !== null) { @@ -246,9 +307,16 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr processVisemeQueue, clearVisemes, isMeshSet, + setEmotion, + emotion, + getAzureStyleForEmotion, }; - return {children}; + return ( + + {children} + + ); }; export const useViseme = (): VisemeContextType => { @@ -257,4 +325,4 @@ export const useViseme = (): VisemeContextType => { throw new Error('useViseme must be used within a VisemeProvider'); } return context; -}; \ No newline at end of file +};