feat: added emotion to XML audio tag

memori-ai · Oct 8, 2024 · c188673 · c188673
1 parent 80e4889
commit c188673
Show file tree

Hide file tree

Showing 6 changed files with 170 additions and 86 deletions.
diff --git a/src/components/Avatar/Avatar.tsx b/src/components/Avatar/Avatar.tsx
@@ -57,7 +57,7 @@ const Avatar: React.FC<Props> = ({
   const { t } = useTranslation();
   const [isClient, setIsClient] = useState(false);
 
-  const { setMeshRef, clearVisemes } = useViseme();
+  const { setMeshRef, clearVisemes, setEmotion } = useViseme();
 
   useEffect(() => {
     setIsClient(true);
@@ -145,6 +145,7 @@ const Avatar: React.FC<Props> = ({
             setMeshRef={setMeshRef}
             isZoomed={isZoomed}
             chatEmission={chatProps?.dialogState?.emission}
+            setEmotion={setEmotion}
           />
         </ErrorBoundary>
       );

diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx
@@ -16,6 +16,7 @@ interface Props {
   chatEmission: any;
   setMeshRef: any;
   clearVisemes: () => void;
+  setEmotion: (emotion: string) => void;
 }
 
 interface BaseAction {
@@ -59,6 +60,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   speaking,
   halfBody,
   isZoomed,
+  setEmotion,
 }) => {
   const [currentBaseAction, setCurrentBaseAction] = useState({
     action: animation || 'Idle1',
@@ -76,7 +78,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   const [timeScale, setTimeScale] = useState(0.8);
 
   // Set the morph target influences for the given emotions
-  const setEmotion = useCallback((action: string) => {
+  const setEmotionMorphTargetInfluences = useCallback((action: string) => {
     const emotionMap: Record<string, Record<string, number>> = {
       Gioia: { Gioria: 1 },
       Rabbia: { Rabbia: 1 },
@@ -85,6 +87,10 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
       Timore: { Timore: 1 },
     };
 
+    //remove the last character from the action
+    const newEmotion = action.slice(0, -1);
+    setEmotion(newEmotion);
+
     const defaultEmotions = Object.keys(emotionMap).reduce((acc, key) => {
       acc[key] = 0;
       return acc;
@@ -101,7 +107,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   }, []);
 
   const onBaseActionChange = useCallback((action: string) => {
-    setEmotion(action);
+    setEmotionMorphTargetInfluences(action);
     setCurrentBaseAction({
       action,
       weight: 1,

diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx
@@ -90,7 +90,7 @@ export default function HalfBodyAvatar({
   );
 
     // Update morph target influences
-    useFrame((_, delta) => {
+    useFrame((_) => {
       if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
         updateMorphTargetInfluences();
       }

diff --git a/src/components/Avatar/AvatarView/index.tsx b/src/components/Avatar/AvatarView/index.tsx
@@ -24,6 +24,7 @@ export interface Props {
   chatEmission?: any;
   setMeshRef?: any;
   clearVisemes: () => void;
+  setEmotion: (emotion: string) => void;
 }
 
 const defaultStyles = {
@@ -89,6 +90,7 @@ export default function ContainerAvatarView({
   chatEmission,
   setMeshRef,
   clearVisemes,
+  setEmotion,
 }: Props) {
   return (
     <Canvas
@@ -114,6 +116,7 @@ export default function ContainerAvatarView({
           chatEmission={chatEmission}
           setMeshRef={setMeshRef}
           clearVisemes={clearVisemes}
+          setEmotion={setEmotion}
         />
       </Suspense>
     </Canvas>

diff --git a/src/components/MemoriWidget/MemoriWidget.tsx b/src/components/MemoriWidget/MemoriWidget.tsx
@@ -538,7 +538,13 @@ const MemoriWidget = ({
   );
   const [hideEmissions, setHideEmissions] = useState(false);
 
-  const { addVisemeToQueue, processVisemeQueue, clearVisemes } = useViseme();
+  const {
+    addVisemeToQueue,
+    processVisemeQueue,
+    clearVisemes,
+    emotion,
+    getAzureStyleForEmotion,
+  } = useViseme();
 
   useEffect(() => {
     setIsPlayingAudio(!!speechSynthesizer);
@@ -1962,13 +1968,13 @@ const MemoriWidget = ({
       });
     };
 
-    speechSynthesizer.speakSsmlAsync(
-      `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(
+    console.log(getAzureStyleForEmotion(emotion))
+    speechSynthesizer.speakSsmlAsync(`<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" xml:lang="${getCultureCodeByLanguage(
         userLang
-      )}"><voice name="${getTTSVoice(userLang)}"><s>${replaceTextWithPhonemes(
+      )}"><voice name="${getTTSVoice(userLang)}"> <mstts:express-as style="${getAzureStyleForEmotion(emotion)}"><s>${replaceTextWithPhonemes(
         escapeHTML(stripMarkdown(stripEmojis(stripOutputTags(text)))),
         userLang.toLowerCase()
-      )}</s></voice></speak>`,
+      )}</s></mstts:express-as></voice></speak>`,
       result => {
         if (result) {
           setIsPlayingAudio(true);