Skip to content

Commit

Permalink
fix: reinserted inside half body the blink and speak animation
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrepat0
andrepat0 committed Oct 8, 2024
1 parent 88ec8e9 commit 80e4889
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 34 deletions.
3 changes: 3 additions & 0 deletions src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,10 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
setMorphTargetInfluences={setMorphTargetInfluences}
headMovement={headMovement}
speaking={speaking}
eyeBlink={eyeBlink}
morphTargetInfluences={morphTargetInfluences}
clearVisemes={clearVisemes}
setMorphTargetDictionary={setMorphTargetDictionary}
/>
) : (
<FullbodyAvatar
Expand Down
39 changes: 35 additions & 4 deletions src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import React, { useEffect, useMemo } from 'react';
import React, { useEffect, useMemo, useRef } from 'react';
import { Object3D, SkinnedMesh, Vector3 } from 'three';
import { useGLTF } from '@react-three/drei';
import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
import { useGraph, dispose } from '@react-three/fiber';
import { useGraph, dispose, useFrame } from '@react-three/fiber';
import { useAvatarBlink } from '../../utils/useEyeBlink';
import useHeadMovement from '../../utils/useHeadMovement';
import { hideHands } from '../../utils/utils';
Expand All @@ -16,23 +16,32 @@ interface HalfBodyAvatarProps {
onLoaded?: () => void;
setMeshRef: (mesh: Object3D) => void;
clearVisemes: () => void;
setMorphTargetDictionary: (morphTargetDictionary: any) => void;
eyeBlink?: boolean;
morphTargetInfluences: any;
}

const AVATAR_POSITION = new Vector3(0, -0.6, 0);

const lerp = (start: number, end: number, alpha: number): number => {
return start * (1 - alpha) + end * alpha;
};
export default function HalfBodyAvatar({
url,
setMorphTargetInfluences,
setMorphTargetDictionary,
headMovement,
eyeBlink,
setMeshRef,
onLoaded,
clearVisemes,
morphTargetInfluences,
}: HalfBodyAvatarProps) {
const { scene } = useGLTF(url);
const { nodes, materials } = useGraph(scene);
const avatarMeshRef = useRef<SkinnedMesh | null>(null);

useAvatarBlink({
enabled: true,
enabled: eyeBlink || false,
setMorphTargetInfluences,
config: {
minInterval: 1500,
Expand All @@ -50,7 +59,9 @@ export default function HalfBodyAvatar({
const firstSkinnedMesh = Object.values(nodes).find(isSkinnedMesh) as SkinnedMesh;
if (firstSkinnedMesh) {
setMeshRef(firstSkinnedMesh);
avatarMeshRef.current = firstSkinnedMesh;
if (firstSkinnedMesh.morphTargetDictionary && firstSkinnedMesh.morphTargetInfluences) {
setMorphTargetDictionary(firstSkinnedMesh.morphTargetDictionary);
const initialInfluences = Object.keys(
firstSkinnedMesh.morphTargetDictionary
).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
Expand Down Expand Up @@ -78,6 +89,26 @@ export default function HalfBodyAvatar({
[nodes]
);

// Update morph target influences
useFrame((_, delta) => {
if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
updateMorphTargetInfluences();
}

function updateMorphTargetInfluences() {
Object.entries(morphTargetInfluences).forEach(([key, value]) => {
const index = avatarMeshRef.current!.morphTargetDictionary![key];
if (typeof index === 'number' &&
avatarMeshRef.current!.morphTargetInfluences) {
const currentValue = avatarMeshRef.current!.morphTargetInfluences[index];
const smoothValue = lerp(currentValue, value as number, 0.1);
avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
}
});
}
});


return (
<group position={AVATAR_POSITION}>
{nodes.Hips && <primitive key="armature" object={nodes.Hips} />}
Expand Down
61 changes: 31 additions & 30 deletions src/context/visemeContext.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,36 +20,37 @@ interface VisemeContextType {

const VisemeContext = createContext<VisemeContextType | undefined>(undefined);

const VISEME_SMOOTHING = 0.35;
const VISEME_SMOOTHING = 0.37;
const DEFAULT_VISEME_DURATION = 0.1;
const MINIMUM_ELAPSED_TIME = 0.01;
const VISEME_SPEED_FACTOR = 1.0;
const AUDIO_PLAYBACK_RATE = 1.0;
const VISEME_BASE_SPEED = 1.0;

const VISEME_MAP: { [key: number]: string } = {
0: 'viseme_sil',
1: 'viseme_aa',
2: 'viseme_aa',
3: 'viseme_O',
4: 'viseme_E',
5: 'viseme_I',
6: 'viseme_I',
7: 'viseme_U',
8: 'viseme_O',
9: 'viseme_aa',
10: 'viseme_O',
11: 'viseme_aa',
12: 'viseme_CH',
13: 'viseme_RR',
14: 'viseme_SS',
15: 'viseme_SS',
16: 'viseme_CH',
17: 'viseme_TH',
18: 'viseme_FF',
19: 'viseme_DD',
20: 'viseme_kk',
21: 'viseme_PP',
0: 'viseme_sil', // silence
1: 'viseme_PP', // p, b, m
2: 'viseme_FF', // f, v
3: 'viseme_TH', // th, dh
4: 'viseme_DD', // t, d, n, l
5: 'viseme_kk', // k, g, ng
6: 'viseme_CH', // tS, dZ, S, Z
7: 'viseme_SS', // s, z
8: 'viseme_nn', // Not explicitly defined in Azure mapping, keeping for compatibility
9: 'viseme_RR', // r
10: 'viseme_aa', // A:
11: 'viseme_E', // e
12: 'viseme_I', // I
13: 'viseme_O', // O
14: 'viseme_U', // u
// Mapping the rest based on closest matches or keeping them as in the original mapping
15: 'viseme_kk', // g, k (same as 5)
16: 'viseme_CH', // ch, j, sh, zh (same as 6)
17: 'viseme_SS', // s, z (same as 7)
18: 'viseme_TH', // th, dh (same as 3)
19: 'viseme_RR', // r (same as 9)
20: 'viseme_kk', // w (closest match, could be debated)
21: 'viseme_PP', // y (closest match, could be debated)
};

export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
Expand All @@ -74,15 +75,15 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
if (mesh && mesh.morphTargetDictionary && mesh.morphTargetInfluences) {
meshRef.current = mesh;
setIsMeshSet(true);
console.log('Mesh set successfully:', mesh);
// console.log('Mesh set successfully:', mesh);
} else {
console.error('Invalid mesh provided:', mesh);
}
}, [meshRef]);

const addVisemeToQueue = useCallback((viseme: AzureViseme) => {
visemeQueueRef.current.push(viseme);
console.log('Viseme added to queue:', viseme);
// console.log('Viseme added to queue:', viseme);
}, []);

const getCurrentViseme = useCallback((elapsedTime: number) => {
Expand Down Expand Up @@ -129,7 +130,7 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
const visemeIndex = meshRef.current.morphTargetDictionary?.[viseme.name];
if (typeof visemeIndex === 'number' && meshRef.current.morphTargetInfluences) {
meshRef.current.morphTargetInfluences[visemeIndex] = currentVisemeWeightRef.current[viseme.name];
console.log(`Applied viseme: ${viseme.name}, weight: ${currentVisemeWeightRef.current[viseme.name]}`);
// console.log(`Applied viseme: ${viseme.name}, weight: ${currentVisemeWeightRef.current[viseme.name]}`);
} else {
console.error(`Viseme not found in morph target dictionary: ${viseme.name}`);
}
Expand Down Expand Up @@ -166,7 +167,7 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
visemeQueueRef.current = [];

if (azureVisemes.length === 0) {
console.log('No visemes to process');
// console.log('No visemes to process');
return [];
}

Expand All @@ -183,7 +184,7 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
weight: 1,
startTime: currentViseme.audioOffset / 10000000,
};
console.log('Processed viseme:', processedViseme);
//console.log('Processed viseme:', processedViseme);
return processedViseme;
}
);
Expand All @@ -194,7 +195,7 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
if (!isAnimatingRef.current) {
isAnimatingRef.current = true;
startTimeRef.current = performance.now();
console.log('Starting animation');
// console.log('Starting animation');
animationFrameRef.current = requestAnimationFrame(animate);
} else {
// If already animating, adjust the start time for the new visemes
Expand Down Expand Up @@ -228,7 +229,7 @@ export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ childr
currentVisemeWeightRef.current = {};
startTimeRef.current = null;
isAnimatingRef.current = false;
console.log('Visemes cleared');
// console.log('Visemes cleared');
}, []);

useEffect(() => {
Expand Down

0 comments on commit 80e4889

Please sign in to comment.