Skip to content

Commit

Permalink
feat: removed unused hooks and linked context function for lips anima…
Browse files Browse the repository at this point in the history 
…tion
  • Loading branch information
@andrepat0
andrepat0 committed Oct 8, 2024
1 parent 14c76a8 commit dea3740
Show file tree
Hide file tree
Showing 8 changed files with 86 additions and 273 deletions.
48 changes: 28 additions & 20 deletions src/components/Avatar/Avatar.test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,32 @@ import React from 'react';
import { render } from '@testing-library/react';
import { memori, tenant, integration } from '../../mocks/data';
import Avatar from './Avatar';

import { VisemeProvider } from '../../context/visemeContext';
const integrationConfig = JSON.parse(integration.customData ?? '{}');

it('renders defualt Avatar (blob) unchanged', () => {
const { container } = render(
<Avatar
memori={memori}
<VisemeProvider>
<Avatar
memori={memori}
tenant={tenant}
instruct={false}
avatar3dVisible={true}
setAvatar3dVisible={() => {}}
hasUserActivatedSpeak={false}
isPlayingAudio={false}
/>
isPlayingAudio={false}
/>
</VisemeProvider>
);
expect(container).toMatchSnapshot();
});

it('renders Avatar with blob and avatar in blob unchanged', () => {
const { container } = render(
<Avatar
memori={memori}
integrationConfig={{
<VisemeProvider>
<Avatar
memori={memori}
integrationConfig={{
...integrationConfig,
avatar: 'userAvatar',
avatarURL: memori.avatarURL,
Expand All @@ -35,17 +38,19 @@ it('renders Avatar with blob and avatar in blob unchanged', () => {
setAvatar3dVisible={() => {}}
hasUserActivatedSpeak={false}
isPlayingAudio={false}
/>
/>
</VisemeProvider>
);
expect(container).toMatchSnapshot();
});

it('renders Avatar with custom glb model unchanged', () => {
const { container } = render(
<Avatar
memori={memori}
integration={integration}
integrationConfig={{
<VisemeProvider>
<Avatar
memori={memori}
integration={integration}
integrationConfig={{
...integrationConfig,
avatar: 'customglb',
avatarURL:
Expand All @@ -57,15 +62,17 @@ it('renders Avatar with custom glb model unchanged', () => {
setAvatar3dVisible={() => {}}
hasUserActivatedSpeak={false}
isPlayingAudio={false}
/>
/>
</VisemeProvider>
);
expect(container).toMatchSnapshot();
});

it('renders Avatar with rpm 3d avatar unchanged', () => {
const { container } = render(
<Avatar
memori={memori}
const { container } = render(
<VisemeProvider>
<Avatar
memori={memori}
integration={integration}
integrationConfig={{
...integrationConfig,
Expand All @@ -77,9 +84,10 @@ it('renders Avatar with rpm 3d avatar unchanged', () => {
instruct={false}
avatar3dVisible={true}
setAvatar3dVisible={() => {}}
hasUserActivatedSpeak={false}
isPlayingAudio={false}
/>
hasUserActivatedSpeak={false}
isPlayingAudio={false}
/>
</VisemeProvider>
);
expect(container).toMatchSnapshot();
});
6 changes: 6 additions & 0 deletions src/components/Avatar/Avatar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import EyeInvisible from '../icons/EyeInvisible';
import Edit from '../icons/Edit';
import cx from 'classnames';
import ContainerAvatarView from './AvatarView';
import { useViseme } from '../../context/visemeContext';

export interface Props {
memori: Memori;
Expand Down Expand Up @@ -52,9 +53,12 @@ const Avatar: React.FC<Props> = ({
isZoomed = false,
chatProps,
}) => {

const { t } = useTranslation();
const [isClient, setIsClient] = useState(false);

const { setMeshRef, clearVisemes } = useViseme();

useEffect(() => {
setIsClient(true);
}, []);
Expand Down Expand Up @@ -137,6 +141,8 @@ const Avatar: React.FC<Props> = ({
speaking={isPlayingAudio}
loading={loading}
style={getAvatarStyle()}
clearVisemes={clearVisemes}
setMeshRef={setMeshRef}
isZoomed={isZoomed}
chatEmission={chatProps?.dialogState?.emission}
/>
Expand Down
35 changes: 10 additions & 25 deletions src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import React, { useState, useEffect, useCallback } from 'react';
import AnimationControlPanel from './components/controls';
import FullbodyAvatar from './components/fullbodyAvatar';
import HalfBodyAvatar from './components/halfbodyAvatar';
import { useViseme } from '../utils/useViseme';

interface Props {
showControls: boolean;
Expand All @@ -15,6 +14,8 @@ interface Props {
speaking: boolean;
isZoomed: boolean;
chatEmission: any;
setMeshRef: any;
clearVisemes: () => void;
}

interface BaseAction {
Expand Down Expand Up @@ -46,10 +47,11 @@ const baseActions: Record<string, BaseAction> = {
};

export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
setMeshRef,
clearVisemes,
chatEmission,
showControls,
animation,
// loading,
url,
sex,
eyeBlink,
Expand All @@ -63,6 +65,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
weight: 1,
});


const [morphTargetInfluences, setMorphTargetInfluences] = useState<{
[key: string]: number;
}>({});
Expand All @@ -72,8 +75,6 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({

const [timeScale, setTimeScale] = useState(0.8);

const { createVisemeSequence, currentVisemes, clearVisemes } = useViseme();

// Set the morph target influences for the given emotions
const setEmotion = useCallback((action: string) => {
const emotionMap: Record<string, Record<string, number>> = {
Expand Down Expand Up @@ -107,6 +108,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
});
}, []);


const onMorphTargetInfluencesChange = useCallback(
(influences: { [key: string]: number }) => {
setMorphTargetInfluences(prevInfluences => ({
Expand All @@ -130,9 +132,6 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({

// Set the emotion based on the chatEmission
useEffect(() => {
if (chatEmission) {
createVisemeSequence(chatEmission);
}

//Check if chatEmission has a tag
const hasOutputTag = chatEmission?.includes(
Expand All @@ -156,23 +155,6 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
}
}, [chatEmission]);

const resetToIdle = useCallback(() => {
const randomIdle = Math.floor(Math.random() * 5) + 1;
setCurrentBaseAction({
action: `Idle${randomIdle}`,
weight: 1,
});
setMorphTargetInfluences({ mouthSmile: 0, eyesClosed: 0 });
}, []);


//Set a loading state to true if the avatar is loading
// useEffect(() => {
// if (loading) {
// resetToIdle();
// }
// }, [loading]);

return (
<>
{showControls && (
Expand All @@ -190,9 +172,11 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
{halfBody ? (
<HalfBodyAvatar
url={url}
setMeshRef={setMeshRef}
setMorphTargetInfluences={setMorphTargetInfluences}
headMovement={headMovement}
speaking={speaking}
clearVisemes={clearVisemes}
/>
) : (
<FullbodyAvatar
Expand All @@ -207,7 +191,8 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
morphTargetInfluences={morphTargetInfluences}
morphTargetDictionary={morphTargetDictionary}
isZoomed={isZoomed}
currentVisemes={currentVisemes}
setMeshRef={setMeshRef}
clearVisemes={clearVisemes}
/>
)}
</>
Expand Down
40 changes: 17 additions & 23 deletions src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { useAnimations, useGLTF } from '@react-three/drei';
import { useGraph, dispose, useFrame } from '@react-three/fiber';
import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
import { useAvatarBlink } from '../../utils/useEyeBlink';
import { useMouthAnimation } from '../../utils/useMouthAnimation';
import { useViseme } from '../../../../../context/visemeContext';

const lerp = (start: number, end: number, alpha: number): number => {
return start * (1 - alpha) + end * alpha;
Expand All @@ -33,13 +33,9 @@ interface FullbodyAvatarProps {
setMorphTargetDictionary: (dictionary: { [key: string]: number }) => void;
morphTargetInfluences: { [key: string]: number };
morphTargetDictionary: { [key: string]: number };
setMeshRef: any;
eyeBlink?: boolean;
currentVisemes: {
name: string;
duration: number;
weight: number;
startTime: number;
}[];
clearVisemes: () => void;
}

const AVATAR_POSITION = new Vector3(0, -1, 0);
Expand All @@ -64,7 +60,8 @@ export default function FullbodyAvatar({
setMorphTargetDictionary,
morphTargetInfluences,
eyeBlink,
currentVisemes,
setMeshRef,
clearVisemes,
}: FullbodyAvatarProps) {
const { scene } = useGLTF(url);
const { animations } = useGLTF(ANIMATION_URLS[sex]);
Expand All @@ -75,11 +72,8 @@ export default function FullbodyAvatar({
const avatarMeshRef = useRef<SkinnedMesh>();
const currentActionRef = useRef<AnimationAction | null>(null);
const isTransitioningRef = useRef(false);
const { handleMouthMovement } = useMouthAnimation({
currentVisemes,
avatarMeshRef: avatarMeshRef as React.RefObject<SkinnedMesh>,
});


// Blink animation
useAvatarBlink({
enabled: eyeBlink || false,
setMorphTargetInfluences,
Expand All @@ -90,6 +84,7 @@ export default function FullbodyAvatar({
},
});

// Idle animation when emotion animation is finished
const transitionToIdle = useCallback(() => {
if (!actions || isTransitioningRef.current) return;

Expand All @@ -107,7 +102,6 @@ export default function FullbodyAvatar({
};

const startIdleAnimation = () => {
// Choose a random Idle animation
const idleAnimations = Object.keys(actions).filter(key =>
key.startsWith('Idle')
);
Expand Down Expand Up @@ -137,7 +131,7 @@ export default function FullbodyAvatar({
}
}, [actions]);

// Handle base animation
// Base animation
useEffect(() => {
if (!actions || !currentBaseAction.action || isTransitioningRef.current)
return;
Expand All @@ -153,7 +147,6 @@ export default function FullbodyAvatar({
const fadeOutDuration = 0.8;
const fadeInDuration = 0.8;

// If the new action is not an Idle animation, set up the transition back to idle
if (!currentBaseAction.action.startsWith('Idle')) {
setTimeout(() => {
transitionToIdle();
Expand All @@ -169,7 +162,7 @@ export default function FullbodyAvatar({
currentActionRef.current = newAction;
}, [currentBaseAction, timeScale, actions, transitionToIdle]);

// Handle avatar blend shape animation
// Set up the mesh reference and morph target influences
useEffect(() => {
correctMaterials(materials);

Expand All @@ -179,6 +172,7 @@ export default function FullbodyAvatar({
(object.name === 'Wolf3D_Avatar020' || object.name === 'Wolf3D_Avatar')
) {
avatarMeshRef.current = object;
setMeshRef(object);

if (object.morphTargetDictionary && object.morphTargetInfluences) {
setMorphTargetDictionary(object.morphTargetDictionary);
Expand All @@ -196,6 +190,7 @@ export default function FullbodyAvatar({
return () => {
Object.values(materials).forEach(dispose);
Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
clearVisemes();
};
}, [
materials,
Expand All @@ -204,16 +199,15 @@ export default function FullbodyAvatar({
onLoaded,
setMorphTargetDictionary,
setMorphTargetInfluences,
setMeshRef,
clearVisemes,
]);

// Frame update for morph target influences and animation mixer
useFrame((state, delta) => {
// Update morph target influences
// Update morph target influences
useFrame((_, delta) => {
if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
handleMouthMovement(state.clock.elapsedTime);
updateMorphTargetInfluences();
}
// Update the animation mixer
mixer.update(delta * 0.001);

function updateMorphTargetInfluences() {
Expand All @@ -237,4 +231,4 @@ export default function FullbodyAvatar({
<primitive object={scene} />
</group>
);
}
}
Loading

0 comments on commit dea3740

Please sign in to comment.