Skip to content

Commit

Permalink
Revert "fix: avatar 3d animations, fine tuned lip sync values and rem…
Browse files Browse the repository at this point in the history 
…oved useless function to useEffect"

This reverts commit 1a51c26.
  • Loading branch information
@nzambello
nzambello committed Oct 15, 2024
1 parent f80bb9b commit f4d9f37
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 47 deletions.
12 changes: 5 additions & 7 deletions src/components/Avatar/Avatar.stories.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Meta, Story } from '@storybook/react';
import { memori, tenant, integration } from '../../mocks/data';
import I18nWrapper from '../../I18nWrapper';
import Avatar, { Props } from './Avatar';
import { VisemeProvider } from '../../context/visemeContext';

import './Avatar.css';

const meta: Meta = {
Expand Down Expand Up @@ -32,9 +32,8 @@ const Template: Story<Props> = args => {

return (
<I18nWrapper>
<VisemeProvider>
<div
style={
<div
style={
args.integrationConfig?.avatar === 'customglb' ||
args.integrationConfig?.avatar === 'readyplayerme' ||
args.integrationConfig?.avatar === 'readyplayerme-full'
Expand All @@ -57,9 +56,8 @@ const Template: Story<Props> = args => {
avatar3dVisible={avatar3dVisible}
setAvatar3dVisible={setAvatar3dVisible}
key={Date.now()}
/>
</div>
</VisemeProvider>
/>
</div>
</I18nWrapper>
);
};
Expand Down
4 changes: 3 additions & 1 deletion src/components/Avatar/Avatar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ const Avatar: React.FC<Props> = ({
const { t } = useTranslation();
const [isClient, setIsClient] = useState(false);

const { updateCurrentViseme } = useViseme();
const { stopProcessing, updateCurrentViseme, resetVisemeQueue } = useViseme();

useEffect(() => {
setIsClient(true);
Expand Down Expand Up @@ -143,6 +143,8 @@ const Avatar: React.FC<Props> = ({
speaking={isPlayingAudio}
loading={loading}
style={getAvatarStyle()}
stopProcessing={stopProcessing}
resetVisemeQueue={resetVisemeQueue}
isZoomed={isZoomed}
chatEmission={chatProps?.dialogState?.emission}
/>
Expand Down
12 changes: 12 additions & 0 deletions src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ interface Props {
speaking: boolean;
isZoomed: boolean;
chatEmission: any;
stopProcessing: () => void;
resetVisemeQueue: () => void;
updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
}

Expand Down Expand Up @@ -50,6 +52,7 @@ const baseActions: Record<string, BaseAction> = {


export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
stopProcessing,
chatEmission,
showControls,
animation,
Expand All @@ -62,6 +65,7 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
loading,
isZoomed,
updateCurrentViseme,
resetVisemeQueue,
}) => {
const [currentBaseAction, setCurrentBaseAction] = useState({
action: animation || 'Idle1',
Expand Down Expand Up @@ -178,6 +182,12 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
}
}, [loading]);

// useEffect(() => {
// if (speaking && currentBaseAction.action !== 'Idle1') {
// const animation = `Idle1`;
// onBaseActionChange(animation);
// }
// }, [speaking]);

return (
<>
Expand Down Expand Up @@ -207,12 +217,14 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
<FullbodyAvatar
url={url}
sex={sex}
resetVisemeQueue={resetVisemeQueue}
eyeBlink={eyeBlink}
currentBaseAction={currentBaseAction}
timeScale={timeScale}
morphTargetInfluences={morphTargetInfluences}
isZoomed={isZoomed}
updateCurrentViseme={updateCurrentViseme}
stopProcessing={stopProcessing}
setMorphTargetDictionary={setMorphTargetDictionary}
setMorphTargetInfluences={setMorphTargetInfluences}
emotionMorphTargets={emotionMorphTargets}
Expand Down
8 changes: 7 additions & 1 deletion src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ interface FullbodyAvatarProps {
timeScale: number;
isZoomed?: boolean;
eyeBlink?: boolean;
stopProcessing: () => void;
resetVisemeQueue: () => void;
updateCurrentViseme: (
currentTime: number
) => { name: string; weight: number } | null;
Expand Down Expand Up @@ -66,10 +68,12 @@ export default function FullbodyAvatar({
timeScale,
isZoomed,
eyeBlink,
stopProcessing,
morphTargetSmoothing = 0.5,
updateCurrentViseme,
setMorphTargetDictionary,
setMorphTargetInfluences,
resetVisemeQueue,
emotionMorphTargets,
}: FullbodyAvatarProps) {
const { scene } = useGLTF(url);
Expand Down Expand Up @@ -118,8 +122,10 @@ export default function FullbodyAvatar({
Object.values(nodes)
.filter(isSkinnedMesh)
.forEach(mesh => mesh.geometry.dispose());
stopProcessing();
resetVisemeQueue();
};
}, [materials, nodes, url, onLoaded, scene]);
}, [materials, nodes, url, onLoaded, stopProcessing, resetVisemeQueue, scene]);

// Handle base animation changes
useEffect(() => {
Expand Down
60 changes: 24 additions & 36 deletions src/components/Avatar/AvatarView/AvatarView.stories.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,19 +96,17 @@ Default.args = {
headMovement: false,
rotateAvatar: false,
speaking: false,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/d8035229-08cf-42a7-a532-ab051df2603d.png',
};

export const EyeBlink = Template.bind({});
EyeBlink.args = {
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
eyeBlink: true,
headMovement: false,
rotateAvatar: false,
Expand All @@ -120,9 +118,8 @@ EyeBlink.args = {

export const HeadMovement = Template.bind({});
HeadMovement.args = {
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
eyeBlink: false,
headMovement: true,
rotateAvatar: false,
Expand All @@ -134,9 +131,8 @@ HeadMovement.args = {

export const RotateAvatar = Template.bind({});
RotateAvatar.args = {
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
eyeBlink: false,
headMovement: false,
rotateAvatar: true,
Expand All @@ -148,9 +144,8 @@ RotateAvatar.args = {

export const Speaking = Template.bind({});
Speaking.args = {
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
eyeBlink: false,
headMovement: false,
rotateAvatar: false,
Expand All @@ -167,9 +162,8 @@ Fullbody.args = {
headMovement: true,
rotateAvatar: true,
speaking: false,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
Expand All @@ -184,9 +178,8 @@ FullbodyZoomed.args = {
rotateAvatar: true,
speaking: false,
isZoomed: true,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://assets.memori.ai/api/v2/asset/3f5ef41c-6c4c-449c-888d-cf9c89782528.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
Expand All @@ -200,9 +193,8 @@ FullbodyAnimatedIdle.args = {
headMovement: true,
rotateAvatar: true,
speaking: false,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
Expand All @@ -217,9 +209,8 @@ FullbodyAnimatedLoading.args = {
headMovement: true,
rotateAvatar: true,
speaking: false,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
Expand All @@ -234,9 +225,8 @@ FullbodyAnimatedSpeaking.args = {
headMovement: true,
rotateAvatar: true,
speaking: true,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
Expand All @@ -251,9 +241,8 @@ FullbodyFemale.args = {
headMovement: true,
rotateAvatar: true,
speaking: false,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
Expand All @@ -267,9 +256,8 @@ FullbodyAnimatedFemale.args = {
headMovement: true,
rotateAvatar: true,
speaking: true,
stopProcessing: () => {},
resetVisemeQueue: () => {},
updateCurrentViseme: () => null,
clearVisemes: () => {},
setMeshRef: () => {},
url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb',
fallbackImg:
'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png',
Expand Down
6 changes: 6 additions & 0 deletions src/components/Avatar/AvatarView/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ export interface Props {
isZoomed?: boolean;
chatEmission?: any;
setMeshRef?: any;
stopProcessing: () => void;
resetVisemeQueue: () => void;
updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null;
}

Expand Down Expand Up @@ -87,6 +89,8 @@ export default function ContainerAvatarView({
showControls = false,
isZoomed,
chatEmission,
stopProcessing,
resetVisemeQueue,
updateCurrentViseme,
}: Props) {
return (
Expand All @@ -112,6 +116,8 @@ export default function ContainerAvatarView({
halfBody={halfBody || false}
chatEmission={chatEmission}
updateCurrentViseme={updateCurrentViseme}
stopProcessing={stopProcessing}
resetVisemeQueue={resetVisemeQueue}
/>
</Suspense>
</Canvas>
Expand Down
4 changes: 2 additions & 2 deletions src/context/visemeContext.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ const VISEME_MAP: { [key: number]: string } = {
const DEFAULT_VISEME_DURATION = 0.04 //0; // Reduced from 0.4 for smoother transitions
const VISEME_OVERLAP = 0.35; // Slightly increased from 0.04 for more overlap
const SMOOTHING_FACTOR = 0.35 // New constant for weight smoothing
const TIME_OFFSET =-0.15; // Adjust this value as needed (in seconds)
const PRELOAD_TIME = 0.515; // Preload visemes 0.5 seconds in advance
const TIME_OFFSET =-0.25; // Adjust this value as needed (in seconds)
const PRELOAD_TIME = 0.525; // Preload visemes 0.5 seconds in advance

export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
const visemeQueueRef = useRef<Viseme[]>([]);
Expand Down

0 comments on commit f4d9f37

Please sign in to comment.