From f4d9f377c467e63dbbe324fe7bf2a8883e161da3 Mon Sep 17 00:00:00 2001 From: nzambello Date: Tue, 15 Oct 2024 16:06:16 +0300 Subject: [PATCH] Revert "fix: avatar 3d animations, fine tuned lip sync values and removed useless function to useEffect" This reverts commit 1a51c26a0220bc9a312c7c34072188e642742111. --- src/components/Avatar/Avatar.stories.tsx | 12 ++-- src/components/Avatar/Avatar.tsx | 4 +- .../AvatarComponent/avatarComponent.tsx | 12 ++++ .../components/fullbodyAvatar.tsx | 8 ++- .../Avatar/AvatarView/AvatarView.stories.tsx | 60 ++++++++----------- src/components/Avatar/AvatarView/index.tsx | 6 ++ src/context/visemeContext.tsx | 4 +- 7 files changed, 59 insertions(+), 47 deletions(-) diff --git a/src/components/Avatar/Avatar.stories.tsx b/src/components/Avatar/Avatar.stories.tsx index dc641a40..cb18262e 100644 --- a/src/components/Avatar/Avatar.stories.tsx +++ b/src/components/Avatar/Avatar.stories.tsx @@ -3,7 +3,7 @@ import { Meta, Story } from '@storybook/react'; import { memori, tenant, integration } from '../../mocks/data'; import I18nWrapper from '../../I18nWrapper'; import Avatar, { Props } from './Avatar'; -import { VisemeProvider } from '../../context/visemeContext'; + import './Avatar.css'; const meta: Meta = { @@ -32,9 +32,8 @@ const Template: Story = args => { return ( - -
= args => { avatar3dVisible={avatar3dVisible} setAvatar3dVisible={setAvatar3dVisible} key={Date.now()} - /> -
- + /> + ); }; diff --git a/src/components/Avatar/Avatar.tsx b/src/components/Avatar/Avatar.tsx index af13439b..54e9beb8 100644 --- a/src/components/Avatar/Avatar.tsx +++ b/src/components/Avatar/Avatar.tsx @@ -57,7 +57,7 @@ const Avatar: React.FC = ({ const { t } = useTranslation(); const [isClient, setIsClient] = useState(false); - const { updateCurrentViseme } = useViseme(); + const { stopProcessing, updateCurrentViseme, resetVisemeQueue } = useViseme(); useEffect(() => { setIsClient(true); @@ -143,6 +143,8 @@ const Avatar: React.FC = ({ speaking={isPlayingAudio} loading={loading} style={getAvatarStyle()} + stopProcessing={stopProcessing} + resetVisemeQueue={resetVisemeQueue} isZoomed={isZoomed} chatEmission={chatProps?.dialogState?.emission} /> diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx index b1c69d45..402d6f35 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx @@ -14,6 +14,8 @@ interface Props { speaking: boolean; isZoomed: boolean; chatEmission: any; + stopProcessing: () => void; + resetVisemeQueue: () => void; updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null; } @@ -50,6 +52,7 @@ const baseActions: Record = { export const AvatarView: React.FC = ({ + stopProcessing, chatEmission, showControls, animation, @@ -62,6 +65,7 @@ export const AvatarView: React.FC = ({ loading, isZoomed, updateCurrentViseme, + resetVisemeQueue, }) => { const [currentBaseAction, setCurrentBaseAction] = useState({ action: animation || 'Idle1', @@ -178,6 +182,12 @@ export const AvatarView: React.FC = ({ } }, [loading]); + // useEffect(() => { + // if (speaking && currentBaseAction.action !== 'Idle1') { + // const animation = `Idle1`; + // onBaseActionChange(animation); + // } + // }, [speaking]); return ( <> @@ -207,12 +217,14 @@ export const AvatarView: React.FC = ({ void; + resetVisemeQueue: () => void; updateCurrentViseme: ( currentTime: number ) => { name: string; weight: number } | null; @@ -66,10 +68,12 @@ export default function FullbodyAvatar({ timeScale, isZoomed, eyeBlink, + stopProcessing, morphTargetSmoothing = 0.5, updateCurrentViseme, setMorphTargetDictionary, setMorphTargetInfluences, + resetVisemeQueue, emotionMorphTargets, }: FullbodyAvatarProps) { const { scene } = useGLTF(url); @@ -118,8 +122,10 @@ export default function FullbodyAvatar({ Object.values(nodes) .filter(isSkinnedMesh) .forEach(mesh => mesh.geometry.dispose()); + stopProcessing(); + resetVisemeQueue(); }; - }, [materials, nodes, url, onLoaded, scene]); + }, [materials, nodes, url, onLoaded, stopProcessing, resetVisemeQueue, scene]); // Handle base animation changes useEffect(() => { diff --git a/src/components/Avatar/AvatarView/AvatarView.stories.tsx b/src/components/Avatar/AvatarView/AvatarView.stories.tsx index 98be5e3c..d2fa2684 100644 --- a/src/components/Avatar/AvatarView/AvatarView.stories.tsx +++ b/src/components/Avatar/AvatarView/AvatarView.stories.tsx @@ -96,9 +96,8 @@ Default.args = { headMovement: false, rotateAvatar: false, speaking: false, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://assets.memori.ai/api/v2/asset/b791f77c-1a94-4272-829e-eca82fcc62b7.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/d8035229-08cf-42a7-a532-ab051df2603d.png', @@ -106,9 +105,8 @@ Default.args = { export const EyeBlink = Template.bind({}); EyeBlink.args = { - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, eyeBlink: true, headMovement: false, rotateAvatar: false, @@ -120,9 +118,8 @@ EyeBlink.args = { export const HeadMovement = Template.bind({}); HeadMovement.args = { - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, eyeBlink: false, headMovement: true, rotateAvatar: false, @@ -134,9 +131,8 @@ HeadMovement.args = { export const RotateAvatar = Template.bind({}); RotateAvatar.args = { - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, eyeBlink: false, headMovement: false, rotateAvatar: true, @@ -148,9 +144,8 @@ RotateAvatar.args = { export const Speaking = Template.bind({}); Speaking.args = { - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, eyeBlink: false, headMovement: false, rotateAvatar: false, @@ -167,9 +162,8 @@ Fullbody.args = { headMovement: true, rotateAvatar: true, speaking: false, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png', @@ -184,9 +178,8 @@ FullbodyZoomed.args = { rotateAvatar: true, speaking: false, isZoomed: true, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://assets.memori.ai/api/v2/asset/3f5ef41c-6c4c-449c-888d-cf9c89782528.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png', @@ -200,9 +193,8 @@ FullbodyAnimatedIdle.args = { headMovement: true, rotateAvatar: true, speaking: false, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png', @@ -217,9 +209,8 @@ FullbodyAnimatedLoading.args = { headMovement: true, rotateAvatar: true, speaking: false, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png', @@ -234,9 +225,8 @@ FullbodyAnimatedSpeaking.args = { headMovement: true, rotateAvatar: true, speaking: true, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://models.readyplayer.me/63b55751f17e295642bf07a2.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png', @@ -251,9 +241,8 @@ FullbodyFemale.args = { headMovement: true, rotateAvatar: true, speaking: false, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png', @@ -267,9 +256,8 @@ FullbodyAnimatedFemale.args = { headMovement: true, rotateAvatar: true, speaking: true, - stopProcessing: () => {}, - resetVisemeQueue: () => {}, - updateCurrentViseme: () => null, + clearVisemes: () => {}, + setMeshRef: () => {}, url: 'https://models.readyplayer.me/650d50c2663b19e0d2831b2b.glb', fallbackImg: 'https://assets.memori.ai/api/v2/asset/3049582f-db5f-452c-913d-e4340d4afd0a.png', diff --git a/src/components/Avatar/AvatarView/index.tsx b/src/components/Avatar/AvatarView/index.tsx index 4f8d4848..cd9bac69 100644 --- a/src/components/Avatar/AvatarView/index.tsx +++ b/src/components/Avatar/AvatarView/index.tsx @@ -23,6 +23,8 @@ export interface Props { isZoomed?: boolean; chatEmission?: any; setMeshRef?: any; + stopProcessing: () => void; + resetVisemeQueue: () => void; updateCurrentViseme: (currentTime: number) => { name: string; weight: number } | null; } @@ -87,6 +89,8 @@ export default function ContainerAvatarView({ showControls = false, isZoomed, chatEmission, + stopProcessing, + resetVisemeQueue, updateCurrentViseme, }: Props) { return ( @@ -112,6 +116,8 @@ export default function ContainerAvatarView({ halfBody={halfBody || false} chatEmission={chatEmission} updateCurrentViseme={updateCurrentViseme} + stopProcessing={stopProcessing} + resetVisemeQueue={resetVisemeQueue} /> diff --git a/src/context/visemeContext.tsx b/src/context/visemeContext.tsx index 13c11ed8..9c01ab71 100644 --- a/src/context/visemeContext.tsx +++ b/src/context/visemeContext.tsx @@ -47,8 +47,8 @@ const VISEME_MAP: { [key: number]: string } = { const DEFAULT_VISEME_DURATION = 0.04 //0; // Reduced from 0.4 for smoother transitions const VISEME_OVERLAP = 0.35; // Slightly increased from 0.04 for more overlap const SMOOTHING_FACTOR = 0.35 // New constant for weight smoothing -const TIME_OFFSET =-0.15; // Adjust this value as needed (in seconds) -const PRELOAD_TIME = 0.515; // Preload visemes 0.5 seconds in advance +const TIME_OFFSET =-0.25; // Adjust this value as needed (in seconds) +const PRELOAD_TIME = 0.525; // Preload visemes 0.5 seconds in advance export const VisemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { const visemeQueueRef = useRef([]);