From 05c0c4ebbfe6f6e3664a90104ada5ffaf4142728 Mon Sep 17 00:00:00 2001 From: andrepat0 Date: Thu, 7 Nov 2024 17:48:10 +0100 Subject: [PATCH] feat: half body and full body dynamic position update --- .../AvatarComponent/avatarComponent.tsx | 28 +-- .../FullbodyAvatar/fullbodyAvatar.tsx | 84 +++++-- .../components/PositionController.ts | 4 +- .../components/halfbodyAvatar.tsx | 224 ++++++++---------- 4 files changed, 178 insertions(+), 162 deletions(-) diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx index f97bb5a7..a5d89650 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx @@ -4,6 +4,7 @@ import { FullbodyAvatar } from './components/FullbodyAvatar/FullbodyAvatar'; import HalfBodyAvatar from './components/halfbodyAvatar'; import PositionControls from './positionControls/positionControls'; import { PerspectiveCamera, Vector3 } from 'three'; +import { getLocalConfig } from '../../../../helpers/configuration'; interface Props { showControls: boolean; @@ -23,8 +24,6 @@ interface Props { updateCurrentViseme: ( currentTime: number ) => { name: string; weight: number } | null; - enablePositionControls?: boolean; - setEnablePositionControls?: (value: boolean) => void; setCameraZ: (value: number) => void; } @@ -72,10 +71,10 @@ export const AvatarView: React.FC = ({ halfBody, loading, isZoomed, + avatarHeight, + avatarDepth, updateCurrentViseme, resetVisemeQueue, - enablePositionControls, - setEnablePositionControls, setCameraZ, }) => { const [currentBaseAction, setCurrentBaseAction] = useState({ @@ -95,9 +94,6 @@ export const AvatarView: React.FC = ({ const [timeScale, setTimeScale] = useState(0.8); - const [avatarHeight, setAvatarHeight] = useState(0); - const [avatarDepth, setAvatarDepth] = useState(0); - // Set the morph target influences for the given emotions const setEmotionMorphTargetInfluences = useCallback((action: string) => { if ( @@ -217,16 +213,6 @@ export const AvatarView: React.FC = ({ modifyTimeScale={modifyTimeScale} /> )} - { - enablePositionControls && ( - - ) - } {halfBody ? ( = ({ setMorphTargetInfluences={setMorphTargetInfluences} setMorphTargetDictionary={setMorphTargetDictionary} updateCurrentViseme={updateCurrentViseme} - avatarHeight={avatarHeight} - avatarDepth={avatarDepth} + avatarHeight={avatarHeight || 50} + avatarDepth={avatarDepth || -50} /> ) : ( = ({ emotionMorphTargets={emotionMorphTargets} halfBody={halfBody} onCameraZChange={setCameraZ} - avatarHeight={avatarHeight} - avatarDepth={avatarDepth} + avatarHeight={avatarHeight || 50} + avatarDepth={avatarDepth || -50} /> )} diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx index 82bbee22..67c0bd61 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx @@ -1,28 +1,26 @@ -import React, { useEffect, useRef, useMemo, useCallback } from 'react'; +import { useEffect, useRef, useMemo } from 'react'; import { - Vector3, - Euler, AnimationMixer, SkinnedMesh, Object3D, - MathUtils, AnimationAction, - LoopOnce, } from 'three'; import { useAnimations, useGLTF } from '@react-three/drei'; import { useFrame } from '@react-three/fiber'; import { AnimationState, FullbodyAvatarProps } from './types'; import { AnimationController } from './AnimationController'; -import { MorphTargetController } from './MorhTargetController'; +import { MorphTargetController } from '../MorphTargetController'; +import { AvatarPositionController } from '../PositionController'; import { AVATAR_POSITION, AVATAR_ROTATION, AVATAR_POSITION_ZOOMED, ANIMATION_URLS, DEFAULT_CONFIG, -} from './constants'; + SCALE_LERP_FACTOR, +} from '../constants'; -export default function FullbodyAvatar({ +export function FullbodyAvatar({ url, sex, currentBaseAction, @@ -33,6 +31,9 @@ export default function FullbodyAvatar({ setMorphTargetDictionary, setMorphTargetInfluences, emotionMorphTargets, + avatarHeight = 50, + avatarDepth = 0, + onCameraZChange, }: FullbodyAvatarProps) { const { scene } = useGLTF(url); const { animations } = useGLTF(ANIMATION_URLS[sex]); @@ -40,6 +41,8 @@ export default function FullbodyAvatar({ const animationControllerRef = useRef(); const morphTargetControllerRef = useRef(); + const positionControllerRef = useRef(); + const blinkStateRef = useRef({ isBlinking: false, lastBlinkTime: 0, @@ -47,10 +50,49 @@ export default function FullbodyAvatar({ blinkStartTime: 0, }); + // Initialize controllers + useEffect(() => { + if (!positionControllerRef.current) { + positionControllerRef.current = new AvatarPositionController(AVATAR_POSITION); + } + + if (!actions || !scene) return; + + const mixer = new AnimationMixer(scene); + animationControllerRef.current = new AnimationController( + mixer, + actions as Record, + { ...DEFAULT_CONFIG } + ); + + if (headMesh) { + morphTargetControllerRef.current = new MorphTargetController(headMesh); + + if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) { + setMorphTargetDictionary(headMesh.morphTargetDictionary); + const initialInfluences = Object.keys(headMesh.morphTargetDictionary) + .reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); + setMorphTargetInfluences(initialInfluences); + } + } + }, [actions, scene]); + useEffect(() => { + if (positionControllerRef.current) { + positionControllerRef.current.updateHeight(avatarHeight, false); + } + }, [avatarHeight]); + + useEffect(() => { + if (positionControllerRef.current && onCameraZChange) { + const newCameraZ = positionControllerRef.current.updateDepth(avatarDepth, false); + onCameraZChange(newCameraZ); + } + }, [avatarDepth, onCameraZChange]); + // Find head mesh const headMesh = useMemo(() => { let foundMesh: SkinnedMesh | undefined; - scene.traverse((object: Object3D) => { + scene?.traverse((object: Object3D) => { if ( object instanceof SkinnedMesh && (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') @@ -61,8 +103,8 @@ export default function FullbodyAvatar({ return foundMesh; }, [scene]); - // Initialize controllers - useEffect(() => { + // Initialize controllers + useEffect(() => { if (!actions || !headMesh) return; const mixer = new AnimationMixer(scene); @@ -116,12 +158,13 @@ export default function FullbodyAvatar({ animationControllerRef.current?.setTimeScale(timeScale); }, [timeScale]); - // Animation update loop - useFrame(state => { + + // Animation and scaling update loop + useFrame((state, delta) => { const currentTime = state.clock.elapsedTime * 1000; // Update animations - animationControllerRef.current?.update(state.clock.getDelta()); + animationControllerRef.current?.update(delta); // Update morph targets if (morphTargetControllerRef.current) { @@ -134,14 +177,23 @@ export default function FullbodyAvatar({ blinkStateRef.current ); } + + // Update scale with smooth transition + if (scene && positionControllerRef.current) { + const newScale = positionControllerRef.current.updateScale(SCALE_LERP_FACTOR); + scene.scale.copy(newScale); + } }); + // Get current position from controller + const position = positionControllerRef.current?.getPosition() || AVATAR_POSITION; + return ( ); -} +} \ No newline at end of file diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/PositionController.ts b/src/components/Avatar/AvatarView/AvatarComponent/components/PositionController.ts index b901c1b6..0d83cd68 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/PositionController.ts +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/PositionController.ts @@ -28,9 +28,9 @@ export class AvatarPositionController { private mapHeightToScale(sliderValue: number, isHalfBody: boolean): number { // Convert slider value to scale factor if (isHalfBody) { - return MathUtils.lerp(1.6, 3, sliderValue / 100); + return MathUtils.lerp(1.4, 1.8, sliderValue / 100); } else { - return MathUtils.lerp(0, 1.5, sliderValue / 100); + return MathUtils.lerp(0.5, 1.5, sliderValue / 100); } } diff --git a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx index 0a013e3b..48b37c99 100644 --- a/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx +++ b/src/components/Avatar/AvatarView/AvatarComponent/components/halfbodyAvatar.tsx @@ -1,75 +1,94 @@ import React, { useEffect, useMemo, useRef } from 'react'; -import { Object3D, SkinnedMesh, Vector3 } from 'three'; +import { MathUtils, Object3D, SkinnedMesh, Vector3 } from 'three'; import { useGLTF } from '@react-three/drei'; +import { useGraph, useFrame, useThree } from '@react-three/fiber'; import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils'; -import { useGraph, dispose, useFrame } from '@react-three/fiber'; -import { useAvatarBlink } from '../../utils/useEyeBlink'; -import useHeadMovement from '../../utils/useHeadMovement'; -import { hideHands } from '../../utils/utils'; -import { AnimationMixer, MathUtils } from 'three'; +import { MorphTargetController } from './MorphTargetController'; +import { AvatarPositionController } from './PositionController'; +import { + AVATAR_POSITION, + SCALE_LERP_FACTOR, + AVATAR_POSITION_ZOOMED, +} from './constants'; interface HalfBodyAvatarProps { url: string; setMorphTargetInfluences: (morphTargetInfluences: any) => void; - headMovement?: boolean; - speaking?: boolean; - onLoaded?: () => void; setMorphTargetDictionary: (morphTargetDictionary: any) => void; - eyeBlink?: boolean; - morphTargetInfluences: any; updateCurrentViseme: (currentTime: number) => any; + eyeBlink?: boolean; + isZoomed?: boolean; + heightValue?: number; // 0-100 slider value + avatarHeight: number; + avatarDepth: number; morphTargetSmoothing?: number; + onLoaded?: () => void; + onCameraZChange: (value: number) => void; } -const AVATAR_POSITION = new Vector3(0, -0.6, 0); -// Blink configuration -const BLINK_CONFIG = { - minInterval: 1000, - maxInterval: 5000, - blinkDuration: 150, -}; - export default function HalfBodyAvatar({ url, setMorphTargetInfluences, setMorphTargetDictionary, - eyeBlink, - onLoaded, - morphTargetSmoothing = 0.5, updateCurrentViseme, + eyeBlink = false, + isZoomed = false, + avatarHeight = 50, + avatarDepth = 0, + morphTargetSmoothing = 0.5, + onLoaded, + onCameraZChange, }: HalfBodyAvatarProps) { const { scene } = useGLTF(url); const { nodes, materials } = useGraph(scene); - const mixer = useRef(new AnimationMixer(scene)); - const avatarMeshRef = useRef(null); - - // Blink state - const lastBlinkTime = useRef(0); - const nextBlinkTime = useRef(0); - const isBlinking = useRef(false); - const blinkStartTime = useRef(0); + const { camera } = useThree(); + + const morphTargetControllerRef = useRef(); + const positionControllerRef = useRef(); + const targetCameraZRef = useRef(camera.position.z); + + const blinkStateRef = useRef({ + isBlinking: false, + lastBlinkTime: 0, + nextBlinkTime: 0, + blinkStartTime: 0, + }); - const headMeshRef = useRef(); + // Find head mesh + const headMesh = useMemo(() => { + let foundMesh: SkinnedMesh | undefined; + scene?.traverse((object: Object3D) => { + if ( + object instanceof SkinnedMesh && + (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') + ) { + foundMesh = object; + } + }); + return foundMesh; + }, [scene]); + // Initialize controllers useEffect(() => { - correctMaterials(materials); + if (!positionControllerRef.current) { + positionControllerRef.current = new AvatarPositionController( + AVATAR_POSITION, + AVATAR_POSITION_ZOOMED, + ); + } - scene.traverse((object: Object3D) => { - if (object instanceof SkinnedMesh) { - if (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar') { - headMeshRef.current = object; - if (object.morphTargetDictionary && object.morphTargetInfluences) { - setMorphTargetDictionary(object.morphTargetDictionary); - - const initialInfluences = Object.keys( - object.morphTargetDictionary - ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); - setMorphTargetInfluences(initialInfluences); - } - } + if (headMesh) { + morphTargetControllerRef.current = new MorphTargetController(headMesh); + + if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) { + setMorphTargetDictionary(headMesh.morphTargetDictionary); + const initialInfluences = Object.keys(headMesh.morphTargetDictionary) + .reduce((acc, key) => ({ ...acc, [key]: 0 }), {}); + setMorphTargetInfluences(initialInfluences); } - }); + } + correctMaterials(materials); onLoaded?.(); return () => { @@ -78,91 +97,50 @@ export default function HalfBodyAvatar({ .filter(isSkinnedMesh) .forEach(mesh => mesh.geometry.dispose()); }; - }, [materials, nodes, url, onLoaded, scene]); - useFrame(state => { - - if ( - headMeshRef.current && - headMeshRef.current.morphTargetDictionary && - headMeshRef.current.morphTargetInfluences - ) { - const currentTime = state.clock.getElapsedTime() * 1000; // Convert to milliseconds - - // Handle blinking - let blinkValue = 0; - if (eyeBlink) { - if (currentTime >= nextBlinkTime.current && !isBlinking.current) { - isBlinking.current = true; - blinkStartTime.current = currentTime; - lastBlinkTime.current = currentTime; - nextBlinkTime.current = - currentTime + - Math.random() * - (BLINK_CONFIG.maxInterval - BLINK_CONFIG.minInterval) + - BLINK_CONFIG.minInterval; - } - - if (isBlinking.current) { - const blinkProgress = - (currentTime - blinkStartTime.current) / BLINK_CONFIG.blinkDuration; - if (blinkProgress <= 0.5) { - // Eyes closing - blinkValue = blinkProgress * 2; - } else if (blinkProgress <= 1) { - // Eyes opening - blinkValue = 2 - blinkProgress * 2; - } else { - // Blink finished - isBlinking.current = false; - blinkValue = 0; - } - } - } + }, [materials, nodes, url, onLoaded, scene, headMesh]); - const currentViseme = updateCurrentViseme(currentTime / 1000); + useEffect(() => { + if (positionControllerRef.current) { + positionControllerRef.current.updateHeight(avatarHeight, true); + } + }, [avatarHeight]); - // Update morph targets - Object.entries(headMeshRef.current.morphTargetDictionary).forEach( - ([key, index]) => { - if (typeof index === 'number') { - let targetValue = 0; - - // Handle visemes (additive layer) - if (currentViseme && key === currentViseme.name) { - targetValue += currentViseme.weight * 1.3; // Amplify the effect - } - - // Handle blinking (additive layer, only for 'eyesClosed') - if (key === 'eyesClosed' && eyeBlink) { - targetValue += blinkValue; - } - - // Clamp the final value between 0 and 1 - targetValue = MathUtils.clamp(targetValue, 0, 1); - - // Apply smoothing - if ( - headMeshRef.current && - headMeshRef.current.morphTargetInfluences - ) { - headMeshRef.current.morphTargetInfluences[index] = MathUtils.lerp( - headMeshRef.current.morphTargetInfluences[index], - targetValue, - morphTargetSmoothing - ); - } - } - } + useEffect(() => { + if (positionControllerRef.current && onCameraZChange) { + const newCameraZ = positionControllerRef.current.updateDepth(avatarDepth, true); + onCameraZChange(newCameraZ); + } + }, [avatarDepth, onCameraZChange]); + + // Animation and morphing update loop + useFrame((state) => { + const currentTime = state.clock.elapsedTime * 1000; + + // Update morph targets + if (morphTargetControllerRef.current) { + const currentViseme = updateCurrentViseme(currentTime / 1000); + morphTargetControllerRef.current.updateMorphTargets( + currentTime, + {}, + currentViseme, + eyeBlink, + blinkStateRef.current, ); + } - // Update the animation mixer - mixer.current.update(0.01); // Fixed delta time for consistent animation speed + // Update scale with smooth transition + if (scene && positionControllerRef.current) { + const newScale = positionControllerRef.current.updateScale(SCALE_LERP_FACTOR); + scene.scale.copy(newScale); } }); + // Get current position from controller + const position = positionControllerRef.current?.getPosition() || AVATAR_POSITION; + return ( - + ); -} +} \ No newline at end of file