Merge pull request #28 from memori-ai/feat_totem_layout

Feat totem layout
memori-ai · Nov 15, 2024 · eb21b78 · eb21b78
2 parents e2f4f5f + 4960165
commit eb21b78
Show file tree

Hide file tree

Showing 30 changed files with 1,402 additions and 314 deletions.
diff --git a/src/components/Avatar/Avatar.test.tsx b/src/components/Avatar/Avatar.test.tsx
@@ -11,6 +11,8 @@ it('renders defualt Avatar (blob) unchanged', () => {
       <Avatar
         memori={memori}
       tenant={tenant}
+      isTotem={false}
+      setEnablePositionControls={() => {}}
       instruct={false}
       avatar3dVisible={true}
       setAvatar3dVisible={() => {}}
@@ -34,6 +36,8 @@ it('renders Avatar with blob and avatar in blob unchanged', () => {
       }}
       tenant={tenant}
       instruct={false}
+      isTotem={false}
+      setEnablePositionControls={() => {}}
       avatar3dVisible={true}
       setAvatar3dVisible={() => {}}
       hasUserActivatedSpeak={false}
@@ -50,6 +54,8 @@ it('renders Avatar with custom glb model unchanged', () => {
       <Avatar
         memori={memori}
         integration={integration}
+        isTotem={false}
+        setEnablePositionControls={() => {}}
         integrationConfig={{
         ...integrationConfig,
         avatar: 'customglb',
@@ -74,6 +80,8 @@ it('renders Avatar with rpm 3d avatar unchanged', () => {
       <Avatar
         memori={memori}
       integration={integration}
+      isTotem={false}
+      setEnablePositionControls={() => {}}
       integrationConfig={{
         ...integrationConfig,
         avatar: 'readyplayerme',

diff --git a/src/components/Avatar/Avatar.tsx b/src/components/Avatar/Avatar.tsx
@@ -17,6 +17,8 @@ import Edit from '../icons/Edit';
 import cx from 'classnames';
 import ContainerAvatarView from './AvatarView';
 import { useViseme } from '../../context/visemeContext';
+import PositionControls from './AvatarView/AvatarComponent/positionControls/positionControls';
+import { getLocalConfig } from '../../helpers/configuration';
 
 export interface Props {
   memori: Memori;
@@ -34,6 +36,10 @@ export interface Props {
   animation?: string;
   isZoomed?: boolean;
   chatProps?: any;
+  enablePositionControls?: boolean;
+  setEnablePositionControls: (value: boolean) => void;
+  avatarType?: 'blob' | 'avatar3d' | null;
+  isTotem?: boolean;
 }
 
 const Avatar: React.FC<Props> = ({
@@ -52,6 +58,10 @@ const Avatar: React.FC<Props> = ({
   animation,
   isZoomed = false,
   chatProps,
+  avatarType = null,
+  enablePositionControls,
+  setEnablePositionControls,
+  isTotem = false,
 }) => {
   const { t } = useTranslation();
   const [isClient, setIsClient] = useState(false);
@@ -86,7 +96,8 @@ const Avatar: React.FC<Props> = ({
         integrationConfig?.avatar === 'readyplayerme-full' ||
         integrationConfig?.avatar === 'customglb' ||
         integrationConfig?.avatar === 'customrpm') &&
-      integrationConfig?.avatarURL
+      integrationConfig?.avatarURL &&
+      (avatarType && avatarType !== 'blob')
     ) {
       return (
         <>
@@ -114,11 +125,10 @@ const Avatar: React.FC<Props> = ({
 
   const renderAvatarContent = () => {
     if (!isClient) return null;
-
     if (
       integrationConfig?.avatar === 'readyplayerme' ||
       integrationConfig?.avatar === 'readyplayerme-full' ||
-      integrationConfig?.avatar === 'customrpm'
+      integrationConfig?.avatar === 'customrpm' 
     ) {
       return (
         <ErrorBoundary
@@ -131,6 +141,7 @@ const Avatar: React.FC<Props> = ({
           }
         >
           <ContainerAvatarView
+            enablePositionControls={enablePositionControls}
             updateCurrentViseme={updateCurrentViseme}
             url={integrationConfig.avatarURL}
             sex={memori.voiceType === 'FEMALE' ? 'FEMALE' : 'MALE'}
@@ -145,7 +156,9 @@ const Avatar: React.FC<Props> = ({
             stopProcessing={stopProcessing}
             resetVisemeQueue={resetVisemeQueue}
             isZoomed={isZoomed}
+            isTotem={isTotem}
             chatEmission={chatProps?.dialogState?.emission}
+            setEnablePositionControls={setEnablePositionControls}
           />
         </ErrorBoundary>
       );
@@ -182,10 +195,10 @@ const Avatar: React.FC<Props> = ({
   const getAvatarStyle = () => {
     if (integrationConfig?.avatar === 'readyplayerme') {
       return {
-        width: '300px',
-        height: '300px',
+        width: '100%',
+        height: '100%',
         backgroundColor: 'none',
-        borderRadius: '100%',
+        // borderRadius: '100%',
         boxShadow: 'none',
       };
     }

diff --git a/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx b/src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx
@@ -1,6 +1,6 @@
 import React, { useState, useEffect, useCallback } from 'react';
 import AnimationControlPanel from './components/controls';
-import FullbodyAvatar from './components/FullbodyAvatar/fullbodyAvatar';
+import { FullbodyAvatar } from './components/FullbodyAvatar/fullbodyAvatar'
 import HalfBodyAvatar from './components/halfbodyAvatar';
 
 interface Props {
@@ -14,11 +14,14 @@ interface Props {
   speaking: boolean;
   isZoomed: boolean;
   chatEmission: any;
+  avatarHeight?: number;
+  avatarDepth?: number;
   stopProcessing: () => void;
   resetVisemeQueue: () => void;
   updateCurrentViseme: (
     currentTime: number
   ) => { name: string; weight: number } | null;
+  setCameraZ: (value: number) => void;
 }
 
 interface BaseAction {
@@ -61,12 +64,15 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
   sex,
   eyeBlink,
   headMovement,
-  speaking,
+  // speaking,
   halfBody,
   loading,
-  isZoomed,
+  // isZoomed,
+  avatarHeight,
+  avatarDepth,
   updateCurrentViseme,
   resetVisemeQueue,
+  setCameraZ,
 }) => {
   const [currentBaseAction, setCurrentBaseAction] = useState({
     action: animation || 'Idle1',
@@ -207,13 +213,13 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
       {halfBody ? (
         <HalfBodyAvatar
           url={url}
-          headMovement={headMovement}
-          speaking={speaking}
-          eyeBlink={eyeBlink}
-          morphTargetInfluences={morphTargetInfluences}
+          onCameraZChange={setCameraZ}
           setMorphTargetInfluences={setMorphTargetInfluences}
           setMorphTargetDictionary={setMorphTargetDictionary}
           updateCurrentViseme={updateCurrentViseme}
+          avatarHeight={avatarHeight || 50}
+          avatarDepth={avatarDepth || -50}
+          headMovement={headMovement}
         />
       ) : (
         <FullbodyAvatar
@@ -224,12 +230,15 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
           currentBaseAction={currentBaseAction}
           timeScale={timeScale}
           morphTargetInfluences={morphTargetInfluences}
-          isZoomed={isZoomed}
           updateCurrentViseme={updateCurrentViseme}
           stopProcessing={stopProcessing}
           setMorphTargetDictionary={setMorphTargetDictionary}
           setMorphTargetInfluences={setMorphTargetInfluences}
           emotionMorphTargets={emotionMorphTargets}
+          halfBody={halfBody}
+          onCameraZChange={setCameraZ}
+          avatarHeight={avatarHeight || 50}
+          avatarDepth={avatarDepth || -50}
         />
       )}
     </>

diff --git a/...onents/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/AnimationController.ts b/...onents/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/AnimationController.ts
@@ -1,6 +1,6 @@
 import { AnimationState, AnimationConfig } from './types';
 import { AnimationAction, AnimationMixer, LoopOnce } from 'three';
-import { DEFAULT_CONFIG } from './constants';
+import { DEFAULT_CONFIG } from '../constants';
 
 /**
  * Controller class for managing avatar animations and transitions between states

diff --git a/...components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx b/...components/Avatar/AvatarView/AvatarComponent/components/FullbodyAvatar/fullbodyAvatar.tsx
@@ -1,56 +1,97 @@
-import React, { useEffect, useRef, useMemo, useCallback } from 'react';
+import { useEffect, useRef, useMemo } from 'react';
 import {
-  Vector3,
-  Euler,
   AnimationMixer,
   SkinnedMesh,
   Object3D,
-  MathUtils,
   AnimationAction,
-  LoopOnce,
 } from 'three';
 import { useAnimations, useGLTF } from '@react-three/drei';
 import { useFrame } from '@react-three/fiber';
 import { AnimationState, FullbodyAvatarProps } from './types';
 import { AnimationController } from './AnimationController';
-import { MorphTargetController } from './MorhTargetController';
+import { MorphTargetController } from '../MorphTargetController';
+import { AvatarPositionController } from '../PositionController';
 import {
   AVATAR_POSITION,
   AVATAR_ROTATION,
   AVATAR_POSITION_ZOOMED,
   ANIMATION_URLS,
   DEFAULT_CONFIG,
-} from './constants';
+  SCALE_LERP_FACTOR,
+} from '../constants';
 
-export default function FullbodyAvatar({
+export function FullbodyAvatar({
   url,
   sex,
   currentBaseAction,
   timeScale,
-  isZoomed,
   eyeBlink,
   updateCurrentViseme,
   setMorphTargetDictionary,
   setMorphTargetInfluences,
   emotionMorphTargets,
+  avatarHeight = 50,
+  avatarDepth = 0,
+  onCameraZChange,
 }: FullbodyAvatarProps) {
   const { scene } = useGLTF(url);
   const { animations } = useGLTF(ANIMATION_URLS[sex]);
   const { actions } = useAnimations(animations, scene);
 
   const animationControllerRef = useRef<AnimationController>();
   const morphTargetControllerRef = useRef<MorphTargetController>();
+  const positionControllerRef = useRef<AvatarPositionController>();
+
   const blinkStateRef = useRef({
     isBlinking: false,
     lastBlinkTime: 0,
     nextBlinkTime: 0,
     blinkStartTime: 0,
   });
 
+  // Initialize controllers
+  useEffect(() => {
+    if (!positionControllerRef.current) {
+      positionControllerRef.current = new AvatarPositionController(AVATAR_POSITION);
+    }
+
+    if (!actions || !scene) return;
+
+    const mixer = new AnimationMixer(scene);
+    animationControllerRef.current = new AnimationController(
+      mixer,
+      actions as Record<string, AnimationAction>,
+      { ...DEFAULT_CONFIG }
+    );
+
+    if (headMesh) {
+      morphTargetControllerRef.current = new MorphTargetController(headMesh);
+
+      if (headMesh.morphTargetDictionary && headMesh.morphTargetInfluences) {
+        setMorphTargetDictionary(headMesh.morphTargetDictionary);
+        const initialInfluences = Object.keys(headMesh.morphTargetDictionary)
+          .reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
+        setMorphTargetInfluences(initialInfluences);
+      }
+    }
+  }, [actions, scene]);
+  useEffect(() => {
+    if (positionControllerRef.current) {
+      positionControllerRef.current.updateHeight(avatarHeight, false);
+    }
+  }, [avatarHeight]);
+
+  useEffect(() => {
+    if (positionControllerRef.current && onCameraZChange) {
+      const newCameraZ = positionControllerRef.current.updateDepth(avatarDepth, false);
+      onCameraZChange(newCameraZ);
+    }
+  }, [avatarDepth, onCameraZChange]);
+
   // Find head mesh
   const headMesh = useMemo(() => {
     let foundMesh: SkinnedMesh | undefined;
-    scene.traverse((object: Object3D) => {
+    scene?.traverse((object: Object3D) => {
       if (
         object instanceof SkinnedMesh &&
         (object.name === 'GBNL__Head' || object.name === 'Wolf3D_Avatar')
@@ -61,8 +102,8 @@ export default function FullbodyAvatar({
     return foundMesh;
   }, [scene]);
 
-  // Initialize controllers
-  useEffect(() => {
+   // Initialize controllers
+   useEffect(() => {
     if (!actions || !headMesh) return;
 
     const mixer = new AnimationMixer(scene);
@@ -96,14 +137,14 @@ export default function FullbodyAvatar({
     if (!animationControllerRef.current) return;
 
     if (currentBaseAction.action.startsWith('Loading')) {
+      animationControllerRef.current.updateIsChatAlreadyStarted(true);
       animationControllerRef.current.transitionTo(
         AnimationState.LOADING,
         currentBaseAction.action
       );
     } else if (currentBaseAction.action.startsWith('Idle')) {
       animationControllerRef.current.transitionTo(AnimationState.IDLE);
     } else {
-      animationControllerRef.current.updateIsChatAlreadyStarted(true);
       animationControllerRef.current.transitionTo(
         AnimationState.EMOTION,
         currentBaseAction.action
@@ -116,12 +157,13 @@ export default function FullbodyAvatar({
     animationControllerRef.current?.setTimeScale(timeScale);
   }, [timeScale]);
 
-  // Animation update loop
-  useFrame(state => {
+
+  // Animation and scaling update loop
+  useFrame((state, delta) => {
     const currentTime = state.clock.elapsedTime * 1000;
 
     // Update animations
-    animationControllerRef.current?.update(state.clock.getDelta());
+    animationControllerRef.current?.update(delta);
 
     // Update morph targets
     if (morphTargetControllerRef.current) {
@@ -134,14 +176,23 @@ export default function FullbodyAvatar({
         blinkStateRef.current
       );
     }
+
+    // Update scale with smooth transition
+    if (scene && positionControllerRef.current) {
+      const newScale = positionControllerRef.current.updateScale(SCALE_LERP_FACTOR);
+      scene.scale.copy(newScale);
+    }
   });
 
+  // Get current position from controller
+  const position = positionControllerRef.current?.getPosition() || AVATAR_POSITION;
+
   return (
     <group
-      position={isZoomed ? AVATAR_POSITION_ZOOMED : AVATAR_POSITION}
+      position={position}
       rotation={AVATAR_ROTATION}
     >
       <primitive object={scene} />
     </group>
   );
-}
+}