Fix xrui pointer input & improved follow camera (#10471)

* Cleanup pointer event handling

* Update ClientInputSystem.tsx

* fix gizmos

* Update CameraOrbitSystem.tsx

* Update ClientInputSystem.tsx

* Update ClientInputSystem.tsx

* wip

* Update getThumbstickOrThumbpadAxes.ts

* wip

* wip

* Update ClientInputSystem.tsx

* wip

* wip

* Update ClientInputSystem.tsx

* fix for avatar movement

* fix for followcamera orbit not working and decreased sensitivity (due to no longer calling setTargetCameraRotation() twice)

* some improvements

* Update FollowCameraComponent.ts

* Update FollowCameraComponent.ts

* tweaks

* fixing rotation issues with cameras when you're capturing another entity

* better fix for the orbiting with followcam, also fixing a logical bug in the orbit camera

* Update LinkComponent.ts

* fix ts errors

* attempting to fix some build errors

* Update ClickPlacementSystem.tsx

* fixes to clickPlacement after camera changes

* fixing what i broke earlier

* sorting fixes for objects with multiple InputComponents and only one InteractableComponent

* fixing possible nullref in updateboundingbox

* Update tsconfig.json

* Update tsconfig.json

* reduce angle for follow camera raycasts

* fix imports

* cleanup

* fixing type check

---------

Co-authored-by: Gheric Speiginer <[email protected]>
Co-authored-by: Daniel Belmes <[email protected]>
Co-authored-by: HexaField <[email protected]>

packages/client-core/src/common/components/TouchGamepad/index.tsx

@@ -62,7 +62,7 @@ const handleStop = () => {
 
 const buttonsConfig: Array<{ button: AnyButton; label: React.ReactElement }> = [
   {
-    button: XRStandardGamepadButton.Trigger,
+    button: XRStandardGamepadButton.XRStandardGamepadTrigger,
     label: <Icon type="TouchApp" />
   }
 ]

packages/client-core/src/systems/AvatarUISystem.tsx

@@ -133,7 +133,7 @@ const raycastComponentData = {
 
 const onSecondaryClick = () => {
   const { physicsWorld } = getState(PhysicsState)
-  const inputPointerEntity = InputPointerComponent.getPointerForCanvas(Engine.instance.viewerEntity)
+  const inputPointerEntity = InputPointerComponent.getPointersForCamera(Engine.instance.viewerEntity)[0]
   if (!inputPointerEntity) return
   const pointerPosition = getComponent(inputPointerEntity, InputPointerComponent).position
   const hits = Physics.castRayFromCamera(

packages/client-core/src/systems/WidgetUISystem.tsx

@@ -125,7 +125,7 @@ const execute = () => {
     const inputSource = getComponent(inputSourceEntity, InputSourceComponent)
     const keys = inputSource.buttons
     if (inputSource.source.gamepad?.mapping === 'xr-standard') {
-      if (keys[XRStandardGamepadButton.ButtonA]?.down)
+      if (keys[XRStandardGamepadButton.XRStandardGamepadButtonA]?.down)
         toggleWidgetsMenu(inputSource.source.handedness === 'left' ? 'right' : 'left')
     }
     /** @todo allow non HMDs to access the widget menu too */

packages/client-core/src/systems/createAnchorWidget.tsx

@@ -77,7 +77,7 @@ export function createAnchorWidget() {
         if (inputComponent.source.gamepad?.mapping !== 'xr-standard') continue
         if (inputComponent.source.handedness !== preferredHand) continue
 
-        const buttonInputPressed = inputComponent.buttons[XRStandardGamepadButton.Trigger]?.down
+        const buttonInputPressed = inputComponent.buttons[XRStandardGamepadButton.XRStandardGamepadTrigger]?.down
 
         if (buttonInputPressed) {
           xrState.scenePlacementMode.set('placed')
@@ -86,8 +86,8 @@ export function createAnchorWidget() {
 
         const { deltaSeconds } = getState(ECSState)
 
-        const xAxisInput = inputComponent.source.gamepad.axes[XRStandardGamepadAxes.ThumbstickX]
-        const yAxisInput = inputComponent.source.gamepad.axes[XRStandardGamepadAxes.ThumbstickY]
+        const xAxisInput = inputComponent.source.gamepad.axes[XRStandardGamepadAxes.XRStandardGamepadThumbstickX]
+        const yAxisInput = inputComponent.source.gamepad.axes[XRStandardGamepadAxes.XRStandardGamepadThumbstickY]
 
         const xDelta = xAxisInput * Math.PI * deltaSeconds
         getMutableState(XRState).sceneRotationOffset.set((currentValue) => currentValue + xDelta)
@@ -97,7 +97,7 @@ export function createAnchorWidget() {
           xrState.sceneScaleTarget.set((currentValue) => MathUtils.clamp(currentValue + yDelta, 0.01, 0.2))
         }
 
-        const triggerButtonPressed = inputComponent.buttons[XRStandardGamepadButton.Stick]?.down
+        const triggerButtonPressed = inputComponent.buttons[XRStandardGamepadButton.XRStandardGamepadStick]?.down
 
         if (triggerButtonPressed) {
           xrState.sceneScaleAutoMode.set(!xrState.sceneScaleAutoMode.value)

packages/ecs/src/ComponentFunctions.ts

@@ -508,6 +508,7 @@ export function _use(promise) {
  * Use a component in a reactive context (a React component)
  */
 export function useComponent<C extends Component<any>>(entity: Entity, Component: C) {
+  if (entity === UndefinedEntity) throw new Error('InvalidUsage: useComponent called with UndefinedEntity')
   if (!Component.stateMap[entity]) Component.stateMap[entity] = hookstate(none)
   const componentState = Component.stateMap[entity]!
   // use() will suspend the component (by throwing a promise) and resume when the promise is resolved

packages/editor/src/components/properties/CameraPropertiesNodeEditor.tsx

@@ -31,7 +31,7 @@ import { getOptionalComponent, useComponent } from '@etherealengine/ecs/src/Comp
 import { defineQuery } from '@etherealengine/ecs/src/QueryFunctions'
 import { CameraSettingsComponent } from '@etherealengine/engine/src/scene/components/CameraSettingsComponent'
 import { ModelComponent } from '@etherealengine/engine/src/scene/components/ModelComponent'
-import { CameraMode } from '@etherealengine/spatial/src/camera/types/CameraMode'
+import { FollowCameraMode } from '@etherealengine/spatial/src/camera/types/FollowCameraMode'
 import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent'
 import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree'
 
@@ -46,28 +46,29 @@ import { commitProperties, commitProperty, EditorComponentType, updateProperty }
 const cameraModeSelect = [
   {
     label: 'First Person',
-    value: CameraMode.FirstPerson
+    value: FollowCameraMode.FirstPerson
   },
   {
     label: 'Shoulder Cam',
-    value: CameraMode.ShoulderCam
+    value: FollowCameraMode.ShoulderCam
   },
   {
     label: 'Third Person',
-    value: CameraMode.ThirdPerson
+    value: FollowCameraMode.ThirdPerson
   },
   {
     label: 'Top Down',
-    value: CameraMode.TopDown
-  },
-  {
-    label: 'Strategic',
-    value: CameraMode.Strategic
-  },
-  {
-    label: 'Dynamic',
-    value: CameraMode.Dynamic
+    value: FollowCameraMode.TopDown
   }
+  // These are not currently defined or implemented:
+  // {
+  //   label: 'Strategic',
+  //   value: FollowCameraMode.Strategic
+  // },
+  // {
+  //   label: 'Dynamic',
+  //   value: FollowCameraMode.Dynamic
+  // }
 ]
 
 /** Types copied from Camera Modes of engine. */

packages/editor/src/functions/gizmoHelper.ts

@@ -483,7 +483,7 @@ export function controlUpdate(gizmoEntity: Entity) {
 
 function pointerHover(gizmoEntity) {
   // TODO support gizmos in multiple viewports
-  const inputPointerEntity = InputPointerComponent.getPointerForCanvas(Engine.instance.viewerEntity)
+  const inputPointerEntity = InputPointerComponent.getPointersForCamera(Engine.instance.viewerEntity)[0]
   if (!inputPointerEntity) return
   const pointerPosition = getComponent(inputPointerEntity, InputPointerComponent).position
   const gizmoControlComponent = getMutableComponent(gizmoEntity, TransformGizmoControlComponent)
@@ -509,7 +509,7 @@ function pointerHover(gizmoEntity) {
 
 function pointerDown(gizmoEntity) {
   // TODO support gizmos in multiple viewports
-  const inputPointerEntity = InputPointerComponent.getPointerForCanvas(Engine.instance.viewerEntity)
+  const inputPointerEntity = InputPointerComponent.getPointersForCamera(Engine.instance.viewerEntity)[0]
   if (!inputPointerEntity) return
   const pointer = getComponent(inputPointerEntity, InputPointerComponent)
   const gizmoControlComponent = getMutableComponent(gizmoEntity, TransformGizmoControlComponent)
@@ -768,7 +768,7 @@ function applyPivotRotation(entity, pivotToOriginMatrix, originToPivotMatrix, ro
 
 function pointerMove(gizmoEntity) {
   // TODO support gizmos in multiple viewports
-  const inputPointerEntity = InputPointerComponent.getPointerForCanvas(Engine.instance.viewerEntity)
+  const inputPointerEntity = InputPointerComponent.getPointersForCamera(Engine.instance.viewerEntity)[0]
   if (!inputPointerEntity) return
   const pointer = getComponent(inputPointerEntity, InputPointerComponent)
   const gizmoControlComponent = getMutableComponent(gizmoEntity, TransformGizmoControlComponent)
@@ -901,7 +901,7 @@ function pointerMove(gizmoEntity) {
 
 function pointerUp(gizmoEntity) {
   // TODO support gizmos in multiple viewports
-  const inputPointerEntity = InputPointerComponent.getPointerForCanvas(Engine.instance.viewerEntity)
+  const inputPointerEntity = InputPointerComponent.getPointersForCamera(Engine.instance.viewerEntity)[0]
   if (!inputPointerEntity) return
   const pointer = getComponent(inputPointerEntity, InputPointerComponent)
 

packages/editor/src/systems/ClickPlacementSystem.tsx

@@ -241,7 +241,7 @@ export const ClickPlacementSystem = defineSystem({
   execute: () => {
     const editorHelperState = getState(EditorHelperState)
     if (editorHelperState.placementMode !== PlacementMode.CLICK) return
-    const clickState = getState(ClickPlacementState)
+    const clickState = getMutableState(ClickPlacementState)
     const placementEntity = clickState.placementEntity
     if (!placementEntity) return
 
@@ -262,7 +262,7 @@ export const ClickPlacementSystem = defineSystem({
     let targetIntersection: { point: Vector3; normal: Vector3 } | null = null
 
     const viewerEntity = Engine.instance.viewerEntity
-    const mouseEntity = InputPointerComponent.getPointerForCanvas(viewerEntity)
+    const mouseEntity = InputPointerComponent.getPointersForCamera(viewerEntity)[0]
     if (!mouseEntity) return
 
     const buttons = InputComponent.getMergedButtons(viewerEntity)
@@ -271,14 +271,14 @@ export const ClickPlacementSystem = defineSystem({
     const zoom = axes[MouseScroll.VerticalScroll]
 
     if (buttons.SecondaryClick?.pressed) {
-      clickState.maxDistance -= zoom
+      clickState.maxDistance.set(clickState.maxDistance.value - zoom)
     }
 
     if (buttons.KeyE?.up) {
-      clickState.yawOffset += Math.PI / 4
+      clickState.yawOffset.set(clickState.yawOffset.value + Math.PI / 4)
     }
     if (buttons.KeyQ?.up) {
-      clickState.yawOffset -= Math.PI / 4
+      clickState.yawOffset.set(clickState.yawOffset.value - Math.PI / 4)
     }
     if (buttons.PrimaryClick?.up) {
       clickListener()
@@ -292,7 +292,7 @@ export const ClickPlacementSystem = defineSystem({
     const cameraPosition = pointerScreenRaycaster.ray.origin
     const cameraDirection = pointerScreenRaycaster.ray.direction
     const physicsIntersection = physicsWorld.castRayAndGetNormal(new Ray(cameraPosition, cameraDirection), 1000, false)
-    if (physicsIntersection && physicsIntersection.toi < clickState.maxDistance) {
+    if (physicsIntersection && physicsIntersection.toi < clickState.maxDistance.value) {
       const intersectPosition = cameraPosition
         .clone()
         .add(cameraDirection.clone().multiplyScalar(physicsIntersection.toi))
@@ -311,7 +311,7 @@ export const ClickPlacementSystem = defineSystem({
     //if (intersect.length === 0 && !targetIntersection) return
     for (let i = 0; i < intersect.length; i++) {
       const intersected = intersect[i]
-      if (intersected.distance > clickState.maxDistance) continue
+      if (intersected.distance > clickState.maxDistance.value) continue
       if (isPlacementDescendant(intersected.object.entity)) continue
       targetIntersection = {
         point: intersected.point,
@@ -321,16 +321,16 @@ export const ClickPlacementSystem = defineSystem({
     }
 
     if (!targetIntersection) {
-      const point = cameraPosition.clone().add(cameraDirection.clone().multiplyScalar(clickState.maxDistance))
+      const point = cameraPosition.clone().add(cameraDirection.clone().multiplyScalar(clickState.maxDistance.value))
       targetIntersection = { point, normal: new Vector3(0, 1, 0) }
     }
     const position = targetIntersection.point
     let rotation = new Quaternion().setFromUnitVectors(new Vector3(), targetIntersection.normal ?? new Vector3(0, 1, 0))
     const offset = new Quaternion().setFromEuler(
-      new Euler(clickState.pitchOffset, clickState.yawOffset, clickState.rollOffset)
+      new Euler(clickState.pitchOffset.value, clickState.yawOffset.value, clickState.rollOffset.value)
     )
     rotation = offset.multiply(rotation)
-    setComponent(placementEntity, TransformComponent, { position, rotation })
+    setComponent(placementEntity.value, TransformComponent, { position, rotation })
   }
 })
 

packages/engine/src/avatar/AvatarModule.ts

@@ -28,7 +28,6 @@ import { AvatarState } from './state/AvatarNetworkState'
 import { AnimationSystem } from './systems/AnimationSystem'
 import { AvatarAnimationSystem } from './systems/AvatarAnimationSystem'
 import { AvatarAutopilotSystem } from './systems/AvatarAutopilotSystem'
-import { AvatarCameraInputSystem } from './systems/AvatarCameraInputSystem'
 import { AvatarControllerSystem } from './systems/AvatarControllerSystem'
 import { AvatarInputSystem } from './systems/AvatarInputSystem'
 import { AvatarLoadingSystem } from './systems/AvatarLoadingSystem'
@@ -42,7 +41,6 @@ export default {
   AnimationSystem,
   AvatarAnimationSystem,
   AvatarAutopilotSystem,
-  AvatarCameraInputSystem,
   AvatarControllerSystem,
   AvatarIKTargetState,
   AvatarInputSystem,

packages/engine/src/avatar/components/AvatarControllerComponent.ts

@@ -48,6 +48,8 @@ import { CameraComponent } from '../../../../spatial/src/camera/components/Camer
 import { setAvatarColliderTransform } from '../functions/spawnAvatarReceptor'
 import { AvatarComponent } from './AvatarComponent'
 
+export const eyeOffset = 0.25
+
 export const AvatarControllerComponent = defineComponent({
   name: 'AvatarControllerComponent',
 
@@ -109,7 +111,8 @@ export const AvatarControllerComponent = defineComponent({
       const cameraEntity = avatarControllerComponent.cameraEntity.value
       if (cameraEntity && entityExists(cameraEntity) && hasComponent(cameraEntity, FollowCameraComponent)) {
         const cameraComponent = getComponent(cameraEntity, FollowCameraComponent)
-        cameraComponent.offset.set(0, avatarComponent.eyeHeight.value, 0)
+        cameraComponent.firstPersonOffset.set(0, avatarComponent.eyeHeight.value, eyeOffset)
+        cameraComponent.thirdPersonOffset.set(0, avatarComponent.eyeHeight.value, 0)
       }
     }, [avatarComponent.avatarHeight, camera.near])
 

packages/engine/src/avatar/functions/autopilotFunctions.ts

@@ -65,7 +65,7 @@ export const autopilotSetPosition = (entity: Entity) => {
 
   const { physicsWorld } = getState(PhysicsState)
 
-  const inputPointerEntity = InputPointerComponent.getPointerForCanvas(Engine.instance.viewerEntity)
+  const inputPointerEntity = InputPointerComponent.getPointersForCamera(Engine.instance.viewerEntity)[0]
   if (!inputPointerEntity) return
   const pointerPosition = getComponent(inputPointerEntity, InputPointerComponent).position
 

packages/engine/src/avatar/functions/spawnAvatarReceptor.ts

@@ -63,8 +63,7 @@ import { proxifyParentChildRelationships } from '../../scene/functions/loadGLTFM
 import { AnimationComponent } from '../components/AnimationComponent'
 import { AvatarAnimationComponent, AvatarRigComponent } from '../components/AvatarAnimationComponent'
 import { AvatarComponent } from '../components/AvatarComponent'
-import { AvatarColliderComponent, AvatarControllerComponent } from '../components/AvatarControllerComponent'
-import { eyeOffset } from '../systems/AvatarTransparencySystem'
+import { AvatarColliderComponent, AvatarControllerComponent, eyeOffset } from '../components/AvatarControllerComponent'
 
 export const spawnAvatarReceptor = (entityUUID: EntityUUID) => {
   const entity = UUIDComponent.getEntityByUUID(entityUUID)