From 00d095c8a717205494aba287d32c21fd6204e97f Mon Sep 17 00:00:00 2001 From: Chung Min Kim Date: Sat, 16 Dec 2023 00:29:45 -0800 Subject: [PATCH] Fix camera orientation for scenepointer (#151) * Fix camera orientation for scenepointer * Fix raycast coordinate frame * Fix raycast coordinate frame --------- Co-authored-by: Brent Yi --- pyproject.toml | 2 +- src/viser/client/src/App.tsx | 27 +++++++++++++-------- src/viser/client/src/CameraControls.tsx | 4 +-- src/viser/client/src/WebsocketInterface.tsx | 8 +++--- src/viser/client/src/WorldTransformUtils.ts | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9edd37234..62c6e9f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "viser" -version = "0.1.15" +version = "0.1.16" description = "3D visualization + Python" readme = "README.md" license = { text="MIT" } diff --git a/src/viser/client/src/App.tsx b/src/viser/client/src/App.tsx index 43947917b..ec7f09611 100644 --- a/src/viser/client/src/App.tsx +++ b/src/viser/client/src/App.tsx @@ -49,6 +49,7 @@ import { useSceneTreeState } from "./SceneTreeState"; import { GetRenderRequestMessage, Message } from "./WebsocketMessages"; import { makeThrottledMessageSender } from "./WebsocketFunctions"; import { useDisclosure } from "@mantine/hooks"; +import { computeT_threeworld_world } from "./WorldTransformUtils"; export type ViewerContextContents = { // Zustand hooks. @@ -234,6 +235,7 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) { mouseVector.y = 1 - 2 * (e.nativeEvent.offsetY / viewer.canvasRef.current!.clientHeight); + console.log(mouseVector.x, mouseVector.y); if ( mouseVector.x > 1 || mouseVector.x < -1 || @@ -245,19 +247,24 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) { const raycaster = new THREE.Raycaster(); raycaster.setFromCamera(mouseVector, viewer.cameraRef.current!); + const T_world_threeworld = computeT_threeworld_world(viewer).invert(); + + const origin = raycaster.ray.origin + .clone() + .applyMatrix4(T_world_threeworld); + + // Compute just the rotation term without new memory allocation; this + // will mutate T_world_threeworld! + const R_world_threeworld = T_world_threeworld.setPosition(0.0, 0.0, 0); + const direction = raycaster.ray.direction + .clone() + .applyMatrix4(R_world_threeworld); + sendClickThrottled({ type: "ScenePointerMessage", event_type: "click", - ray_origin: [ - raycaster.ray.origin.x, - -raycaster.ray.origin.z, - raycaster.ray.origin.y, - ], - ray_direction: [ - raycaster.ray.direction.x, - -raycaster.ray.direction.z, - raycaster.ray.direction.y, - ], + ray_origin: [origin.x, origin.y, origin.z], + ray_direction: [direction.x, direction.y, direction.z], }); }} > diff --git a/src/viser/client/src/CameraControls.tsx b/src/viser/client/src/CameraControls.tsx index 477aff33d..983309cee 100644 --- a/src/viser/client/src/CameraControls.tsx +++ b/src/viser/client/src/CameraControls.tsx @@ -6,7 +6,7 @@ import * as holdEvent from "hold-event"; import React, { useContext, useRef } from "react"; import { PerspectiveCamera } from "three"; import * as THREE from "three"; -import { getT_threeworld_world } from "./WorldTransformUtils"; +import { computeT_threeworld_world } from "./WorldTransformUtils"; export function SynchronizedCameraControls() { const viewer = useContext(ViewerContext)!; @@ -57,7 +57,7 @@ export function SynchronizedCameraControls() { const R_threecam_cam = new THREE.Quaternion().setFromEuler( new THREE.Euler(Math.PI, 0.0, 0.0), ); - const T_world_threeworld = getT_threeworld_world(viewer).invert(); + const T_world_threeworld = computeT_threeworld_world(viewer).invert(); const T_world_camera = T_world_threeworld.clone() .multiply( new THREE.Matrix4() diff --git a/src/viser/client/src/WebsocketInterface.tsx b/src/viser/client/src/WebsocketInterface.tsx index 2a4d9421f..a803fa075 100644 --- a/src/viser/client/src/WebsocketInterface.tsx +++ b/src/viser/client/src/WebsocketInterface.tsx @@ -28,7 +28,7 @@ import { useFrame } from "@react-three/fiber"; import GeneratedGuiContainer from "./ControlPanel/Generated"; import { MantineProvider, Paper, Progress } from "@mantine/core"; import { IconCheck } from "@tabler/icons-react"; -import { getT_threeworld_world } from "./WorldTransformUtils"; +import { computeT_threeworld_world } from "./WorldTransformUtils"; /** Convert raw RGB color buffers to linear color buffers. **/ function threeColorBufferFromUint8Buffer(colors: ArrayBuffer) { @@ -427,7 +427,7 @@ function useMessageHandler() { case "SetCameraLookAtMessage": { const cameraControls = viewer.cameraControlRef.current!; - const T_threeworld_world = getT_threeworld_world(viewer); + const T_threeworld_world = computeT_threeworld_world(viewer); const target = new THREE.Vector3( message.look_at[0], message.look_at[1], @@ -440,7 +440,7 @@ function useMessageHandler() { case "SetCameraUpDirectionMessage": { const camera = viewer.cameraRef.current!; const cameraControls = viewer.cameraControlRef.current!; - const T_threeworld_world = getT_threeworld_world(viewer); + const T_threeworld_world = computeT_threeworld_world(viewer); const updir = new THREE.Vector3( message.position[0], message.position[1], @@ -478,7 +478,7 @@ function useMessageHandler() { message.position[2], ); - const T_threeworld_world = getT_threeworld_world(viewer); + const T_threeworld_world = computeT_threeworld_world(viewer); position_cmd.applyMatrix4(T_threeworld_world); cameraControls.setPosition( diff --git a/src/viser/client/src/WorldTransformUtils.ts b/src/viser/client/src/WorldTransformUtils.ts index 1abb87551..a45799b6b 100644 --- a/src/viser/client/src/WorldTransformUtils.ts +++ b/src/viser/client/src/WorldTransformUtils.ts @@ -4,7 +4,7 @@ import * as THREE from "three"; /** Helper for computing the transformation between the three.js world and the * Python-exposed world frames. This is useful for things like switching * between +Y and +Z up directions for the world frame. */ -export function getT_threeworld_world(viewer: ViewerContextContents) { +export function computeT_threeworld_world(viewer: ViewerContextContents) { const wxyz = viewer.nodeAttributesFromName.current[""]!.wxyz!; const position = viewer.nodeAttributesFromName.current[""]!.position ?? [ 0, 0, 0,