Update camera orientation for ray in SceneNodeClickMessage (#155)

* Update camera orientation for ray in SceneNodeClickMessage

* Rename to camelcase

src/viser/client/src/App.tsx

@@ -43,7 +43,7 @@ import { useSceneTreeState } from "./SceneTreeState";
 import { GetRenderRequestMessage, Message } from "./WebsocketMessages";
 import { makeThrottledMessageSender } from "./WebsocketFunctions";
 import { useDisclosure } from "@mantine/hooks";
-import { computeT_threeworld_world } from "./WorldTransformUtils";
+import { rayToViserCoords } from "./WorldTransformUtils";
 
 export type ViewerContextContents = {
   // Zustand hooks.
@@ -243,24 +243,14 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) {
         const raycaster = new THREE.Raycaster();
         raycaster.setFromCamera(mouseVector, viewer.cameraRef.current!);
 
-        const T_world_threeworld = computeT_threeworld_world(viewer).invert();
-
-        const origin = raycaster.ray.origin
-          .clone()
-          .applyMatrix4(T_world_threeworld);
-
-        // Compute just the rotation term without new memory allocation; this
-        // will mutate T_world_threeworld!
-        const R_world_threeworld = T_world_threeworld.setPosition(0.0, 0.0, 0);
-        const direction = raycaster.ray.direction
-          .clone()
-          .applyMatrix4(R_world_threeworld);
+        // Convert ray to viser coordinates.
+        const ray = rayToViserCoords(viewer, raycaster.ray);
 
         sendClickThrottled({
           type: "ScenePointerMessage",
           event_type: "click",
-          ray_origin: [origin.x, origin.y, origin.z],
-          ray_direction: [direction.x, direction.y, direction.z],
+          ray_origin: [ray.origin.x, ray.origin.y, ray.origin.z],
+          ray_direction: [ray.direction.x, ray.direction.y, ray.direction.z],
         });
       }}
     >

src/viser/client/src/SceneTree.tsx

@@ -10,8 +10,10 @@ import { immerable } from "immer";
 import { Text } from "@mantine/core";
 import { useSceneTreeState } from "./SceneTreeState";
 import { ErrorBoundary } from "react-error-boundary";
+import { rayToViserCoords } from "./WorldTransformUtils";
 import { HoverableContext } from "./ThreeAssets";
 
+
 export type MakeObject<T extends THREE.Object3D = THREE.Object3D> = (
   ref: React.Ref<T>,
 ) => React.ReactNode;
@@ -290,16 +292,14 @@ export function SceneNodeThreeObject(props: {
               e.stopPropagation();
               const state = dragInfo.current;
               if (state.dragging) return;
+              // Convert ray to viser coordinates.
+              const ray = rayToViserCoords(viewer, e.ray);
               sendClicksThrottled({
                 type: "SceneNodeClickMessage",
                 name: props.name,
                 // Note that the threejs up is +Y, but we expose a +Z up.
-                ray_origin: [e.ray.origin.x, -e.ray.origin.z, e.ray.origin.y],
-                ray_direction: [
-                  e.ray.direction.x,
-                  -e.ray.direction.z,
-                  e.ray.direction.y,
-                ],
+                ray_origin: [ray.origin.x, ray.origin.y, ray.origin.z],
+                ray_direction: [ray.direction.x, ray.direction.y, ray.direction.z],
               });
             }}
             onPointerOver={(e) => {

src/viser/client/src/WorldTransformUtils.ts

@@ -15,3 +15,26 @@ export function computeT_threeworld_world(viewer: ViewerContextContents) {
     )
     .setPosition(position[0], position[1], position[2]);
 }
+
+/** Helper for converting a ray from the three.js world frame to the Python
+ * world frame. Applies the transformation from computeT_threeworld_world.
+ */
+export function rayToViserCoords(
+  viewer: ViewerContextContents,
+  ray: THREE.Ray,
+): THREE.Ray {
+  const T_world_threeworld = computeT_threeworld_world(viewer).invert();
+
+  const origin = ray.origin
+    .clone()
+    .applyMatrix4(T_world_threeworld);
+
+  // Compute just the rotation term without new memory allocation; this
+  // will mutate T_world_threeworld!
+  const R_world_threeworld = T_world_threeworld.setPosition(0.0, 0.0, 0);
+  const direction = ray.direction
+    .clone()
+    .applyMatrix4(R_world_threeworld);
+
+  return new THREE.Ray(origin, direction);
+}