From 4feee0838b4dfafc7b5232a3ad72bae9d79434e8 Mon Sep 17 00:00:00 2001 From: brentyi Date: Wed, 18 Dec 2024 02:28:05 -0800 Subject: [PATCH 1/7] Camera orientation tool --- src/viser/client/src/CameraControls.tsx | 190 ++++++++++++++++-- .../client/src/ControlPanel/GuiState.tsx | 2 + .../src/ControlPanel/ServerControls.tsx | 22 ++ 3 files changed, 199 insertions(+), 15 deletions(-) diff --git a/src/viser/client/src/CameraControls.tsx b/src/viser/client/src/CameraControls.tsx index 4ead77608..d183cd0f7 100644 --- a/src/viser/client/src/CameraControls.tsx +++ b/src/viser/client/src/CameraControls.tsx @@ -7,6 +7,71 @@ import { PerspectiveCamera } from "three"; import * as THREE from "three"; import { computeT_threeworld_world } from "./WorldTransformUtils"; import { useThrottledMessageSender } from "./WebsocketFunctions"; +import { Grid, PivotControls } from "@react-three/drei"; + +function CameraOrientationTool({ + pivotRef, + onPivotChange, + update, +}: { + pivotRef: React.RefObject; + onPivotChange: (matrix: THREE.Matrix4) => void; + update: () => void; +}) { + const viewer = useContext(ViewerContext)!; + const showCameraControls = viewer.useGui((state) => state.showCameraControls); + React.useEffect(update, [showCameraControls]); + + if (!showCameraControls) return null; + + return ( + { + onPivotChange(pivotRef.current!.matrix); + }} + > + + + + + + + ); +} export function SynchronizedCameraControls() { const viewer = useContext(ViewerContext)!; @@ -20,7 +85,97 @@ export function SynchronizedCameraControls() { lookAt: THREE.Vector3; } | null>(null); + const pivotRef = useRef(null); + + const cameraControlRef = viewer.cameraControlRef; + + const updateCameraLookAtAndUpFromPivotControl = (matrix: THREE.Matrix4) => { + if (!cameraControlRef.current) return; + + const targetPosition = new THREE.Vector3(); + targetPosition.setFromMatrixPosition(matrix); + + // const upVector = new THREE.Vector3(); + // upVector.setFromMatrixColumn(matrix, 1); + + const cameraControls = cameraControlRef.current; + const camera = cameraControlRef.current.camera; + camera.up.setFromMatrixColumn(matrix, 1); + + // Back up position. + const prevPosition = new THREE.Vector3(); + cameraControls.getPosition(prevPosition); + + cameraControls.updateCameraUp(); + // Restore position, which can get unexpectedly mutated in updateCameraUp(). + cameraControls.setPosition( + prevPosition.x, + prevPosition.y, + prevPosition.z, + false, + ); + + cameraControls.setLookAt( + cameraControls.camera.position.x, + cameraControls.camera.position.y, + cameraControls.camera.position.z, + targetPosition.x, + targetPosition.y, + targetPosition.z, + true, + // { + // up: upVector, + // }, + ); + }; + + const updatePivotControlFromCameraLookAtAndup = () => { + if (!cameraControlRef.current) return; + if (!pivotRef.current) return; + + const cameraControls = cameraControlRef.current; + const lookAt = cameraControls.getTarget(new THREE.Vector3()); + + // Rotate matrix s.t. it's y-axis aligns with the camera's up vector. + // We'll do this with math. + const origRotation = new THREE.Matrix4().extractRotation( + pivotRef.current.matrix, + ); + + const cameraUp = camera.up.clone().normalize(); + const pivotUp = new THREE.Vector3(0, 1, 0) + .applyMatrix4(origRotation) + .normalize(); + const axis = new THREE.Vector3() + .crossVectors(pivotUp, cameraUp) + .normalize(); + const angle = Math.acos(Math.min(1, Math.max(-1, cameraUp.dot(pivotUp)))); + + // Create rotation matrix + const rotationMatrix = new THREE.Matrix4(); + if (axis.lengthSq() > 0.0001) { + // Check if cross product is valid + rotationMatrix.makeRotationAxis(axis, angle); + } + // rotationMatrix.premultiply(origRotation); + + // Combine rotation with position + const matrix = new THREE.Matrix4(); + matrix.multiply(rotationMatrix); + matrix.multiply(origRotation); + matrix.setPosition(lookAt); + + pivotRef.current.matrix.copy(matrix); + pivotRef.current.updateMatrixWorld(true); + }; + viewer.resetCameraViewRef.current = () => { + viewer.cameraRef.current!.up.set( + initialCameraRef.current!.camera.up.x, + initialCameraRef.current!.camera.up.y, + initialCameraRef.current!.camera.up.z, + ); + viewer.cameraControlRef.current!.updateCameraUp(); viewer.cameraControlRef.current!.setLookAt( initialCameraRef.current!.camera.position.x, initialCameraRef.current!.camera.position.y, @@ -30,12 +185,6 @@ export function SynchronizedCameraControls() { initialCameraRef.current!.lookAt.z, true, ); - viewer.cameraRef.current!.up.set( - initialCameraRef.current!.camera.up.x, - initialCameraRef.current!.camera.up.y, - initialCameraRef.current!.camera.up.z, - ); - viewer.cameraControlRef.current!.updateCameraUp(); }; // Callback for sending cameras. @@ -51,6 +200,8 @@ export function SynchronizedCameraControls() { const t_world_camera = new THREE.Vector3(); const scale = new THREE.Vector3(); const sendCamera = React.useCallback(() => { + updatePivotControlFromCameraLookAtAndup(); + const three_camera = camera; const camera_control = viewer.cameraControlRef.current; @@ -247,14 +398,23 @@ export function SynchronizedCameraControls() { }, [CameraControls]); return ( - + <> + + + updateCameraLookAtAndUpFromPivotControl(matrix) + } + update={updatePivotControlFromCameraLookAtAndup} + /> + ); } diff --git a/src/viser/client/src/ControlPanel/GuiState.tsx b/src/viser/client/src/ControlPanel/GuiState.tsx index 7e84ac910..2aae25fa8 100644 --- a/src/viser/client/src/ControlPanel/GuiState.tsx +++ b/src/viser/client/src/ControlPanel/GuiState.tsx @@ -16,6 +16,7 @@ interface GuiState { shareUrl: string | null; websocketConnected: boolean; backgroundAvailable: boolean; + showCameraControls: boolean; guiUuidSetFromContainerUuid: { [containerUuid: string]: { [uuid: string]: true } | undefined; }; @@ -65,6 +66,7 @@ const cleanGuiState: GuiState = { shareUrl: null, websocketConnected: false, backgroundAvailable: false, + showCameraControls: false, guiUuidSetFromContainerUuid: {}, modals: [], guiOrderFromUuid: {}, diff --git a/src/viser/client/src/ControlPanel/ServerControls.tsx b/src/viser/client/src/ControlPanel/ServerControls.tsx index 38d7cd214..102505996 100644 --- a/src/viser/client/src/ControlPanel/ServerControls.tsx +++ b/src/viser/client/src/ControlPanel/ServerControls.tsx @@ -7,6 +7,7 @@ import { Switch, Text, TextInput, + Tooltip, } from "@mantine/core"; import { IconHomeMove, IconPhoto } from "@tabler/icons-react"; import { Stats } from "@react-three/drei"; @@ -130,6 +131,27 @@ export default function ServerControls() { > Reset View + + Show tool for setting the up direction +
and orbit center of the camera. + + } + refProp="rootRef" + position="top-start" + > + { + viewer.useGui.setState({ + showCameraControls: event.currentTarget.checked, + }); + }} + size="sm" + /> + Date: Mon, 23 Dec 2024 11:17:21 -0800 Subject: [PATCH 2/7] colors --- src/viser/client/src/CameraControls.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/viser/client/src/CameraControls.tsx b/src/viser/client/src/CameraControls.tsx index d183cd0f7..4205802f3 100644 --- a/src/viser/client/src/CameraControls.tsx +++ b/src/viser/client/src/CameraControls.tsx @@ -30,7 +30,7 @@ function CameraOrientationTool({ scale={200} lineWidth={4} fixed={true} - axisColors={["#aaaaaa", "#ff33ff", "#aaaaaa"]} + axisColors={["#ffaaff", "#ff33ff", "#ffaaff"]} disableScaling={true} onDragEnd={() => { onPivotChange(pivotRef.current!.matrix); @@ -63,8 +63,9 @@ function CameraOrientationTool({ Date: Thu, 26 Dec 2024 14:56:19 -0800 Subject: [PATCH 3/7] animate + polish --- src/viser/client/src/CameraControls.tsx | 131 +++++++++++++----- .../client/src/ControlPanel/GuiState.tsx | 4 +- .../src/ControlPanel/ServerControls.tsx | 12 +- 3 files changed, 108 insertions(+), 39 deletions(-) diff --git a/src/viser/client/src/CameraControls.tsx b/src/viser/client/src/CameraControls.tsx index 4205802f3..ecf965774 100644 --- a/src/viser/client/src/CameraControls.tsx +++ b/src/viser/client/src/CameraControls.tsx @@ -2,7 +2,8 @@ import { ViewerContext } from "./App"; import { CameraControls } from "@react-three/drei"; import { useThree } from "@react-three/fiber"; import * as holdEvent from "hold-event"; -import React, { useContext, useRef } from "react"; +import React, { useContext, useRef, useState } from "react"; +import { useFrame } from "@react-three/fiber"; import { PerspectiveCamera } from "three"; import * as THREE from "three"; import { computeT_threeworld_world } from "./WorldTransformUtils"; @@ -19,7 +20,9 @@ function CameraOrientationTool({ update: () => void; }) { const viewer = useContext(ViewerContext)!; - const showCameraControls = viewer.useGui((state) => state.showCameraControls); + const showCameraControls = viewer.useGui( + (state) => state.showCameraControlsTool, + ); React.useEffect(update, [showCameraControls]); if (!showCameraControls) return null; @@ -90,47 +93,107 @@ export function SynchronizedCameraControls() { const cameraControlRef = viewer.cameraControlRef; + // Animation state interface + interface CameraAnimation { + startUp: THREE.Vector3; + targetUp: THREE.Vector3; + startLookAt: THREE.Vector3; + targetLookAt: THREE.Vector3; + startTime: number; + duration: number; + } + + const [cameraAnimation, setCameraAnimation] = + useState(null); + + // Animation parameters + const ANIMATION_DURATION = 0.5; // seconds + + useFrame((state) => { + if (cameraAnimation && cameraControlRef.current) { + const cameraControls = cameraControlRef.current; + const camera = cameraControls.camera; + + const elapsed = state.clock.getElapsedTime() - cameraAnimation.startTime; + const progress = Math.min(elapsed / cameraAnimation.duration, 1); + + // Smooth step easing + const t = progress * progress * (3 - 2 * progress); + + // Interpolate up vector + const newUp = new THREE.Vector3() + .copy(cameraAnimation.startUp) + .lerp(cameraAnimation.targetUp, t) + .normalize(); + + // Interpolate look-at position + const newLookAt = new THREE.Vector3() + .copy(cameraAnimation.startLookAt) + .lerp(cameraAnimation.targetLookAt, t); + + camera.up.copy(newUp); + + // Back up position + const prevPosition = new THREE.Vector3(); + cameraControls.getPosition(prevPosition); + + cameraControls.updateCameraUp(); + + // Restore position and set new look-at + cameraControls.setPosition( + prevPosition.x, + prevPosition.y, + prevPosition.z, + false, + ); + + cameraControls.setLookAt( + prevPosition.x, + prevPosition.y, + prevPosition.z, + newLookAt.x, + newLookAt.y, + newLookAt.z, + false, + ); + + // Clear animation when complete + if (progress >= 1) { + setCameraAnimation(null); + } + } + }); + + const { clock } = useThree(); + const updateCameraLookAtAndUpFromPivotControl = (matrix: THREE.Matrix4) => { if (!cameraControlRef.current) return; const targetPosition = new THREE.Vector3(); targetPosition.setFromMatrixPosition(matrix); - // const upVector = new THREE.Vector3(); - // upVector.setFromMatrixColumn(matrix, 1); - const cameraControls = cameraControlRef.current; const camera = cameraControlRef.current.camera; - camera.up.setFromMatrixColumn(matrix, 1); - - // Back up position. - const prevPosition = new THREE.Vector3(); - cameraControls.getPosition(prevPosition); - - cameraControls.updateCameraUp(); - // Restore position, which can get unexpectedly mutated in updateCameraUp(). - cameraControls.setPosition( - prevPosition.x, - prevPosition.y, - prevPosition.z, - false, - ); - cameraControls.setLookAt( - cameraControls.camera.position.x, - cameraControls.camera.position.y, - cameraControls.camera.position.z, - targetPosition.x, - targetPosition.y, - targetPosition.z, - true, - // { - // up: upVector, - // }, - ); + // Get target up vector from matrix + const targetUp = new THREE.Vector3().setFromMatrixColumn(matrix, 1); + + // Get current look-at position + const currentLookAt = cameraControls.getTarget(new THREE.Vector3()); + + // Start new animation + setCameraAnimation({ + startUp: camera.up.clone(), + targetUp: targetUp, + startLookAt: currentLookAt, + targetLookAt: targetPosition, + startTime: clock.getElapsedTime(), + duration: ANIMATION_DURATION, + }); }; const updatePivotControlFromCameraLookAtAndup = () => { + if (cameraAnimation !== null) return; if (!cameraControlRef.current) return; if (!pivotRef.current) return; @@ -411,9 +474,9 @@ export function SynchronizedCameraControls() { /> - updateCameraLookAtAndUpFromPivotControl(matrix) - } + onPivotChange={(matrix) => { + updateCameraLookAtAndUpFromPivotControl(matrix); + }} update={updatePivotControlFromCameraLookAtAndup} /> diff --git a/src/viser/client/src/ControlPanel/GuiState.tsx b/src/viser/client/src/ControlPanel/GuiState.tsx index 2aae25fa8..f1371bbc7 100644 --- a/src/viser/client/src/ControlPanel/GuiState.tsx +++ b/src/viser/client/src/ControlPanel/GuiState.tsx @@ -16,7 +16,7 @@ interface GuiState { shareUrl: string | null; websocketConnected: boolean; backgroundAvailable: boolean; - showCameraControls: boolean; + showCameraControlsTool: boolean; guiUuidSetFromContainerUuid: { [containerUuid: string]: { [uuid: string]: true } | undefined; }; @@ -66,7 +66,7 @@ const cleanGuiState: GuiState = { shareUrl: null, websocketConnected: false, backgroundAvailable: false, - showCameraControls: false, + showCameraControlsTool: false, guiUuidSetFromContainerUuid: {}, modals: [], guiOrderFromUuid: {}, diff --git a/src/viser/client/src/ControlPanel/ServerControls.tsx b/src/viser/client/src/ControlPanel/ServerControls.tsx index 102505996..4a35645de 100644 --- a/src/viser/client/src/ControlPanel/ServerControls.tsx +++ b/src/viser/client/src/ControlPanel/ServerControls.tsx @@ -134,8 +134,14 @@ export default function ServerControls() { - Show tool for setting the up direction -
and orbit center of the camera. + Show tool for setting the look-at point and +
+ up direction of the camera. +
+
+ These can be used to re-orient the camera's +
+ orbit controls. } refProp="rootRef" @@ -146,7 +152,7 @@ export default function ServerControls() { label="Camera Orientation Tool" onChange={(event) => { viewer.useGui.setState({ - showCameraControls: event.currentTarget.checked, + showCameraControlsTool: event.currentTarget.checked, }); }} size="sm" From d010c374c10e83abfd8afa0889b17f9fcb69067f Mon Sep 17 00:00:00 2001 From: brentyi Date: Thu, 26 Dec 2024 18:30:49 -0800 Subject: [PATCH 4/7] comment --- src/viser/client/src/CameraControls.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/viser/client/src/CameraControls.tsx b/src/viser/client/src/CameraControls.tsx index ecf965774..9a365335d 100644 --- a/src/viser/client/src/CameraControls.tsx +++ b/src/viser/client/src/CameraControls.tsx @@ -48,6 +48,7 @@ function CameraOrientationTool({ size: { value: 200.0 }, }} vertexShader={` + // Custom shader for defining sphere size in screen space. uniform float size; void main() { vec4 clipPos = projectionMatrix * modelViewMatrix * vec4(0.0, 0.0, 0.0, 1.0); From cd0f03ed6497062622cab741c68b82a858faa5f7 Mon Sep 17 00:00:00 2001 From: brentyi Date: Thu, 26 Dec 2024 18:31:19 -0800 Subject: [PATCH 5/7] downgrade ubuntu for Python 3.8 --- .github/workflows/pyright.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyright.yml b/.github/workflows/pyright.yml index d8f47ba45..5332f58bd 100644 --- a/.github/workflows/pyright.yml +++ b/.github/workflows/pyright.yml @@ -8,7 +8,7 @@ on: jobs: pyright: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From d49af897ff09058c39398df802c1a638fc5d603b Mon Sep 17 00:00:00 2001 From: brentyi Date: Sat, 4 Jan 2025 03:10:00 -0800 Subject: [PATCH 6/7] Rename + show if logging camera --- src/viser/client/src/CameraControls.tsx | 11 +++++++---- src/viser/client/src/ControlPanel/GuiState.tsx | 4 ++-- src/viser/client/src/ControlPanel/ServerControls.tsx | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/viser/client/src/CameraControls.tsx b/src/viser/client/src/CameraControls.tsx index d11095879..430c3c81b 100644 --- a/src/viser/client/src/CameraControls.tsx +++ b/src/viser/client/src/CameraControls.tsx @@ -10,22 +10,24 @@ import { computeT_threeworld_world } from "./WorldTransformUtils"; import { useThrottledMessageSender } from "./WebsocketFunctions"; import { Grid, PivotControls } from "@react-three/drei"; -function CameraOrientationTool({ +function OrbitOriginTool({ + forceShow, pivotRef, onPivotChange, update, }: { + forceShow: boolean; pivotRef: React.RefObject; onPivotChange: (matrix: THREE.Matrix4) => void; update: () => void; }) { const viewer = useContext(ViewerContext)!; const showCameraControls = viewer.useGui( - (state) => state.showCameraControlsTool, + (state) => state.showOrbitOriginTool, ); React.useEffect(update, [showCameraControls]); - if (!showCameraControls) return null; + if (!showCameraControls && !forceShow) return null; return ( - { updateCameraLookAtAndUpFromPivotControl(matrix); diff --git a/src/viser/client/src/ControlPanel/GuiState.tsx b/src/viser/client/src/ControlPanel/GuiState.tsx index f1371bbc7..c6aa3124c 100644 --- a/src/viser/client/src/ControlPanel/GuiState.tsx +++ b/src/viser/client/src/ControlPanel/GuiState.tsx @@ -16,7 +16,7 @@ interface GuiState { shareUrl: string | null; websocketConnected: boolean; backgroundAvailable: boolean; - showCameraControlsTool: boolean; + showOrbitOriginTool: boolean; guiUuidSetFromContainerUuid: { [containerUuid: string]: { [uuid: string]: true } | undefined; }; @@ -66,7 +66,7 @@ const cleanGuiState: GuiState = { shareUrl: null, websocketConnected: false, backgroundAvailable: false, - showCameraControlsTool: false, + showOrbitOriginTool: false, guiUuidSetFromContainerUuid: {}, modals: [], guiOrderFromUuid: {}, diff --git a/src/viser/client/src/ControlPanel/ServerControls.tsx b/src/viser/client/src/ControlPanel/ServerControls.tsx index 4a35645de..c7faf975a 100644 --- a/src/viser/client/src/ControlPanel/ServerControls.tsx +++ b/src/viser/client/src/ControlPanel/ServerControls.tsx @@ -149,10 +149,10 @@ export default function ServerControls() { > { viewer.useGui.setState({ - showCameraControlsTool: event.currentTarget.checked, + showOrbitOriginTool: event.currentTarget.checked, }); }} size="sm" From 2085d76d192811060850704f883b28fc4d333a92 Mon Sep 17 00:00:00 2001 From: brentyi Date: Sat, 4 Jan 2025 03:26:40 -0800 Subject: [PATCH 7/7] Control panel nits --- src/viser/client/src/ControlPanel/ControlPanel.tsx | 2 +- src/viser/client/src/ControlPanel/ServerControls.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/viser/client/src/ControlPanel/ControlPanel.tsx b/src/viser/client/src/ControlPanel/ControlPanel.tsx index 6a143de20..b48561189 100644 --- a/src/viser/client/src/ControlPanel/ControlPanel.tsx +++ b/src/viser/client/src/ControlPanel/ControlPanel.tsx @@ -83,7 +83,7 @@ export default function ControlPanel(props: { > {showSettings ? ( diff --git a/src/viser/client/src/ControlPanel/ServerControls.tsx b/src/viser/client/src/ControlPanel/ServerControls.tsx index c7faf975a..4741faf13 100644 --- a/src/viser/client/src/ControlPanel/ServerControls.tsx +++ b/src/viser/client/src/ControlPanel/ServerControls.tsx @@ -139,9 +139,9 @@ export default function ServerControls() { up direction of the camera.

- These can be used to re-orient the camera's + These can be used to set the origin of the
- orbit controls. + camera's orbit controls. } refProp="rootRef"