diff --git a/examples/22_custom_gui_component.py b/examples/22_custom_gui_component.py new file mode 100644 index 000000000..6c1bd4a1b --- /dev/null +++ b/examples/22_custom_gui_component.py @@ -0,0 +1,65 @@ +"""Advanced GUI - custom GUI components""" + +import time +from pathlib import Path + +import numpy as onp + +import trimesh +import viser +import viser.transforms as tf +from viser import Icon + +mesh = trimesh.load_mesh(Path(__file__).parent / "assets/dragon.obj") +assert isinstance(mesh, trimesh.Trimesh) +mesh.apply_scale(0.05) + +vertices = mesh.vertices +faces = mesh.faces +print(f"Loaded mesh with {vertices.shape} vertices, {faces.shape} faces") + +server = viser.ViserServer() + +import_button = server.add_gui_button("Import", icon=Icon.FOLDER_OPEN) +export_button = server.add_gui_button("Export", icon=Icon.DOWNLOAD) + +fps = server.add_gui_number("FPS", 24, min=1, icon=Icon.KEYFRAMES, hint="Frames per second") +duration = server.add_gui_number("Duration", 4.0, min=0.1, icon=Icon.CLOCK_HOUR_5, hint="Duration in seconds") +width = server.add_gui_number("Width", 1920, min=100, icon=Icon.ARROWS_HORIZONTAL, hint="Width in px") +height = server.add_gui_number("Height", 1080, min=100, icon=Icon.ARROWS_VERTICAL, hint="Height in px") +fov = server.add_gui_number("FOV", 75, min=1, max=179, icon=Icon.CAMERA, hint="Field of view") +smoothness = server.add_gui_slider("Smoothness", 0.5, min=0.0, max=1.0, step=0.01, hint="Trajectory smoothing") + + +duration = 4 +cameras_slider = server.add_gui_multi_slider( + "Timeline", + min=0., + max=1., + step=0.01, + initial_value=[0.0, 0.5, 1.0], + disabled=False, + marks=[(x, f'{x*duration:.1f}s') for x in [0., 0.5, 1.0]], +) + +@duration.on_update +def _(_) -> None: + cameras_slider.marks=[(x, f'{x*duration.value:.1f}s') for x in [0., 0.5, 1.0]], + +server.add_mesh_simple( + name="/simple", + vertices=vertices, + faces=faces, + wxyz=tf.SO3.from_x_radians(onp.pi / 2).wxyz, + position=(0.0, 0.0, 0.0), +) +server.add_mesh_trimesh( + name="/trimesh", + mesh=mesh.smoothed(), + wxyz=tf.SO3.from_x_radians(onp.pi / 2).wxyz, + position=(0.0, 5.0, 0.0), +) +panel = server.add_gui_camera_trajectory_panel() + +while True: + time.sleep(10.0) \ No newline at end of file diff --git a/src/viser/_gui_api.py b/src/viser/_gui_api.py index 16b188fa3..0140f057e 100644 --- a/src/viser/_gui_api.py +++ b/src/viser/_gui_api.py @@ -38,11 +38,11 @@ GuiModalHandle, GuiTabGroupHandle, SupportsRemoveProtocol, + GuiCameraTrajectoryPanelHandle, _GuiHandleState, _GuiInputHandle, _make_unique_id, ) -from ._icons import base64_from_icon from ._icons_enum import Icon from ._message_api import MessageApi, cast_vector @@ -272,7 +272,7 @@ def add_gui_tab_group( return GuiTabGroupHandle( _tab_group_id=tab_group_id, _labels=[], - _icons_base64=[], + _icons=[], _tabs=[], _gui_api=self, _container_id=self._get_container_id(), @@ -366,7 +366,7 @@ def add_gui_button( hint=hint, initial_value=False, color=color, - icon_base64=None if icon is None else base64_from_icon(icon), + icon=None if icon is None else icon.value, ), disabled=disabled, visible=visible, @@ -534,6 +534,7 @@ def add_gui_number( visible: bool = True, hint: Optional[str] = None, order: Optional[float] = None, + icon: Optional[Icon] = None, ) -> GuiInputHandle[IntOrFloat]: """Add a number input to the GUI, with user-specifiable bound and precision parameters. @@ -584,6 +585,7 @@ def add_gui_number( min=min, max=max, precision=_compute_precision_digits(step), + icon=icon.value if icon is not None else None, step=step, ), disabled=disabled, @@ -740,6 +742,41 @@ def add_gui_dropdown( ) -> GuiDropdownHandle[TString]: ... + def add_gui_camera_trajectory_panel( + self, + order: Optional[float] = None, + visible: bool = True, + ) -> GuiCameraTrajectoryPanelHandle: + """ + Add a camera trajectory panel to the GUI. + + Returns: + A handle that can be used to interact with the GUI element. + """ + id = _make_unique_id() + order = _apply_default_order(order) + + # Send add GUI input message. + self._get_api()._queue(_messages.GuiAddCameraTrajectoryPanelMessage( + order=order, + id=id, + container_id=self._get_container_id(), + )) + + # Construct handle. + handle = GuiCameraTrajectoryPanelHandle( + _gui_api=self, + _container_id=self._get_container_id(), + _visible=True, + _id=id, + _order=order, + ) + + # Set the visible field. These will queue messages under-the-hood. + if not visible: + handle.visible = visible + return handle + def add_gui_dropdown( self, label: str, @@ -852,6 +889,80 @@ def add_gui_slider( is_button=False, ) + def add_gui_multi_slider( + self, + label: str, + min: IntOrFloat, + max: IntOrFloat, + step: IntOrFloat, + initial_value: List[IntOrFloat], + disabled: bool = False, + visible: bool = True, + min_range: Optional[IntOrFloat] = None, + hint: Optional[str] = None, + order: Optional[float] = None, + marks: Optional[List[Tuple[IntOrFloat, Optional[str]]]] = None, + ) -> GuiInputHandle[IntOrFloat]: + """Add a multi slider to the GUI. Types of the min, max, step, and initial value should match. + + Args: + label: Label to display on the slider. + min: Minimum value of the slider. + max: Maximum value of the slider. + step: Step size of the slider. + initial_value: Initial values of the slider. + disabled: Whether the slider is disabled. + visible: Whether the slider is visible. + min_range: Optional minimum difference between two values of the slider. + hint: Optional hint to display on hover. + order: Optional ordering, smallest values will be displayed first. + + Returns: + A handle that can be used to interact with the GUI element. + """ + assert max >= min + if step > max - min: + step = max - min + assert all(max >= x >= min for x in initial_value) + + # GUI callbacks cast incoming values to match the type of the initial value. If + # the min, max, or step is a float, we should cast to a float. + if len(initial_value) > 0 and (type(initial_value[0]) is int and ( + type(min) is float or type(max) is float or type(step) is float + )): + initial_value = [float(x) for x in initial_value] # type: ignore + + # TODO: as of 6/5/2023, this assert will break something in nerfstudio. (at + # least LERF) + # + # assert type(min) == type(max) == type(step) == type(initial_value) + + id = _make_unique_id() + order = _apply_default_order(order) + return self._create_gui_input( + initial_value=initial_value, + message=_messages.GuiAddMultiSliderMessage( + order=order, + id=id, + label=label, + container_id=self._get_container_id(), + hint=hint, + min=min, + min_range=min_range, + max=max, + step=step, + initial_value=initial_value, + precision=_compute_precision_digits(step), + marks=[ + _messages.GuiSliderMark(value=x, label=label) + for x, label in marks + ] if marks is not None else None, + ), + disabled=disabled, + visible=visible, + is_button=False, + ) + def add_gui_rgb( self, label: str, diff --git a/src/viser/_gui_handles.py b/src/viser/_gui_handles.py index 614a2f839..7a3ab92f9 100644 --- a/src/viser/_gui_handles.py +++ b/src/viser/_gui_handles.py @@ -26,7 +26,6 @@ import numpy as onp from typing_extensions import Protocol -from ._icons import base64_from_icon from ._icons_enum import Icon from ._message_api import _encode_image_base64 from ._messages import ( @@ -332,7 +331,7 @@ def options(self, options: Iterable[StringType]) -> None: class GuiTabGroupHandle: _tab_group_id: str _labels: List[str] - _icons_base64: List[Optional[str]] + _icons: List[Optional[str]] _tabs: List[GuiTabHandle] _gui_api: GuiApi _container_id: str # Parent. @@ -352,7 +351,7 @@ def add_tab(self, label: str, icon: Optional[Icon] = None) -> GuiTabHandle: out = GuiTabHandle(_parent=self, _id=id) self._labels.append(label) - self._icons_base64.append(None if icon is None else base64_from_icon(icon)) + self._icons.append(None if icon is None else icon.value) self._tabs.append(out) self._sync_with_client() @@ -372,7 +371,7 @@ def _sync_with_client(self) -> None: id=self._tab_group_id, container_id=self._container_id, tab_labels=tuple(self._labels), - tab_icons_base64=tuple(self._icons_base64), + tab_icons=tuple(self._icons), tab_container_ids=tuple(tab._id for tab in self._tabs), ) ) @@ -495,7 +494,7 @@ def remove(self) -> None: self._parent._gui_api._container_handle_from_id.pop(self._id) self._parent._labels.pop(container_index) - self._parent._icons_base64.pop(container_index) + self._parent._icons.pop(container_index) self._parent._tabs.pop(container_index) self._parent._sync_with_client() @@ -596,3 +595,41 @@ def remove(self) -> None: """Permanently remove this markdown from the visualizer.""" api = self._gui_api._get_api() api._queue(GuiRemoveMessage(self._id)) + + +@dataclasses.dataclass +class GuiCameraTrajectoryPanelHandle: + _gui_api: GuiApi + _id: str + _visible: bool + _container_id: str # Parent. + _order: float + + @property + def order(self) -> float: + """Read-only order value, which dictates the position of the GUI element.""" + return self._order + + @property + def visible(self) -> bool: + """Temporarily show or hide this GUI element from the visualizer. Synchronized + automatically when assigned.""" + return self._visible + + @visible.setter + def visible(self, visible: bool) -> None: + if visible == self.visible: + return + + self._gui_api._get_api()._queue(GuiSetVisibleMessage(self._id, visible=visible)) + self._visible = visible + + def __post_init__(self) -> None: + """We need to register ourself after construction for callbacks to work.""" + parent = self._gui_api._container_handle_from_id[self._container_id] + parent._children[self._id] = self + + def remove(self) -> None: + """Permanently remove this markdown from the visualizer.""" + api = self._gui_api._get_api() + api._queue(GuiRemoveMessage(self._id)) diff --git a/src/viser/_messages.py b/src/viser/_messages.py index 86a3a34c4..0d7d304da 100644 --- a/src/viser/_messages.py +++ b/src/viser/_messages.py @@ -4,7 +4,7 @@ from __future__ import annotations import dataclasses -from typing import Any, Optional, Tuple, Union +from typing import Any, Optional, Tuple, Union, List import numpy as onp import numpy.typing as onpt @@ -364,7 +364,7 @@ class GuiAddTabGroupMessage(Message): id: str container_id: str tab_labels: Tuple[str, ...] - tab_icons_base64: Tuple[Union[str, None], ...] + tab_icons: Tuple[Union[str, None], ...] tab_container_ids: Tuple[str, ...] @@ -415,7 +415,7 @@ class GuiAddButtonMessage(_GuiAddInputBase): "teal", ] ] - icon_base64: Optional[str] + icon: Optional[str] @dataclasses.dataclass @@ -427,6 +427,25 @@ class GuiAddSliderMessage(_GuiAddInputBase): precision: int +@dataclasses.dataclass +class GuiSliderMark: + value: float + label: Optional[str] = None + + +@dataclasses.dataclass +class GuiAddMultiSliderMessage(_GuiAddInputBase): + min: float + max: float + step: Optional[float] + min_range: Optional[float] + initial_value: List[float] + precision: int + fixed_endpoints: bool = False + marks: Optional[List[GuiSliderMark]] = None + + + @dataclasses.dataclass class GuiAddNumberMessage(_GuiAddInputBase): initial_value: float @@ -434,6 +453,7 @@ class GuiAddNumberMessage(_GuiAddInputBase): step: float min: Optional[float] max: Optional[float] + icon: Optional[str] @dataclasses.dataclass @@ -486,6 +506,17 @@ class GuiAddButtonGroupMessage(_GuiAddInputBase): options: Tuple[str, ...] +@dataclasses.dataclass +class GuiAddCameraTrajectoryPanelMessage(Message): + order: float + id: str + container_id: str + + fov: float = 75. + render_width: int = 1920 + render_height: int = 1080 + + @dataclasses.dataclass class GuiRemoveMessage(Message): """Sent server->client to remove a GUI element.""" diff --git a/src/viser/client/src/App.tsx b/src/viser/client/src/App.tsx index 02de0b335..40f477199 100644 --- a/src/viser/client/src/App.tsx +++ b/src/viser/client/src/App.tsx @@ -50,6 +50,7 @@ import { GetRenderRequestMessage, Message } from "./WebsocketMessages"; import { makeThrottledMessageSender } from "./WebsocketFunctions"; import { useDisclosure } from "@mantine/hooks"; import { computeR_threeworld_world } from "./WorldTransformUtils"; +import { stat } from "fs"; export type ViewerContextContents = { // Zustand hooks. @@ -72,6 +73,7 @@ export type ViewerContextContents = { wxyz?: [number, number, number, number]; position?: [number, number, number]; visibility?: boolean; + renderModeVisibility?: boolean; }; }>; messageQueueRef: React.MutableRefObject; @@ -81,6 +83,8 @@ export type ViewerContextContents = { >; getRenderRequest: React.MutableRefObject; sceneClickEnable: React.MutableRefObject; + setIsRenderMode: (isRenderMode: boolean) => void; + viewerModeCameraStateBackup: React.MutableRefObject; }; export const ViewerContext = React.createContext( null, @@ -127,6 +131,32 @@ function ViewerRoot() { getRenderRequestState: React.useRef("ready"), getRenderRequest: React.useRef(null), sceneClickEnable: React.useRef(false), + viewerModeCameraStateBackup: React.useRef(null), + setIsRenderMode: (isRenderMode: boolean) => { + viewer.useGui.setState((state) => { + state.isRenderMode = isRenderMode; + const camera = viewer.cameraRef.current!; + // Backup/restore camera state + if (isRenderMode) { + viewer.viewerModeCameraStateBackup.current = [ + [...camera.position.toArray()], + viewer.cameraControlRef.current!.toJSON() + ]; + } else if (viewer.viewerModeCameraStateBackup.current !== null) { + const [cameraData, cameraControl] = viewer.viewerModeCameraStateBackup.current!; + const position = new THREE.Vector3(...cameraData.slice(0, 3)); + const rotation = new THREE.Euler(...cameraData.slice(3, 6)); + //viewer.cameraControlRef.current?.fromJSON(cameraControl, false); + camera.position.copy(position); + camera.rotation.copy(rotation); + camera.updateProjectionMatrix(); + //viewer.cameraControlRef.current?.updateCameraUp(); + //viewer.cameraControlRef.current?.update(1); + viewer.viewerModeCameraStateBackup.current = null; + } + //console.log(camera.up); + }); + } }; return ( @@ -187,7 +217,7 @@ function ViewerContents() { - {viewer.useGui((state) => state.theme.show_logo) ? ( + {viewer.useGui((state) => state.theme.show_logo && !state.isRenderMode) ? ( ) : null} @@ -204,6 +234,7 @@ function ViewerCanvas({ children }: { children: React.ReactNode }) { viewer.websocketRef, 20, ); + const isRenderMode = viewer.useGui((state) => state.isRenderMode); return ( state.camera as PerspectiveCamera); + const isRenderMode = viewer.useGui((state) => state.isRenderMode); const sendCameraThrottled = makeThrottledMessageSender( viewer.websocketRef, @@ -198,6 +199,7 @@ export function SynchronizedCameraControls() { return ( void): void { + const inputElemenet = document.createElement('input'); + inputElemenet.style.display = 'none'; + inputElemenet.type = 'file'; + + inputElemenet.addEventListener('change', () => { + if (inputElemenet.files) { + onFilePicked(inputElemenet.files[0]); + } + }); + + const teardown = () => { + document.body.removeEventListener('focus', teardown, true); + setTimeout(() => { + document.body.removeChild(inputElemenet); + }, 1000); + } + document.body.addEventListener('focus', teardown, true); + + document.body.appendChild(inputElemenet); + inputElemenet.click(); +} + + +export interface CameraTrajectoryPanelProps { + visible: boolean +} + +export interface Camera { + time: number + name: string + fov: number + position: [number, number, number] + wxyz: [number, number, number, number] +} + +function getCameraHash({ fov, position, wxyz }: { fov: number, position: [number, number, number], wxyz: [number, number, number, number] }) { + const data = [fov, ...position, ...wxyz]; + const seed = 0; + let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed; + for(let i = 0, ch; i < data.length; i++) { + h1 = Math.imul(h1 ^ data[i], 2654435761); + h2 = Math.imul(h2 ^ data[i], 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return 4294967296 * (2097151 & h2) + (h1 >>> 0); +} + +function mapNumberToAlphabet(number: number) : string { + let out = "" + const n = 10 + 'z'.charCodeAt(0) - 'a'.charCodeAt(0) + 1; + while (number > 0) { + const current = number % n; + out += (current < 10) ? (current).toString() : String.fromCharCode('a'.charCodeAt(0) + current - 10); + number = Math.floor(number / n); + } + return out; +} + +function rescaleCameras(cameras: Camera[]) : Camera[] { + if (cameras.length === 0) return []; + if (cameras.length == 1) return [{...cameras[0], time: 0}]; + const min = Math.min(...cameras.map(x => x.time)); + const max = Math.max(...cameras.map(x => x.time)); + return cameras.map(x => ({...x, time: (x.time - min) / (max - min)})); +} + + +function getPoseFromCamera(viewer: ViewerContextContents) { + const three_camera = viewer.cameraRef.current!; + const R_threecam_cam = new THREE.Quaternion().setFromEuler( + new THREE.Euler(Math.PI, 0.0, 0.0), + ); + const R_world_threeworld = getR_threeworld_world(viewer).invert(); + const R_world_camera = R_world_threeworld.clone() + .multiply(three_camera.quaternion); + //.multiply(R_threecam_cam); + + return { + wxyz: [ + R_world_camera.w, + R_world_camera.x, + R_world_camera.y, + R_world_camera.z, + ] as [number, number, number, number], + position: three_camera.position + .clone() + .applyQuaternion(R_world_threeworld) + .toArray(), + aspect: three_camera.aspect, + fov: (three_camera.fov * Math.PI) / 180.0, + }; +} + + +export default function CameraTrajectoryPanel(props: CameraTrajectoryPanelProps) { + if (!props.visible) { + return null; + } + + const viewer = React.useContext(ViewerContext)!; + const R_threeworld_world = getR_threeworld_world(viewer); + const removeSceneNode = viewer.useSceneTree((state) => state.removeSceneNode); + const addSceneNode = viewer.useSceneTree((state) => state.addSceneNode); + const nodeFromName = viewer.useSceneTree((state) => state.nodeFromName); + const isRenderMode = viewer.useGui((state) => state.isRenderMode); + + // Get default FOV from camera + const [isCycle, setIsCycle] = React.useState(false); + const [isPlaying, setIsPlaying] = React.useState(false); + const [fps, setFps] = React.useState(24); + const [smoothness, setSmoothness] = React.useState(0.5); + const [cameras, setCameras] = React.useState([]); + const [fov, setFov] = React.useState(viewer.cameraRef.current?.fov ?? 75.0); + const [renderWidth, setRenderWidth] = React.useState(1920); + const [renderHeight, setRenderHeight] = React.useState(1080); + const [playerTime, setPlayerTime] = React.useState(0.); + const aspect = renderWidth / renderHeight; + + const baseTreeName = "CameraTrajectory" + const ensureThreeRootExists = () => { + if (!(baseTreeName in nodeFromName)) { + addSceneNode( + new SceneNode(baseTreeName, (ref) => ( + + )) as SceneNode, + ); + } + if (!(`${baseTreeName}/PlayerCamera` in nodeFromName)) { + addSceneNode( + new SceneNode( + `${baseTreeName}/PlayerCamera`, + (ref) => ( + + ), + ) as SceneNode); + } + const attr = viewer.nodeAttributesFromName.current; + if (attr[`${baseTreeName}/PlayerCamera`] === undefined) attr[`${baseTreeName}/PlayerCamera`] = {}; + if (attr[baseTreeName] === undefined) attr[baseTreeName] = {}; + attr[`${baseTreeName}/PlayerCamera`]!.visibility = false; + attr[`${baseTreeName}/PlayerCamera`]!.renderModeVisibility = false; + attr[baseTreeName]!.renderModeVisibility = false; + } + React.useEffect(() => { + ensureThreeRootExists(); + return () => { + const attr = viewer.nodeAttributesFromName.current; + `${baseTreeName}/PlayerCamera` in nodeFromName && removeSceneNode(`${baseTreeName}/PlayerCamera`); + baseTreeName in nodeFromName && removeSceneNode(baseTreeName); + `${baseTreeName}/PlayerCamera` in attr && delete attr[`${baseTreeName}/PlayerCamera`]; + baseTreeName in attr && delete attr[baseTreeName]; + } + }, []); + + const curveObject = React.useMemo(() => cameras.length > 1 ? get_curve_object_from_cameras( + cameras.map(({fov, wxyz, position, time}: Camera) => ({ + time, + fov, + position: new THREE.Vector3(...position), + quaternion: new THREE.Quaternion(wxyz[1], wxyz[2], wxyz[3], wxyz[0]), + })), isCycle, smoothness) : null, [cameras, isCycle, smoothness]); + + // Update cameras and trajectory + React.useEffect(() => { + ensureThreeRootExists(); + // Update trajectory + if (!(baseTreeName in nodeFromName)) return; + const children = nodeFromName[baseTreeName]!.children; + const enabledCameras = new Set(cameras.map(x => `${baseTreeName}/Camera.${x.name}`)); + children.filter((c) => !enabledCameras.has(c)).forEach((c) => { + removeSceneNode(c); + const attr = viewer.nodeAttributesFromName.current; + if (attr[c] !== undefined) delete attr[c]; + }); + cameras.forEach((camera, index) => { + const nodeName = `${baseTreeName}/Camera.${camera.name}`; + if (!(nodeName in nodeFromName)) { + const node: SceneNode = new SceneNode( + `${baseTreeName}/Camera.${camera.name}`, + (ref) => ( + + ), + ); + addSceneNode(node); + } + const attr = viewer.nodeAttributesFromName.current; + if (attr[nodeName] === undefined) attr[nodeName] = {}; + attr[nodeName]!.wxyz = camera.wxyz; + attr[nodeName]!.position = camera.position; + attr[nodeName]!.renderModeVisibility = false; + }); + }, [cameras, aspect, smoothness]); + + + // Render camera path + React.useEffect(() => { + ensureThreeRootExists(); + const nodeName = `${baseTreeName}/Trajectory`; + if (curveObject !== null) { + const num_points = fps * seconds; + const points = curveObject.curve_positions.getPoints(num_points); + const resolution = new THREE.Vector2( window.innerWidth, window.innerHeight ); + const geometry = new MeshLineGeometry(); + geometry.setPoints(points); + const material = new MeshLineMaterial({ + lineWidth: 0.03, + color: 0xff5024, + resolution, + }); + addSceneNode(new SceneNode(nodeName, (ref) => { + return + }, () => { + geometry.dispose(); + material.dispose(); + }) as SceneNode); + + const attr = viewer.nodeAttributesFromName.current; + if (attr[nodeName] === undefined) attr[nodeName] = {}; + attr[nodeName]!.renderModeVisibility = false; + } else if (nodeName in nodeFromName) { + removeSceneNode(nodeName); + } + }, [curveObject, fps, isRenderMode]); + + React.useEffect(() => { + ensureThreeRootExists(); + // set the camera + if (curveObject !== null) { + if (isRenderMode) { + const point = getKeyframePoint(playerTime); + const position = curveObject.curve_positions.getPoint(point).applyQuaternion(R_threeworld_world); + const lookat = curveObject.curve_lookats.getPoint(point).applyQuaternion(R_threeworld_world); + const up = curveObject.curve_ups.getPoint(point).multiplyScalar(-1).applyQuaternion(R_threeworld_world); + const fov = curveObject.curve_fovs.getPoint(point).z; + + // const cameraControls = viewer.cameraControlRef.current!; + const threeCamera = viewer.cameraRef.current!; + + threeCamera.position.set(...position.toArray()); + threeCamera.up.set(...up.toArray()); + threeCamera.lookAt(...lookat.toArray()); + // cameraControls.updateCameraUp(); + // cameraControls.setLookAt(...position.toArray(), ...lookat.toArray(), false); + // const target = position.clone().add(lookat); + // NOTE: lookat is being ignored when calling setLookAt + // cameraControls.setTarget(...target.toArray(), false); + threeCamera.setFocalLength( + (0.5 * threeCamera.getFilmHeight()) / Math.tan(fov / 2.0), + ); + // cameraControls.update(1.); + } else { + const point = getKeyframePoint(playerTime); + const position = curveObject.curve_positions.getPoint(point); + const lookat = curveObject.curve_lookats.getPoint(point); + const up = curveObject.curve_ups.getPoint(point); + + const mat = get_transform_matrix(position, lookat, up); + const quaternion = new THREE.Quaternion().setFromRotationMatrix(mat); + + addSceneNode( + new SceneNode( + `${baseTreeName}/PlayerCamera`, + (ref) => ( + + ), + ) as SceneNode); + const attr = viewer.nodeAttributesFromName.current; + if (attr[`${baseTreeName}/PlayerCamera`] === undefined) attr[`${baseTreeName}/PlayerCamera`] = {}; + attr[`${baseTreeName}/PlayerCamera`]!.visibility = true; + attr[`${baseTreeName}/PlayerCamera`]!.wxyz = [quaternion.w, quaternion.x, quaternion.y, quaternion.z]; + attr[`${baseTreeName}/PlayerCamera`]!.position = position.toArray(); + attr[`${baseTreeName}/PlayerCamera`]!.renderModeVisibility = false; + } + } else { + const attr = viewer.nodeAttributesFromName.current; + if (attr[`${baseTreeName}/PlayerCamera`] !== undefined) + attr[`${baseTreeName}/PlayerCamera`]!.visibility = false; + } + }, [curveObject, fps, isRenderMode, playerTime]); + + const [seconds, setSeconds] = React.useState(4); + + React.useEffect(() => { + if (isPlaying && cameras.length > 1) { + const interval = setInterval(() => { + setPlayerTime((prev) => { + let out = Math.min(1., prev + 1 / (fps * seconds)) + if (out >= 1) { + setIsPlaying(false); + out = 0; + } + return out; + }); + }, 1000 / fps); + return () => clearInterval(interval); + } + }, [isPlaying, seconds, fps]); + + + const addCamera = () => { + setCameras((cameras) => { + const { position, wxyz } = getPoseFromCamera(viewer); + const hash = getCameraHash({ fov, position, wxyz }); + let name = `${mapNumberToAlphabet(hash).slice(0, 6)}`; + const nameNumber = cameras.filter(x => x.name.startsWith(name)).length; + if (nameNumber > 0) { + name += `-${nameNumber+1}`; + } + if (cameras.length >= 2) { + const mult = 1 - 1/cameras.length; + return [...cameras.map(x => ({...x, time: x.time * mult})), { + time: 1, + name, + position, + wxyz, + fov, + }]; + } else { + return [...cameras, { + time: cameras.length === 0 ? 0 : 1, + name, + position, + wxyz, + fov, + }]; + } + }); + } + + + const displayRenderTime = false; + + + const getKeyframePoint = (progress: number) => { + const times = []; + const ratio = (cameras.length - 1) / cameras.length; + cameras.forEach((camera) => { + const time = camera.time; + times.push(isCycle ? time * ratio : time); + }); + + if (isCycle) { + times.push(1.0); + } + + let new_point = 0.0; + if (progress <= times[0]) { + new_point = 0.0; + } else if (progress >= times[times.length - 1]) { + new_point = 1.0; + } else { + let i = 0; + while ( + i < times.length - 1 && + !(progress >= times[i] && progress < times[i + 1]) + ) { + i += 1; + } + const percentage = (progress - times[i]) / (times[i + 1] - times[i]); + new_point = (i + percentage) / (times.length - 1); + } + return new_point; + }; + + + const loadCameraPath = (camera_path_object: any) => { + const new_camera_list = []; + + setRenderHeight(camera_path_object.render_height); + setRenderWidth(camera_path_object.render_width); + if (camera_path_object.camera_type !== "perspective") { + // TODO: handle this better! + throw new Error("Unsupported camera type: " + camera_path_object.camera_type); + } + + setFps(camera_path_object.fps); + setSeconds(camera_path_object.seconds); + + setSmoothness(camera_path_object.smoothness_value); + setIsCycle(camera_path_object.is_cycle); + + for (let i = 0; i < camera_path_object.keyframes.length; i += 1) { + const keyframe = camera_path_object.keyframes[i]; + + // properties + const properties = new Map(JSON.parse(keyframe.properties)); + const mat = new THREE.Matrix4(); + mat.fromArray(JSON.parse(keyframe.matrix)); + const position = new THREE.Vector3(); + const quaternion = new THREE.Quaternion(); + const scale = new THREE.Vector3(); + mat.decompose(position, quaternion, scale); + + // aspect = keyframe.aspect; + const camera: Camera = { + position: position.toArray(), + wxyz: [quaternion.w, quaternion.x, quaternion.y, quaternion.z], + name: properties.get("NAME") as string, + time: properties.get("TIME") as number, + fov: keyframe.fov, + }; + new_camera_list.push(camera); + } + setCameras(new_camera_list); + } + + + const getCameraPath = () => { + const num_points = Math.round(fps * seconds); + const camera_path = []; + + for (let i = 0; i < num_points; i += 1) { + const pt = getKeyframePoint(i / num_points); + + const position = curveObject!.curve_positions.getPoint(pt); + const lookat = curveObject!.curve_lookats.getPoint(pt); + const up = curveObject!.curve_ups.getPoint(pt); + const fov = curveObject!.curve_fovs.getPoint(pt).z; + + const mat = get_transform_matrix(position, lookat, up); + + if (displayRenderTime) { + const renderTime = curveObject!.curve_render_times.getPoint(pt).z; + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: aspect, + render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] + }); + } else { + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: aspect, + }); + } + } + + const keyframes = []; + for (let i = 0; i < cameras.length; i += 1) { + const camera = cameras[i]; + + const up = new THREE.Vector3(0, 1, 0); // y is up in local space + const lookat = new THREE.Vector3(0, 0, 1); // z is forward in local space + const wxyz = camera.wxyz + const quaternion = new THREE.Quaternion(wxyz[1], wxyz[2], wxyz[3], wxyz[0]); + + up.applyQuaternion(quaternion); + lookat.applyQuaternion(quaternion); + + const matrix = get_transform_matrix(new THREE.Vector3(...camera.position), lookat, up); + keyframes.push({ + matrix: JSON.stringify(matrix.toArray()), + fov: camera.fov, + aspect: aspect, + properties: JSON.stringify([ + ["FOV", camera.fov], + ["NAME", camera.name], + ["TIME", camera.time], + ]), + }); + } + + // const myData + const camera_path_object = { + format: "nerfstudio-viewer", + keyframes, + camera_type: "perspective", + render_height: renderHeight, + render_width: renderWidth, + camera_path, + fps, + seconds, + smoothness_value: smoothness, + is_cycle: isCycle, + crop: null, + }; + return camera_path_object; + } + + + const exportCameraPath = () => { + const cameraPath = getCameraPath(); + + // create file in browser + const json = JSON.stringify(cameraPath, null, 2); + const blob = new Blob([json], { type: 'application/json' }); + const href = URL.createObjectURL(blob); + + // create "a" HTLM element with href to file + const link = document.createElement('a'); + link.href = href; + + const filename = 'camera_path.json'; + link.download = filename; + document.body.appendChild(link); + link.click(); + // clean up "a" element & remove ObjectURL + document.body.removeChild(link); + URL.revokeObjectURL(href); + } + + const uploadCameraPath = () => { + pickFile((file) => { + const fr = new FileReader(); + fr.onload = (res) => { + const camera_path_object = JSON.parse(res.target!.result! as string); + loadCameraPath(camera_path_object); + }; + fr.readAsText(file); + }); + } + + const marks = []; + for (let i = 0; i <= 1; i += 0.25) { + marks.push({ value: i, label: `${(seconds * i).toFixed(1).toString()}s` }); + } + + // const table = useMantineReactTable({ + // data: cameras, + // enableBottomToolbar: false, + // enableTopToolbar: false, + // enableSorting: false, + // enableColumnActions: false, + // enableDensityToggle: false, + // enableRowSelection: false, + // enableRowOrdering: true, + // enableHiding: false, + // enableColumnFilters: false, + // enablePagination: false, + // mantineTableProps: { + // verticalSpacing: 2, + // }, + // enableRowActions: true, + // mantinePaperProps: { shadow: undefined }, + // mantineTableContainerProps: { sx: { maxHeight: "30em" } }, + // mantinePaginationProps: { + // showRowsPerPage: false, + // }, + // displayColumnDefOptions: { + // 'mrt-row-drag': { + // header: "", + // minSize: 0, + // size: 10, + // maxSize: 10, + // }, + // 'mrt-row-actions': { + // header: "", + // minSize: 0, + // size: 10, + // maxSize: 10, + // }, + // }, + // renderRowActions: (row) => { + // return { + // const cameraId = row.row.original.name; + // setCameras(rescaleCameras(cameras.filter(x => x.name !== cameraId))); + // }}> + // + // + // }, + // // mantineTableBodyRowProps={({ row }) => ({ + // // onPointerOver: () => { + // // setLabelVisibility(row.getValue("name"), true); + // // }, + // // onPointerOut: () => { + // // setLabelVisibility(row.getValue("name"), false);/ + // // }, + // // ...(row.subRows === undefined || row.subRows.length === 0 + // // ? {} + // // : { + // // onClick: () => { + // // row.toggleExpanded(); + // // }, + // // sx: { + // // cursor: "pointer", + // // }, + // // }), + // // })} + // mantineRowDragHandleProps: ({table}) => ({ + // onDragEnd: () => { + // const { draggingRow, hoveredRow } = table.getState(); + // if (hoveredRow && draggingRow) { + // const clone = [...cameras]; + // clone.splice(hoveredRow.index, 0, clone.splice(draggingRow.index, 1)[0]); + // } + // }, + // }), + // columns: [{ + // header: "Time", + // minSize: 10, + // size: 30, + // accessorFn(originalRow) { + // return (originalRow.time * seconds).toFixed(2).toString() + "s"; + // }, + // }, { + // size: 30, + // minSize: 10, + // header: "Camera", + // accessorKey: "name", + // }], + // initialState: { + // density: "xs", + // } + // }); + + return (<> + + + + + + } + step={1} + size="xs" + onChange={(newValue) => newValue !== "" && setFps(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + } + step={1.0} + size="xs" + onChange={(newValue) => newValue !== "" && setFov(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + + + } + step={100} + size="xs" + onChange={(newValue) => newValue !== "" && setRenderWidth(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + } + step={100} + size="xs" + onChange={(newValue) => newValue !== "" && setRenderHeight(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + + } + step={1.00} + size="xs" + onChange={(newValue) => newValue !== "" && setSeconds(newValue)} + styles={{ + input: { + minHeight: "1.625rem", + height: "1.625rem", + }, + }} + stepHoldDelay={500} + stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} + /> + Smoothness + + + Timeline + `${(seconds*x).toFixed(2)}s`} + value={cameras.map(x=>x.time)} + onChange={(value) => { + setCameras(cameras.map((camera, index) => ({ + ...camera, + time: value[index], + }))); + }} + onChangeEnd={(value) => { + setCameras(cameras.map((camera, index) => ({ + ...camera, + time: value[index], + }))); + }} + marks={marks} /> + + + + + + + + + + + {cameras.map((camera, index) => { + return ( + + + + + + ) + })} + +
TimeCamera
+ + { + setCameras(rescaleCameras([...cameras.slice(0, index), ...cameras.slice(index + 1)])); + }}> + + + { + const clone = [...cameras]; + const tmp = cameras[index]; + clone[index] = { + ...cameras[index - 1], + time: cameras[index].time + }; + clone[index - 1] = { + ...tmp, + time: cameras[index - 1].time + } + setCameras(clone); + }}> + + + { + const clone = [...cameras]; + const tmp = cameras[index]; + clone[index] = { + ...cameras[index + 1], + time: cameras[index].time + }; + clone[index + 1] = { + ...tmp, + time: cameras[index + 1].time + } + setCameras(clone); + }}> + + + + {(seconds * camera.time).toFixed(2).toString()}s{camera.name}
+ + + Player + + + setPlayerTime(Math.max(0, ...cameras.map(x => x.time).filter(x => x < playerTime)))}> + + + {isPlaying ? setIsPlaying(false)}> + + : setIsPlaying(true)}> + + } + setPlayerTime(Math.min(1, ...cameras.map(x => x.time).filter(x => x > playerTime)))}> + + + + { + viewer.setIsRenderMode(event.currentTarget.checked); + }} + size="sm" + /> + ) +} + +CameraTrajectoryPanel.defaultProps = { + visible: true, +}; \ No newline at end of file diff --git a/src/viser/client/src/ControlPanel/ControlPanel.tsx b/src/viser/client/src/ControlPanel/ControlPanel.tsx index e2223be51..04b527733 100644 --- a/src/viser/client/src/ControlPanel/ControlPanel.tsx +++ b/src/viser/client/src/ControlPanel/ControlPanel.tsx @@ -21,6 +21,7 @@ import BottomPanel from "./BottomPanel"; import FloatingPanel from "./FloatingPanel"; import { ThemeConfigurationMessage } from "../WebsocketMessages"; import SidebarPanel from "./SidebarPanel"; +import CameraTrajectoryPanel from "./CameraTrajectoryPanel"; // Must match constant in Python. const ROOT_CONTAINER_ID = "root"; @@ -77,6 +78,7 @@ export default function ControlPanel(props: { const panelContents = ( <> + diff --git a/src/viser/client/src/ControlPanel/Generated.tsx b/src/viser/client/src/ControlPanel/Generated.tsx index 093aa46bb..6852d905e 100644 --- a/src/viser/client/src/ControlPanel/Generated.tsx +++ b/src/viser/client/src/ControlPanel/Generated.tsx @@ -7,7 +7,7 @@ import { makeThrottledMessageSender } from "../WebsocketFunctions"; import { computeRelativeLuminance } from "./GuiState"; import { Collapse, - Image, + Group, Paper, Tabs, TabsValue, @@ -31,7 +31,18 @@ import React from "react"; import Markdown from "../Markdown"; import { ErrorBoundary } from "react-error-boundary"; import { useDisclosure } from "@mantine/hooks"; -import { IconChevronDown, IconChevronUp } from "@tabler/icons-react"; +import { IconChevronDown, IconChevronUp, TablerIconsProps } from "@tabler/icons-react"; +import CameraTrajectoryPanel from "./CameraTrajectoryPanel"; +import { MultiSlider } from "./MultiSlider"; +import * as TablerIcons from '@tabler/icons-react'; + + +function createIcon(icon: string) : (props: TablerIconsProps) => JSX.Element { + // Icon name is in snake-case + // We need to convert it to PascalCase + const iconPascal = icon.split('-').map((word) => word.charAt(0).toUpperCase() + word.slice(1)).join(''); + return (TablerIcons as any)[("Icon" + iconPascal) as any] as unknown as (props: TablerIconsProps) => JSX.Element; +} /** Root of generated inputs. */ export default function GeneratedGuiContainer({ @@ -120,6 +131,11 @@ function GeneratedInput({ ); } + if (conf.type == "GuiAddCameraTrajectoryPanelMessage") { + return ( + + ); + } const messageSender = makeThrottledMessageSender(viewer.websocketRef, 50); function updateValue(value: any) { @@ -148,6 +164,8 @@ function GeneratedInput({ let labeled = true; let input = null; + const iconString = (conf as { icon?: string }).icon; + const Icon = iconString && createIcon(iconString); switch (conf.type) { case "GuiAddButtonMessage": labeled = false; @@ -176,27 +194,7 @@ function GeneratedInput({ styles={{ inner: { color: inputColor + " !important" } }} disabled={disabled} size="sm" - leftIcon={ - conf.icon_base64 === null ? undefined : ( - - ) - } + leftIcon={Icon && } > {conf.label} @@ -288,6 +286,7 @@ function GeneratedInput({ height: "1.625rem", }, }} + icon={Icon && } disabled={disabled} stepHoldDelay={500} stepHoldInterval={(t) => Math.max(1000 / t ** 2, 25)} @@ -424,7 +423,7 @@ function GeneratedInput({ break; case "GuiAddButtonGroupMessage": input = ( - + {conf.options.map((option, index) => (