add nerf viewer

viewers/nerf_viewer/README.md

@@ -0,0 +1,85 @@
+# Nerf Viewer Quick Start
+
+## 1 Client
+
+### 1.1 Install Dependencies
+
+#### 1.1.1 Install NVM
+
+Please follow the instructions in:
+
+https://juejin.cn/post/7120267871950733320/
+
+#### 1.1.2 Install Node
+
+Once you have nvm correctly configured, install node 18.15.0 using the commands below:
+
+```shell
+# install node 18.15.0
+nvm install 18.15.0
+
+# activate node 18.15.0
+nvm use 18.15.0
+```
+
+#### 1.1.3 Install Node Packages
+
+
+```shell
+# make sure that your current working directory is 'client'
+cd ./client
+
+# install node packages using configuration in client/package.json
+npm install
+```
+
+### 1.2 Start Viewer
+
+Build and run the viewer using:
+
+```shell
+# make sure that your current working directory is 'client'
+cd ./client
+
+# start the viewer
+npm start
+```
+
+Then, check whether viewer is successfully compiled:
+
+![alt viewer_compile_success](./doc/viewer_compile_success.png)
+
+Visit http://localhost:3000/ in the browser:
+
+![alt viewer](./doc/viewer.png)
+
+## 2 Bridge Server
+
+### 2.1 Install Dependencies
+
+```shell
+
+cd ./bridge_server
+
+# create a virtual environment
+conda create -n BridgeServer python=3.7
+
+conda activate BridgeServer
+
+# install dependencies
+pip install -r ./requirements.txt
+```
+
+#### 2.2 Start Server
+
+```
+python ./run_viewer.py
+```
+
+Check whether server is successfully deployed:
+
+![alt run_viewer](./doc/run_viewer.png)
+
+Open the browser and visit http://localhost:3000/. Then, check whether the server is connected with the viewer:
+
+![alt viewer_connected](./doc/viewer_connected.png)

viewers/nerf_viewer/bridge_server/.gitignore

@@ -0,0 +1 @@
+.idea

viewers/nerf_viewer/bridge_server/actions.py

@@ -0,0 +1,23 @@
+"""
+Actions are a series of keys that determine how data flows
+"""
+
+"""for viewer"""
+
+UPDATE_CAMERA_TRANSLATION = "UPDATE_CAMERA_TRANSLATION"
+
+UPDATE_CAMERA_ROTATION = "UPDATE_CAMERA_ROTATION"
+
+UPDATE_CAMERA_FOV = "UPDATE_CAMERA_FOV"
+
+UPDATE_RESOLUTION = "UPDATE_RESOLUTION"
+
+UPDATE_RENDER_TYPE = "UPDATE_RENDER_TYPE"
+
+
+"""for nerf backend"""
+
+# update the scene state
+UPDATE_STATE = "UPDATE_STATE"
+
+UPDATE_RENDER_RESULT = "UPDATE_RENDER_RESULT"

viewers/nerf_viewer/bridge_server/bridge_server_subprocess.py

@@ -0,0 +1,85 @@
+import atexit
+import os
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+
+import server
+
+from typing import Optional
+
+
+def is_port_open(port: int):
+    # check whether the given port is open
+    try:
+        sock = socket.socket()
+        sock.bind(("", port))
+        sock.close()
+
+        return True
+    except OSError:
+        return False
+
+
+def get_free_port(default_port: int = None):
+    if default_port:
+        if is_port_open(default_port):
+            return default_port
+    sock = socket.socket()
+    sock.bind(("", 0))
+    port = sock.getsockname()[1]
+
+    return port
+
+
+def run_bridge_server_as_subprocess(
+    websocket_port: int,
+    zmq_port: Optional[int] = None,
+    ip_address: str = "127.0.0.1",
+):
+    # run bridge server as a sub-process
+    args = [sys.executable, "-u", "-m", server.__name__]
+
+    # find an available port for zmq
+    if zmq_port is None:
+        zmq_port = get_free_port()
+        print(f"Using ZMQ port: {zmq_port}")
+
+    args.append("--zmq-port")
+    args.append(str(zmq_port))
+    args.append("--websocket-port")
+    args.append(str(websocket_port))
+    args.append("--ip-address")
+    args.append(str(ip_address))
+
+    process = subprocess.Popen(args, start_new_session=True)
+
+    def cleanup(process):
+        process.kill()
+        process.wait()
+
+    def poll_process():
+        """
+        Continually check to see if the viewer bridge server process is still running and has not failed.
+        If it fails, alert the user and exit the entire program.
+        """
+        while process.poll() is None:
+            time.sleep(0.5)
+
+        message = "The bridge server subprocess failed."
+        cleanup(process)
+
+        # windows system do not have signal.SIGKILL
+        # os.kill(os.getpid(), signal.SIGKILL)
+        os.kill(os.getpid(), signal.SIGINT)
+
+    watcher_thread = threading.Thread(target=poll_process)
+    watcher_thread.daemon = True
+    watcher_thread.start()
+    # clean up process when it has shut down
+    atexit.register(cleanup, process)
+
+    return zmq_port

viewers/nerf_viewer/bridge_server/requirements.txt

@@ -0,0 +1,5 @@
+pyzmq==25.0.2
+pyngrok==5.2.1
+tornado
+umsgpack==0.1.0
+opencv-contrib-python==4.5.3.56

viewers/nerf_viewer/bridge_server/run_viewer.py

@@ -0,0 +1,14 @@
+from viewer_state import ViewerState
+
+
+def run_viewer():
+    """
+    start the viewer
+    """
+    viewer_state = ViewerState()
+    while True:
+        viewer_state.update_scene()
+
+
+if __name__ == "__main__":
+    run_viewer()

viewers/nerf_viewer/bridge_server/server.py

@@ -0,0 +1,173 @@
+import pickle
+import sys
+from typing import List, Optional, Tuple
+
+# for web networking
+import tornado.gen
+import tornado.ioloop
+import tornado.web
+import tornado.websocket
+
+# for data conversion
+import umsgpack
+
+# ZeroMQ: for messaging
+import zmq
+import zmq.eventloop.ioloop
+from zmq.eventloop.zmqstream import ZMQStream
+
+from pyngrok import ngrok
+
+from actions import \
+    UPDATE_CAMERA_FOV, UPDATE_CAMERA_ROTATION, UPDATE_CAMERA_TRANSLATION, \
+    UPDATE_RENDER_TYPE, UPDATE_RESOLUTION, UPDATE_STATE, UPDATE_RENDER_RESULT
+
+from state import State
+
+
+class WebSocketHandler(tornado.websocket.WebSocketHandler):  # pylint: disable=abstract-method
+    """for receiving and sending commands from/to the viewer"""
+
+    def __init__(self, *args, **kwargs):
+        self.bridge = kwargs.pop("bridge")
+        super().__init__(*args, **kwargs)
+
+    def check_origin(self, origin: str) -> bool:
+        return True
+
+    def open(self, *args: str, **kwargs: str):
+        self.bridge.websocket_pool.add(self)
+        print("opened:", self, file=sys.stderr)
+
+    async def on_message(self, message: bytearray): # pylint: disable=invalid-overridden-method
+        """parses the message from viewer and calls the appropriate function"""
+        data = message
+        unpacked_message = umsgpack.unpackb(message)
+
+        type = unpacked_message["type"]
+        data = unpacked_message["data"]
+        # type_ = m["type"]
+        # path = list(filter(lambda x: len(x) > 0, m["path"].split("/")))
+
+        if type == UPDATE_CAMERA_TRANSLATION:
+            self.bridge.state.camera_translation = data
+        elif type == UPDATE_CAMERA_ROTATION:
+            self.bridge.state.camera_rotation = data
+        elif type == UPDATE_RENDER_TYPE:
+            self.bridge.state.render_type = data
+        elif type == UPDATE_CAMERA_FOV:
+            self.bridge.state.camera_fov = data
+        elif type == UPDATE_RESOLUTION:
+            self.bridge.state.resolution = data
+        else:
+            # TODO: handle exception
+            pass
+
+    def on_close(self) -> None:
+        self.bridge.websocket_pool.remove(self)
+        print("closed: ", self, file=sys.stderr)
+
+
+class ZMQWebSocketBridge:
+
+    context = zmq.Context()  # pylint: disable=abstract-class-instantiated
+
+    def __init__(self, zmq_port: int, websocket_port: int, ip_address: str):
+        self.zmq_port = zmq_port
+        self.websocket_pool = set()
+        self.app = self.make_app()
+        self.ioloop = tornado.ioloop.IOLoop.current()
+
+        # zmq
+        zmq_url = f"tcp://{ip_address}:{self.zmq_port:d}"
+        self.zmq_socket, self.zmq_stream, self.zmq_url = self.setup_zmq(zmq_url)
+
+        # websocket
+        listen_kwargs = {"address" : "0.0.0.0"}
+        self.app.listen(websocket_port, **listen_kwargs)
+        self.websocket_port = websocket_port
+        self.websocket_url = f"0.0.0.0:{self.websocket_port}"
+
+        # state
+        self.state = State()
+
+    def __str__(self) -> str:
+        class_name = self.__class__.__name__
+        return f'{class_name} using zmq_port="{self.zmq_port}" and websocket_port="{self.websocket_port}"'
+
+    def make_app(self):
+        # create an application for the websocket server
+        return tornado.web.Application([(r"/", WebSocketHandler, {"bridge": self})])
+
+    def handle_zmq(self, frames: List[bytes]):
+
+        type_ = frames[0].decode("utf-8")
+        data = frames[1]
+
+        if type_ == UPDATE_RENDER_RESULT:
+            self.forward_to_websockets(frames)
+            self.zmq_socket.send(umsgpack.packb(b"ok"))
+        elif type_ == UPDATE_STATE:
+            serialized = pickle.dumps(self.state)
+            self.zmq_socket.send(serialized)
+        elif type_ == "ping":
+            self.zmq_socket.send(umsgpack.packb(b"ping received"))
+        else:
+            print("type: " + str(type_))
+            self.zmq_socket.send(umsgpack.packb(b"error: unknown command"))
+
+    def forward_to_websockets(
+        self,
+        frames: Tuple[str, str, bytes],
+        websocket_to_skip: Optional[WebSocketHandler] = None
+    ):
+        """forward a zmq message to all websockets"""
+        """nerf backend -> viewer"""
+        _type, _data = frames  # cmd, data
+        for websocket in self.websocket_pool:
+            if websocket_to_skip and websocket == websocket_to_skip:
+                pass
+            else:
+                websocket.write_message(_data, binary=True)
+
+    def setup_zmq(self, url: str):
+        """setup a zmq socket and connect it to the given url"""
+        zmq_socket = self.context.socket(zmq.REP)  # pylint: disable=no-member
+        zmq_socket.bind(url)
+        zmq_stream = ZMQStream(zmq_socket)
+        zmq_stream.on_recv(self.handle_zmq)
+
+        return zmq_socket, zmq_stream, url
+
+    def run(self):
+        """starts and runs the websocet bridge"""
+        self.ioloop.start()
+
+
+def run_bridge_server(
+        zmq_port: int = 6000,
+        websocket_port: int = 4567,
+        ip_address: str = "127.0.0.1",
+        use_ngrok: bool = False
+):
+    # whether expose the zmq port
+    if use_ngrok:
+        http_tunnel = ngrok.connect(addr=str(zmq_port), proto="tcp")
+        print(http_tunnel)
+
+    bridge = ZMQWebSocketBridge(
+        zmq_port=zmq_port,
+        websocket_port=websocket_port,
+        ip_address=ip_address
+    )
+
+    print(bridge)
+
+    try:
+        bridge.run()
+    except KeyboardInterrupt:
+        pass
+
+
+if __name__ == "__main__":
+    run_bridge_server()