Orsi/sample apps (#163)

Added Orsi sample applications

---------
Signed-off-by: Julien Jomier <[email protected]>
Signed-off-by: Jasper <[email protected]>
Co-authored-by: Oliver Kutter <[email protected]>
Co-authored-by: Julien Jomier <[email protected]>

applications/CMakeLists.txt

@@ -91,3 +91,4 @@ add_holohub_application(volume_rendering_xr DEPENDS
                                   XrTransformOp)
 
 add_holohub_application(webrtc_video_server)
+add_subdirectory(orsi)

applications/orsi/CMakeLists.txt

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.20)
+project(orsi_sample_apps LANGUAGES NONE)
+
+# Check if Deltacast is found
+set(ORSI_VIDEO_MASTER_SDK_FOUND OFF)
+
+if(IS_DIRECTORY "/usr/local/deltacast/Include")
+    if(EXISTS "/usr/lib/libvideomasterhd.so")
+        set(ORSI_VIDEO_MASTER_SDK_FOUND ON)
+    endif()
+endif()
+
+set(HOLOSCAN_OPERATORS
+        orsi_format_converter
+        orsi_segmentation_postprocessor
+        orsi_segmentation_preprocessor
+        orsi_visualizer
+)
+
+if(ORSI_VIDEO_MASTER_SDK_FOUND)
+    list(APPEND HOLOSCAN_OPERATORS  deltacast_videomaster)
+endif()
+
+# Download datasets
+option(HOLOHUB_DOWNLOAD_DATASETS "Download datasets" ON)
+
+# Download ORSI sample data from NGC
+if(HOLOHUB_DOWNLOAD_DATASETS)
+  include(holoscan_download_data)
+  holoscan_download_data(orsi
+    URL  https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_orsi_academy_sample_data/versions/20240206/zip
+    DOWNLOAD_NAME orsi.zip
+    URL_MD5 5cf78e64bd9696f9e04796cb1e04d2ad
+    DOWNLOAD_DIR ${HOLOHUB_DATA_DIR}
+    GENERATE_GXF_ENTITIES
+    GXF_ENTITIES_HEIGHT 1080
+    GXF_ENTITIES_WIDTH 1920
+    GXF_ENTITIES_CHANNELS 3
+    GXF_ENTITIES_FRAMERATE 30
+  )
+endif()
+
+add_subdirectory(lib)
+
+add_holohub_application(orsi_multi_ai_ar DEPENDS
+                        OPERATORS ${HOLOSCAN_OPERATORS})
+
+add_holohub_application(orsi_in_out_body DEPENDS
+                        OPERATORS ${HOLOSCAN_OPERATORS})
+
+add_holohub_application(orsi_segmentation_ar DEPENDS
+                        OPERATORS ${HOLOSCAN_OPERATORS})

applications/orsi/Dockerfile

@@ -0,0 +1,88 @@
+# syntax=docker/dockerfile:1
+
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+############################################################
+# Base image
+############################################################
+
+ARG BASE_IMAGE
+
+FROM ${BASE_IMAGE} as base
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+
+# --------------------------------------------------------------------------
+#
+# Holohub run setup 
+#
+
+RUN mkdir -p /tmp/scripts
+COPY run /tmp/scripts/
+RUN chmod +x /tmp/scripts/run
+RUN /tmp/scripts/run setup
+
+
+# - This variable is consumed by all dependencies below as an environment variable (CMake 3.22+)
+# - We use ARG to only set it at docker build time, so it does not affect cmake builds
+#   performed at docker run time in case users want to use a different BUILD_TYPE
+ARG CMAKE_BUILD_TYPE=Release
+
+
+# --------------------------------------------------------------------------
+#
+# VTK 9.2.2  Build
+#
+
+ARG VTK_VERSION=9.2.6
+
+WORKDIR /opt/vtk
+
+RUN apt update &&  apt install  --no-install-recommends -y libglvnd-dev
+RUN  wget https://www.vtk.org/files/release/9.2/VTK-${VTK_VERSION}.tar.gz -P src && cd src && tar xzvf VTK-${VTK_VERSION}.tar.gz && \
+     mkdir -p /opt/vtk/build && \
+     cd /opt/vtk/build  && \ 
+     cmake  -DBUILD_TESTING:BOOL=OFF \
+            -DCMAKE_INSTALL_PREFIX=/opt/vtk/${VTK_VERSION} \
+            /opt/vtk/src/VTK-${VTK_VERSION} && \
+     make -j 16 && \
+     make install && \ 
+     make clean 
+
+RUN rm -rf /opt/vtk/build &&  rm -rf /opt/vtk/src
+
+# --------------------------------------------------------------------------
+#
+# VTK 9.2 environment setup
+#
+
+# VTK 9.2.2
+ENV VTK_INSTALL=/opt/vtk/${VTK_VERSION}
+ENV PATH="${PATH}:${VTK_INSTALL}/bin"
+ENV CMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}:${VTK_INSTALL}"
+
+
+# --------------------------------------------------------------------------
+#
+# GLFW  & GLEW
+#
+
+RUN apt install -y libglfw3-dev libglew-dev 
+
+
+WORKDIR /workspace/holohub

applications/orsi/README.md

@@ -0,0 +1,124 @@
+# Orsi Academy Sample Applications
+
+## Introduction
+
+This folder contains three sample applications, please refer to the respective application README for details on assets and use.
+
+1. **[In and out of body detection and anonymization of surgical video sample app](./orsi_in_out_body/README.md):**
+
+    Often times during surgery the camera is removed for cleaning, possibly capturing sensitive data. To mitigate this, this sample app deploys a binary classification model to accurately classify frames being in or outside of the patient's body. Due to the realtime processing time, it can be applied in streaming settings for surgical teaching to retain patient and medical staff privacy.
+
+2. **[Non Organic Structure Segmentation and with AI enhanced visualization of AR over or pre-operative structures rendered with VTK sample app](./orsi_segmentation_ar/README.md):**
+
+    3D models are used in surgery to improve patient outcomes. They provide information on patient specific anatomies that are not visible in the present surgical scene. Especially in robotic surgery these 3D models give great insights because they can be projected and aligned directly onto the endoscopic video. This augmented reality supports navigation for the surgeon in the console. The downside of the 3D model projection is that it occludes the surgical instruments, creating a possible hazardous situation for the patient. This application uses a deep learning segmentation model to identify non-organic items such as tools, clips and wires and projects them on top of the 3D model. This solves the occlusion problem and adds a sense of depth to the AR application. The paper describing the deployment of this application has been awarded an outstanding paper award at the joint 17th AE-CAI, 10th CARE and 6th OR 2.0 workshop, MICCAI 2023. (https://doi.org/10.1049/htl2.12056) 
+3. **[Multi AI (models from 1. and 2.) and AI enhanced visualization of AR over or pre-operative structures rendered with VTK sample app](./orsi_multi_ai_ar/README.md):**
+
+    A Multi AI pipeline featuring both the segmentation and anonymization models.
+
+The segmentation and Multi AI pipelines have successfully been used in multiple in human procedures:
+- Robot-assisted partial nephrectomy
+- Robot-assisted migrated endovascular stent removal
+- Robot-assisted liver metastasectomy
+- Robot-assisted pulmonary lobectomy
+
+## Press release and related publications
+
+- https://www.orsi-online.com/news/orsi-academy-brings-real-time-artificial-intelligence-or-first-its-kind-augmented-reality-surgery
+- https://www.deltacast.tv/news/news/2023/press-release-augmented-reality-in-robotic-surgery-first-in-human-with-nvidia-holoscan-and-deltacast-video-solutions
+- https://amazingerasmusmc.nl/chirurgie/wereldprimeur-longkankeroperatie-met-augmented-reality/
+- https://www.orsi-online.com/news/worlds-first-lung-cancer-surgery-orsi-innotechs-augmented-reality-technology
+- Hofman, J., De Backer, P.,Manghi, I., Simoens, J., De Groote, R., Van Den Bossche, H., D’Hondt, M., Oosterlinck, T., Lippens, J.,Van Praet, C.,    Ferraguti, F., Debbaut, C., Li, Z., Kutter, O., Mottrie, A., Decaestecker, K.: First-in-human real-time AI-assisted instrument deocclusion during augmented reality robotic surgery. Healthc. Technol. Lett. 1–7 (2023). https://doi.org/10.1049/htl2.12056
+- De Backer, P., Van Praet, C., Simoens, J., et al.: Improving augmented reality through deep learning: Real-time instrument delineation in robotic renal surgery. Eur. Urol. 84(1), 86–91 (2023)
+## Run the sample applications
+### (1) Download video and ONNX model
+
+1. Download the data from NGC.
+2. Create data folder in main holohub folder.
+3. Unzip orsi.zip to data/orsi in main holohub folder.
+
+### (2) Build Dev Container with dependencies for Orsi Sample Apps
+
+#### Holoscan Container from NGC:
+```bash
+ ./dev_container build --docker_file applications/orsi/Dockerfile --img holohub:orsi
+```
+#### Local Holoscan SDK Container
+```bash
+./dev_container build --docker_file applications/orsi/Dockerfile --base_img holoscan-sdk-dev:latest --img holohub:orsi-sdk-local
+```
+### (3) Launch Dev Container
+#### Holoscan Container from NGC:
+
+```bash
+./dev_container launch --img holohub:orsi
+```
+#### Local Holoscan SDK Container:
+```bash
+./dev_container launch --img holohub:orsi-sdk-local --local_sdk_root PATH_TO_LOCAL_HOLOSCAN_SDK
+```
+### (4) Build sample apps
+
+**1. orsi_in_out_body**
+
+```bash
+./run build orsi_in_out_body
+```
+
+
+**2. orsi_segmentation_ar**
+
+```bash
+./run build orsi_segmentation_ar
+```
+
+**3. orsi_multi_ai_ar**
+
+```bash
+./run build orsi_multi_ai_ar
+```
+
+### (5) Run sample apps
+
+**1. orsi_in_out_body**
+
+C++:
+```bash
+./run launch orsi_in_out_body cpp
+```
+
+Python:
+```bash
+./run launch orsi_in_out_body python
+```
+
+**2. orsi_segmentation_ar**
+
+C++:
+```bash
+./run launch orsi_segmentation_ar cpp
+```
+
+Python:
+```bash
+./run launch orsi_segmentation_ar python
+```
+
+**3. orsi_multi_ai_ar**
+
+C++:
+```bash
+./run launch orsi_multi_ai_ar cpp
+```
+
+Python:
+```bash
+./run launch orsi_multi_ai_ar python
+```
+### note:
+This application is patent pending:
+-	EP23163230.8: European patent application “Real-time instrument delineation in robotic surgery”
+-	US18/211,269: US patent application “Real-time instrument delineation in robotic surgery”
+
+
+<center> <img src="./docs/orsi_logo.png" width="400"></center>
+

applications/orsi/lib/CMakeLists.txt

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.20)
+project(orsi_app_lib CXX)
+
+find_package(holoscan 0.6 REQUIRED CONFIG
+             PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install")
+
+add_library(${PROJECT_NAME}
+    orsi_app.hpp
+    orsi_app.cpp
+)
+
+target_link_libraries(${PROJECT_NAME}
+   PUBLIC   holoscan::core
+)
+
+set(VIDEOMASTER_OPERATOR "")
+if(ORSI_VIDEO_MASTER_SDK_FOUND)
+    target_compile_definitions(${PROJECT_NAME} PRIVATE -DUSE_VIDEOMASTER)
+endif()
+
+target_include_directories(${PROJECT_NAME}
+    PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+    PUBLIC
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+        $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+)