WIP Super Resolution Demo

applications/CMakeLists.txt

@@ -80,6 +80,8 @@ add_holohub_application(qt_video_replayer DEPENDS OPERATORS qt_video npp_filter)
 
 add_holohub_application(realsense_visualizer DEPENDS OPERATORS realsense_camera)
 
+add_holohub_application(sr_demo)
+
 add_holohub_application(tao_peoplenet)
 
 add_holohub_application(network_radar_pipeline DEPENDS

applications/sr_demo/CMakeLists.txt

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.20)
+project(sr_demo_app)
+
+add_subdirectory(cpp)

applications/sr_demo/README.md

@@ -0,0 +1,53 @@
+# Super Resolution Demo
+This application demonstrates real-time video super resolution implemented through a holoscan
+inference operator. The demo pipline with latency measurements looks like this:
+![](./images/sr_latency_graph.png)
+
+*Latency measured using HSDK2.7 with cuda 12.6 and TRT 10.3.0 The inference latency will be higher if using TRT 8.6.0 instead*
+
+This simple pipeline consists of a video being read from disk, pre-processed and downsampled for
+ingestion into the inference operator which then upsamples the video frames for display in holoviz.
+
+## Video Replayer
+The video replayed from gxf entity files. Useful scripts for converting to and from gxf entities
+are available on [GitHub](https://github.com/nvidia-holoscan/holoscan-sdk/tree/main/scripts#convert_video_to_gxf_entitiespy).
+The directory and basename in the yaml config file should be updated to match it's location:
+```
+replayer:
+  directory: <path/to/directory>
+  basename: <basename>
+```
+
+## Preprocessing
+After dropping the alpha channel the tensors are converted to float32 with an RGB channel ordering 
+and resized to match the expected input of the tensor using the `FormatConverterOp`. Alternatively, 
+downsampling and zero-padding could be used to match the input shape of the engine file. 
+
+## Inference 
+
+Engine files are obtained from secure link. Make sure to to update `sr_inference: model_path_map:` 
+in the configuration file to match the name and location of the downloaded engine files. The engine 
+file used should match the hardware and software being used for inference. Please note before 
+HSDK 2.6 the TRT version in the base image was 8.6 and in HSDK 2.6 and 2.7 it is 10.3.
+
+The TRT engine expects to receive a 544x960x3 float32 input and outputs a 1080x1920x3 float32 
+tensor. The image is expected to be in RGB channel ordering and normalized on the range [-1,1].
+This can be accomplished using the `FormatConverterOp` for preprocessing if necessary.
+
+## Postprocessing and Display
+`FormatConverterOp` is used again to convert the output to a `unit8` image scaled `0-255`.
+The results are displayed using Holoviz.
+
+
+## Build and Run Instructions
+
+To build the dev container with a specific HSDK release the optional '--base_img' flag can be given
+```sh
+./dev_container build --base_img nvcr.io/nvidia/clara-holoscan/holoscan:v2.3.0-dgpu
+```
+The dev container can then be launch and the application built and run
+```sh
+./dev_container run
+./run build sr_demo
+./run launch sr_demo
+```

applications/sr_demo/cpp/CMakeLists.txt

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cmake_minimum_required(VERSION 3.20)
+project(sr_demo CXX CUDA)
+
+find_package(holoscan 2.3 REQUIRED CONFIG
+             PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install")
+
+
+add_executable(sr_demo
+  main.cpp
+)
+target_link_libraries(sr_demo
+  PRIVATE
+  holoscan::core
+  holoscan::ops::video_stream_replayer
+  holoscan::ops::holoviz
+  holoscan::ops::v4l2
+  holoscan::ops::format_converter
+  holoscan::ops::inference
+  holoscan::ops::inference_processor
+)
+
+# Copy config file
+add_custom_target(sr_demo_yaml
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/sr_demo.yaml" ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS "sr_demo.yaml"
+    BYPRODUCTS "sr_demo.yaml"
+)
+
+
+add_dependencies(sr_demo sr_demo_yaml)

applications/sr_demo/cpp/main.cpp

@@ -0,0 +1,104 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <string>
+
+#include <holoscan/holoscan.hpp>
+#include <holoscan/operators/video_stream_replayer/video_stream_replayer.hpp>
+#include <holoscan/operators/holoviz/holoviz.hpp>
+#include <holoscan/operators/holoviz/holoviz.hpp>
+#include <holoscan/operators/inference/inference.hpp>
+#include <holoscan/operators/inference_processor/inference_processor.hpp>
+#include <holoscan/operators/format_converter/format_converter.hpp>
+
+
+class SrDemoApp : public holoscan::Application {
+ public:
+  void compose() override {
+    using namespace holoscan;
+    auto replayer =
+        make_operator<ops::VideoStreamReplayerOp>("replayer", from_config("replayer"), 
+        Arg("allocator") = make_resource<UnboundedAllocator>("pool_replayer"));
+    auto visualizer = make_operator<ops::HolovizOp>("holoviz", from_config("holoviz"));
+
+
+    auto inference = make_operator<ops::InferenceOp>(
+        "inference",
+        from_config("sr_inference"),
+        Arg("allocator") = make_resource<UnboundedAllocator>("pool_inference"));
+
+
+      auto drop_alpha =
+        make_operator<ops::FormatConverterOp>("drop_alpha", from_config("drop_alpha"),
+        Arg("pool") = make_resource<UnboundedAllocator>("pool_drop_alpha"));
+
+      auto preprocessor =
+        make_operator<ops::FormatConverterOp>("preprocessor", from_config("inference_preprocessor"),
+        Arg("pool") = make_resource<UnboundedAllocator>("pool_preprocessor"));
+
+     auto postprocessor =
+        make_operator<ops::FormatConverterOp>("postprocessor", from_config("inference_postprocessor"),
+         Arg("pool") = make_resource<UnboundedAllocator>("pool_postprocessor"));
+
+
+    add_flow(replayer, drop_alpha);
+    add_flow(drop_alpha, preprocessor);
+    add_flow(preprocessor, inference);
+    add_flow(inference, postprocessor);
+    add_flow(postprocessor, visualizer, {{"tensor", "receivers"}});
+
+  }
+};
+
+int main(int argc, char** argv) {
+
+    int opt;
+    std::string config_file;
+    bool tracking = false;
+
+    auto default_path = std::filesystem::canonical(argv[0]).parent_path();
+    default_path /= std::filesystem::path("sr_demo.yaml");
+
+    while ((opt = getopt(argc, argv, "c:t")) != -1) {
+        switch (opt) {
+            case 'c':
+                config_file = optarg;
+                break;
+            case 't':
+                tracking = true;
+                break;
+            default:
+                std::cerr << "Usage: " << argv[0] << " -c configuration_file [-t]" << std::endl;
+                return EXIT_FAILURE;
+        }
+    }
+
+    if (config_file.empty()) {
+      config_file = default_path.string();
+    }
+
+  auto app = holoscan::make_application<SrDemoApp>();
+  if(tracking){
+    auto& track = app->track();
+    track.enable_logging("super_resolution_demo.log");
+  }
+
+  app->config(config_file);
+  app->run();
+
+  return 0;
+}
+

applications/sr_demo/cpp/metadata.json

@@ -0,0 +1,37 @@
+{
+	"application": {
+		"name": "Super Resolution Demo",
+		"authors": [
+			{
+				"name": "Holoscan Team",
+				"affiliation": "NVIDIA"
+			}
+		],
+		"language": "C++",
+		"version": "1.0",
+		"changelog": {
+			"1.0": "Initial Release"
+		},
+		"holoscan_sdk": {
+			"minimum_required_version": "2.3.0",
+			"tested_versions": [
+				"2.3.0",
+				"2.6.0",
+				"2.7.0"
+			]
+		},
+		"platforms": [
+			"amd64",
+			"arm64"
+		],
+		"tags": [
+			"Endoscopy"
+		],
+		"ranking": 1,
+		"dependencies": {},
+		"run": {
+			"command": "<holohub_app_bin>/sr_demo",
+			"workdir": "holohub_bin"
+		}
+	}
+}

applications/sr_demo/cpp/sr_demo.yaml

@@ -0,0 +1,71 @@
+%YAML 1.2
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+dual_window: false
+
+replayer:
+  directory: "/workspace/holohub/data/rgb_uncompressed_2"
+  basename: "rgb_uncompressed_2"
+  frame_rate: 0   # as specified in timestamps
+  repeat: true    # default: false
+  realtime: true  # default: true 
+  count: 0        # default: 0 (no frame count restriction)
+
+drop_alpha:
+  in_dtype: "rgba8888"
+  out_dtype: "rgb888"
+
+
+inference_preprocessor:
+  in_dtype: "rgb888"
+  out_tensor_name: INPUT__0
+  out_dtype: "float32"
+  resize_width: 960
+  resize_height: 544
+  scale_min: -1
+  scale_max: 1
+  out_channel_order: [2,0,1]
+
+sr_inference:
+  model_path_map:
+    "sr": "/workspace/holohub/data/sr_upsampling/model.NVIDIARTXA6000.8.6.84.trt.8.6.1.6.engine.fp16"
+  backend: "trt"
+  pre_processor_map:
+    "sr": ["INPUT__0"]
+  inference_map:
+    "sr": ["OUTPUT__0"]
+  device_map:
+    "sr": "0"
+  input_on_cuda: true
+  is_engine_path: true
+  enable_fp16: true
+
+inference_postprocessor:
+  in_dtype: "float32"
+  in_tensor_name: OUTPUT__0
+  out_tensor_name: ""
+  out_dtype: "rgb888"
+  scale_min: -1
+  scale_max: 1
+
+holoviz:
+  width: 1920
+  height: 1088
+  tensors:
+    - name: ""
+      type: color
+      opacity: 1.0
+      priority: 0