Frame comparison testing (#172)

* Added testing to endoscopy tool tracking cpp/python
---------

Co-authored-by: [email protected] <[email protected]>

applications/endoscopy_tool_tracking/cpp/CMakeLists.txt

@@ -54,10 +54,21 @@ add_dependencies(endoscopy_tool_tracking endoscopy_tool_tracking_deps)
 
 # Add testing
 if(BUILD_TESTING)
-  # Configure the yaml file to only play 10 frames
+
+  set(RECORDING_DIR ${CMAKE_CURRENT_BINARY_DIR}/recording_output)
+  set(SOURCE_VIDEO_BASENAME cpp_endoscopy_tool_tracking_output)
+  set(VALIDATION_FRAMES_DIR ${CMAKE_SOURCE_DIR}/applications/endoscopy_tool_tracking/testing/)
+
+  file(MAKE_DIRECTORY ${RECORDING_DIR})
+
+  # Configure the yaml file for testing
   file(READ "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.yaml" CONFIG_FILE)
-  string(REGEX REPLACE "^source:[^\n]*" "^source: replayer" CONFIG_FILE ${CONFIG_FILE})
+
   string(REPLACE "count: 0" "count: 10" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "record_type: \"none\"" "record_type: \"visualizer\"" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "directory: \"/tmp\"" "directory: \"${RECORDING_DIR}\"" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "basename: \"tensor\"" "basename: \"${SOURCE_VIDEO_BASENAME}\"" CONFIG_FILE ${CONFIG_FILE})
+
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/endoscopy_tool_tracking_testing.yaml" ${CONFIG_FILE})
 
   # Add test
@@ -68,6 +79,22 @@ if(BUILD_TESTING)
   set_tests_properties(endoscopy_tool_tracking_cpp_test PROPERTIES
                        PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking."
                        FAIL_REGULAR_EXPRESSION "[^a-z]Error;ERROR;Failed")
+
+  # Add a test to check the validity of the frames
+  add_test(NAME endoscopy_tool_tracking_cpp_render_test
+      COMMAND python3 ${CMAKE_SOURCE_DIR}/utilities/video_validation.py
+      --source_video_dir ${RECORDING_DIR}
+      --source_video_basename ${SOURCE_VIDEO_BASENAME}
+      --output_dir ${RECORDING_DIR}
+      --validation_frames_dir ${VALIDATION_FRAMES_DIR}
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+
+  set_tests_properties(endoscopy_tool_tracking_cpp_render_test PROPERTIES
+  DEPENDS endoscopy_tool_tracking_cpp_test
+  PASS_REGULAR_EXPRESSION "Valid video output!"
+  )
+
 endif()
 
 # Flow benchmarking

applications/endoscopy_tool_tracking/python/CMakeLists.txt

@@ -18,17 +18,27 @@ if(BUILD_TESTING)
   # To get the environment path
   find_package(holoscan 0.5 REQUIRED CONFIG PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install")
 
-  # Configure the yaml file to only play 10 frames
+  set(RECORDING_DIR ${CMAKE_CURRENT_BINARY_DIR}/recording_output)
+  set(SOURCE_VIDEO_BASENAME python_endoscopy_tool_tracking_output)
+  set(VALIDATION_FRAMES_DIR ${CMAKE_SOURCE_DIR}/applications/endoscopy_tool_tracking/testing/)
+
+  file(MAKE_DIRECTORY ${RECORDING_DIR})
+
+  # Configure the yaml file for testing
   file(READ "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.yaml" CONFIG_FILE)
-  string(REGEX REPLACE "^source:[^\n]*" "^source: replayer" CONFIG_FILE ${CONFIG_FILE})
+
   string(REPLACE "count: 0" "count: 10" CONFIG_FILE ${CONFIG_FILE})
-  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/endoscopy_tool_tracking_testing.yaml ${CONFIG_FILE})
+  string(REPLACE "directory: \"/tmp\"" "directory: \"${RECORDING_DIR}\"" CONFIG_FILE ${CONFIG_FILE})
+  string(REPLACE "basename: \"tensor\"" "basename: \"${SOURCE_VIDEO_BASENAME}\"" CONFIG_FILE ${CONFIG_FILE})
+
+  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/endoscopy_tool_tracking_testing.yaml" ${CONFIG_FILE})
 
   # Add test
   add_test(NAME endoscopy_tool_tracking_python_test
            COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.py
                    --config ${CMAKE_CURRENT_BINARY_DIR}/endoscopy_tool_tracking_testing.yaml
                    --data ${HOLOHUB_DATA_DIR}/endoscopy
+                   --record_type visualizer
            WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 
   set_property(TEST endoscopy_tool_tracking_python_test PROPERTY ENVIRONMENT
@@ -37,4 +47,20 @@ if(BUILD_TESTING)
   set_tests_properties(endoscopy_tool_tracking_python_test PROPERTIES
                 PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking.;"
                 FAIL_REGULAR_EXPRESSION "[^a-z]Error;ERROR;Failed")
+
+  # Add a test to check the validity of the frames
+  add_test(NAME endoscopy_tool_tracking_python_render_test
+    COMMAND python3 ${CMAKE_SOURCE_DIR}/utilities/video_validation.py
+    --source_video_dir ${RECORDING_DIR}
+    --source_video_basename ${SOURCE_VIDEO_BASENAME}
+    --output_dir ${RECORDING_DIR}
+    --validation_frames_dir ${VALIDATION_FRAMES_DIR}
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+  )
+
+  set_tests_properties(endoscopy_tool_tracking_python_render_test PROPERTIES
+    DEPENDS endoscopy_tool_tracking_python_test
+    PASS_REGULAR_EXPRESSION "Valid video output!"
+  )
+
 endif()

utilities/convert_gxf_entities_to_images.py

@@ -0,0 +1,66 @@
+"""
+ SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ SPDX-License-Identifier: Apache-2.0
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+"""  # noqa: E501
+
+import argparse
+import sys
+
+from gxf_entity_codec import EntityReader
+from PIL import Image
+
+
+def iter_output_frames(entity_iter):
+    for entity in entity_iter:
+        component = entity.components[0]  # assume only one component with a tensor
+        tensor = component.tensor
+        array = tensor.array
+        frame_bytes = array.tobytes()
+        yield frame_bytes
+
+
+def convert_gxf_entity_to_images(entity_dir, entity_basename, output_dir, output_name):
+    with EntityReader(directory=entity_dir, basename=entity_basename) as reader:
+        frame_shape = reader.get_frame(0).shape
+        print(
+            f"Frame array shape: {frame_shape[0]}x{frame_shape[1]}x{frame_shape[2]}"
+            " (height x width x channels)",
+            file=sys.stderr,
+        )
+
+        entities = reader.get_entities()
+        for i, frame_data in enumerate(iter_output_frames(entities)):
+            img = Image.frombytes("RGB", (frame_shape[1], frame_shape[0]), frame_data)
+            img.save(str(output_dir + "/" + output_name + "{:04d}.png").format(i + 1))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Command line utility for reading raw video frames in GXF Tensors for "
+            "stream recording."
+        )
+    )
+    parser.add_argument("--basename", default="tensor", help="Basename for gxf entities to read")
+    parser.add_argument("--directory", default="./", help="Directory for gxf entities to read")
+    parser.add_argument("--outputname", default="tensor", help="Output name for images")
+    parser.add_argument("--outputdir", default="./", help="Directory for output images")
+    args = parser.parse_args()
+
+    convert_gxf_entity_to_images(args.directory, args.basename, args.outputdir, args.outputname)
+
+
+if __name__ == "__main__":
+    main()