Docker setup initiated, ir_utils model storage added.

.gitignore

@@ -2,4 +2,5 @@ data
 *.egg-info
 *.pyc
 charmap.json.gz
-sandbox
+sandbox
+.vscode

README.md

@@ -1,4 +1,4 @@
-This is the Independent Robotics fork of NanoOWK, the original can be found 
+This is the Independent Robotics fork of NanoOWL, the original can be found 
 [here](https://github.com/NVIDIA-AI-IOT/nanoowl), and the origninal README 
 can be found in the repo as ORIGINAL_README.md.
 

docker/jetson/build_image.bash

@@ -0,0 +1 @@
+DOCKER_BUILDKIT=0 docker build -t nanoowl -f ./nanoowl.Dockerfile .

docker/jetson/build_notes.txt

@@ -0,0 +1,8 @@
+To build on a Jetson, at least with current Jetpack version, 
+you need to export DOCKER_BUILDKIT=0 to turn off the Docker buildkit, because
+it somehow disables the Nvidia Container Toolkit and makes it impossible for the 
+container to access CUDA, TensorRT, etc. 
+
+When building on a Jetson device with less than 16GB of video memory, 
+you're going to need to set up a swap file, or the conversion of ONXX
+models into TensorRT Engines will fail, somewhat silently. 

docker/jetson/nanoowl.Dockerfile

@@ -0,0 +1,84 @@
+##############################
+# Initial setup
+##############################
+FROM dustynv/l4t-ml:r36.2.0
+SHELL ["/bin/bash", "-c"]
+ENV PYTHONWARNINGS="ignore:setup.py install is deprecated::setuptools.command.install"
+
+# This tests to make sure that nvidia-container-toolkit libraries will be available.
+# If it fails, make sure that DOCKER_TOOLKIT=0 is set in your environment variables for
+# The build, or check the status of the bug that was causing this before
+# https://stackoverflow.com/a/75629058
+RUN python3 -c "import tensorrt; print(tensorrt.__version__)"
+
+##############################
+# ROS Humble Install
+##############################
+RUN apt update
+RUN apt install -y locales
+RUN locale-gen en_US en_US.UTF-8
+RUN update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+RUN export LANG=en_US.UTF-8
+
+RUN apt install -y software-properties-common
+RUN add-apt-repository universe
+RUN apt install curl -y
+RUN curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg
+RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | tee /etc/apt/sources.list.d/ros2.list > /dev/null
+RUN apt-get purge -y '*opencv*'
+RUN apt update
+RUN apt upgrade -y
+RUN apt install ros-humble-ros-base -y
+RUN rm -r /usr/local/lib/python3.10/dist-packages/cv2
+
+RUN python3 -m pip install opencv-python
+RUN apt-get update
+RUN apt-get install -y curl
+RUN apt-get install -y build-essential
+RUN apt-get install -y libboost-all-dev
+RUN apt-get install -y libgeographic-dev
+RUN apt-get install -y libopencv-dev
+RUN apt-get install -y ros-humble-geographic-msgs
+RUN apt-get install -y ros-humble-angles
+RUN apt-get install -y ros-humble-diagnostic-updater
+RUN apt-get install -y ros-humble-geodesy
+RUN apt-get install -y ros-humble-camera-info-manager
+RUN apt-get install -y ros-humble-image-transport
+RUN apt-get install -y ros-humble-image-transport-plugins
+RUN apt-get install -y ros-humble-cv-bridge
+RUN apt-get install -y ros-humble-rosbag2-storage-mcap
+RUN apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 gstreamer1.0-pulseaudio ffmpeg
+RUN python3 -m pip install pyserial transforms3d pyubx2 colcon-common-extensions albumentations
+
+##############################
+# NanoSAM/NanoOWL setup
+#
+# Install instructions for these largely come from github.com/dusty-nv/jetson-containers
+##############################
+WORKDIR /opt
+
+# Install Python dependencies for NanoSAM
+RUN pip3 install transformers timm matplotlib gdown
+RUN pip3 install git+https://github.com/openai/CLIP.git
+RUN python3 -c "from transformers import (OwlViTProcessor, OwlViTForObjectDetection); \
+                OwlViTProcessor.from_pretrained(\"google/owlvit-base-patch32\");  \
+                OwlViTForObjectDetection.from_pretrained(\"google/owlvit-base-patch32\");"
+
+# Install Torch2TRT
+RUN git clone https://github.com/NVIDIA-AI-IOT/torch2trt && \
+    cd torch2trt && \
+    python3 setup.py install
+
+##############################
+# Final setup for IR Docker systems (uncomment if you're using this container as part of a IR Docker setup. )
+##############################
+# COPY /ir_utils /root/ir_utils
+# WORKDIR /root/ir_utils
+# RUN pip3 install .
+
+# COPY /core_ws /root/core_ws
+# WORKDIR /root/core_ws
+# RUN . /opt/ros/humble/setup.sh && colcon build
+# RUN . install/setup.bash
+
+WORKDIR /root

docker/jetson/run_interactive.bash

@@ -0,0 +1 @@
+docker run -it --net=host --ipc=host --pid=host -v /dev/shm:/dev/shm  -v /home/aqua/workspaces/shared_ws/src/nanosam/:/root/ nanoowl:latest /bin/bash

docker/x86_64/build_image.bash

@@ -0,0 +1,9 @@
+TARGET=$1
+IR_UTILS=/home/michael/code/ir_utils/
+NANOOWL=/home/michael/code/nanoowl/
+
+docker build --tag nanoowl --file ./nanoowl.Dockerfile \
+    --target=$TARGET \
+    --build-context ir_utils=$IR_UTILS \
+    --build-context nanoowl=$NANOOWL \
+    .

docker/x86_64/nanoowl.Dockerfile

@@ -0,0 +1,61 @@
+##############################
+# Initial setup
+##############################
+# If you run into issues with CUDA version, you should switch to a different CUDA image.
+FROM nvcr.io/nvidia/cuda:12.2.2-devel-ubuntu22.04 as base
+SHELL ["/bin/bash", "-c"]
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+RUN apt update
+RUN apt upgrade -y
+RUN apt install -y python3 python3-pip
+RUN apt-get install -y curl wget git build-essential
+RUN apt-get install -y libboost-all-dev
+RUN apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 gstreamer1.0-pulseaudio ffmpeg
+RUN apt-get install -y libnvinfer-bin
+RUN pip install opencv-python
+
+# NanoSAM setup
+WORKDIR /opt/
+
+# Install Torch2TRT
+RUN git clone https://github.com/NVIDIA-AI-IOT/torch2trt && \
+    cd torch2trt && \
+    python3 -m pip install -r requirements/requirements_10.txt && \
+    python3 -m pip install .
+
+# Install Python dependencies for NanoOWL
+RUN python3 -m pip install transformers timm matplotlib scipy
+RUN python3 -m pip install git+https://github.com/openai/CLIP.git
+
+
+# Install IR utils
+COPY --from=ir_utils . /opt/ir_utils
+RUN cd ir_utils && \
+    python3 -m pip install .
+
+#################################
+# Local version: We copy from local directories.
+#################################
+FROM base as local
+# # Grab most recent from build directory.
+COPY --from=nanoowl . /opt/nanoowl
+RUN cd nanoowl && \ 
+    python3 -m pip install .  
+
+RUN cp /opt/nanoowl/examples/owl_predict.py /root/owl_predict.py
+WORKDIR /root
+#################################
+# Github version: We clone from Github
+#################################
+FROM base as github
+
+# Pull most recent from Github
+RUN git clone https://github.com/independentrobotics/nanoowl.git && \
+    cd nanoowl && \
+    python3 -m pip install .
+
+RUN cp /opt/nanosam/examples/basic_usage.py /root/example.py
+WORKDIR /root

docker/x86_64/run_example.bash

@@ -0,0 +1,12 @@
+docker run \
+    -it \
+    --gpus all \
+    --net=host \
+    --ipc=host \
+    --pid=host \
+    --volume="$HOME/.Xauthority:/root/.Xauthority:rw" \
+    --volume="$HOME/.config/independent-robotics.yaml:/root/.config/independent-robotics.yaml" \
+    --volume="$HOME/IndependentRobotics:/root/IndependentRobotics" \
+    --volume="$1:/root/out" \
+    nanoowl:latest \
+    /usr/bin/python3 /root/owl_predict.py "${@:2}"

docker/x86_64/run_interactive.bash

@@ -0,0 +1,11 @@
+docker run \
+    -it \
+    --gpus all \
+    --net=host \
+    --ipc=host \
+    --pid=host \
+    --volume="$HOME/.Xauthority:/root/.Xauthority:rw" \
+    --volume="$HOME/.config/independent-robotics.yaml:/root/.config/independent-robotics.yaml" \
+    --volume="$HOME/IndependentRobotics:/root/IndependentRobotics" \
+    nanoowl:latest \
+    /bin/bash

examples/owl_predict.py

@@ -25,16 +25,14 @@
     draw_owl_output
 )
 
-
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser()
     parser.add_argument("--image", type=str, default="../assets/owl_glove_small.jpg")
     parser.add_argument("--prompt", type=str, default="an owl, a glove")
     parser.add_argument("--threshold", type=str, default="0.1,0.1")
-    parser.add_argument("--output", type=str, default="../data/owl_predict_out.jpg")
+    parser.add_argument("--output", type=str, default="/root/out/owl_predict_out.jpg")
     parser.add_argument("--model", type=str, default="google/owlvit-base-patch32")
-    parser.add_argument("--image_encoder_engine", type=str, default="../data/owl_image_encoder_patch32.engine")
     parser.add_argument("--profile", action="store_true")
     parser.add_argument("--num_profiling_runs", type=int, default=30)
     args = parser.parse_args()
@@ -51,7 +49,6 @@
 
     predictor = OwlPredictor(
         args.model,
-        image_encoder_engine=args.image_encoder_engine
     )
 
     image = PIL.Image.open(args.image)

nanoowl/owl_drawing.py

@@ -24,7 +24,7 @@
 
 
 def get_colors(count: int):
-    cmap = plt.cm.get_cmap("rainbow", count)
+    cmap = plt.get_cmap("rainbow", count)
     colors = []
     for i in range(count):
         color = cmap(i)
@@ -34,9 +34,9 @@ def get_colors(count: int):
 
 
 def draw_owl_output(image, output: OwlDecodeOutput, text: List[str], draw_text=True):
-    is_pil = not isinstance(image, np.ndarray)
+    is_pil = not isinstance(image, np.ndarray) # That isn't really an accurate measurement...
     if is_pil:
-        image = np.asarray(image)
+        image = np.array(image)
     font = cv2.FONT_HERSHEY_SIMPLEX
     font_scale = 0.75
     colors = get_colors(len(text))

nanoowl/owl_predictor.py

@@ -25,6 +25,9 @@
 from transformers.models.owlvit.processing_owlvit import OwlViTProcessor
 from dataclasses import dataclass
 from typing import List, Optional, Union, Tuple
+
+from ir_utils.filesystem_tools import get_dl_model_directory, create_dl_model_directory
+
 from .image_preprocessor import ImagePreprocessor
 
 __all__ = [
@@ -145,17 +148,22 @@ class OwlPredictor(torch.nn.Module):
     def __init__(self,
             model_name: str = "google/owlvit-base-patch32",
             device: str = "cuda",
-            image_encoder_engine: Optional[str] = None,
             image_encoder_engine_max_batch_size: int = 1,
             image_preprocessor: Optional[ImagePreprocessor] = None
         ):
 
         super().__init__()
 
+        try:
+            model_path = get_dl_model_directory("nanoowl")
+        except ValueError:
+            create_dl_model_directory("nanoowl")
+            model_path = get_dl_model_directory("nanoowl")
+
         self.image_size = _owl_get_image_size(model_name)
         self.device = device
-        self.model = OwlViTForObjectDetection.from_pretrained(model_name).to(self.device).eval()
-        self.processor = OwlViTProcessor.from_pretrained(model_name)
+        self.model = OwlViTForObjectDetection.from_pretrained(model_name, cache_dir=model_path).to(self.device).eval()
+        self.processor = OwlViTProcessor.from_pretrained(model_name, cache_dir=model_path)
         self.patch_size = _owl_get_patch_size(model_name)
         self.num_patches_per_side = self.image_size // self.patch_size
         self.box_bias = _owl_compute_box_bias(self.num_patches_per_side).to(self.device)
@@ -166,10 +174,19 @@ def __init__(self,
                 torch.linspace(0., 1., self.image_size)
             )
         ).to(self.device).float()
-        self.image_encoder_engine = None
-        if image_encoder_engine is not None:
-            image_encoder_engine = OwlPredictor.load_image_encoder_engine(image_encoder_engine, image_encoder_engine_max_batch_size)
-        self.image_encoder_engine = image_encoder_engine
+
+        onnx_path = model_path + 'owl_image_encoder.onnx'
+        engine_path = model_path + 'owl_image_encoder_patch32.engine'
+        if not os.path.exists(engine_path):
+            print("Building image encoder engine.")
+            self.build_image_encoder_onnx(onnx_path, 16)
+            self.build_image_encoder_engine(engine_path, onnx_path, True)
+        else:
+            print("Image encoder engine found!")
+
+        self.image_encoder_engine = OwlPredictor.load_image_encoder_engine(engine_path, image_encoder_engine_max_batch_size)
+
+
         self.image_preprocessor = image_preprocessor.to(self.device).eval() if image_preprocessor else ImagePreprocessor().to(self.device).eval()
 
     def get_num_patches(self):

setup.py

@@ -3,6 +3,11 @@
 
 setup(
     name="nanoowl",
-    version="0.0.0",
+    version="1.0.0",
+    description="NanoOWL is a project that optimizes OWL-ViT to run in real-time on NVIDIA Jetson Orin Platforms with NVIDIA TensorRT.",
+    author="Originally authored by NVIDIA, modifications made by Independent Robotics.",
+    maintainer="Michael Fulton",
+    maintainer_email="[email protected]",
+    license="Apache 2.0",
     packages=find_packages()
 )