Merge pull request #600 from luxonis/optimize_pi

Optimize RPi performance

callbacks.py

@@ -6,7 +6,7 @@ def onShowFrame(frame, source):
     pass
 
 
-def onNn(nn_packet):
+def onNn(nn_packet, decoded_data):
     pass
 
 

depthai_demo.py

@@ -44,7 +44,7 @@
 from depthai_helpers.metrics import MetricManager
 from depthai_helpers.version_check import checkRequirementsVersion
 from depthai_sdk import FPSHandler, loadModule, getDeviceInfo, downloadYTVideo, Previews, createBlankFrame
-from depthai_sdk.managers import NNetManager, PreviewManager, PipelineManager, EncodingManager, BlobManager
+from depthai_sdk.managers import NNetManager, SyncedPreviewManager, PreviewManager, PipelineManager, EncodingManager, BlobManager
 
 args = parseArgs()
 
@@ -177,7 +177,7 @@ def setup(self, conf: ConfigManager):
             reportFileP = Path(self._conf.args.reportFile).with_suffix('.csv')
             reportFileP.parent.mkdir(parents=True, exist_ok=True)
             self._reportFile = reportFileP.open('a')
-        self._pm = PipelineManager(openvinoVersion=self._openvinoVersion)
+        self._pm = PipelineManager(openvinoVersion=self._openvinoVersion, lowCapabilities=self._conf.lowCapabilities)
 
         if self._conf.args.xlinkChunkSize is not None:
             self._pm.setXlinkChunkSize(self._conf.args.xlinkChunkSize)
@@ -189,7 +189,7 @@ def setup(self, conf: ConfigManager):
                 zooName=self._conf.getModelName(),
                 progressFunc=self.showDownloadProgress
             )
-            self._nnManager = NNetManager(inputSize=self._conf.inputSize)
+            self._nnManager = NNetManager(inputSize=self._conf.inputSize, bufferSize=10 if self._conf.args.syncPreviews else 0)
 
             if self._conf.getModelDir() is not None:
                 configPath = self._conf.getModelDir() / Path(self._conf.getModelName()).with_suffix(f".json")
@@ -216,25 +216,25 @@ def setup(self, conf: ConfigManager):
         self._cap = cv2.VideoCapture(self._conf.args.video) if not self._conf.useCamera else None
         self._fps = FPSHandler() if self._conf.useCamera else FPSHandler(self._cap)
 
-        if self._conf.useCamera or self._conf.args.sync:
-            self._pv = PreviewManager(display=self._conf.args.show, nnSource=self._conf.getModelSource(), colorMap=self._conf.getColorMap(),
-                                dispMultiplier=self._conf.dispMultiplier, mouseTracker=True, lowBandwidth=self._conf.lowBandwidth,
-                                scale=self._conf.args.scale, sync=self._conf.args.sync, fpsHandler=self._fps, createWindows=self._displayFrames,
-                                depthConfig=self._pm._depthConfig)
+        if self._conf.useCamera:
+            pvClass = SyncedPreviewManager if self._conf.args.syncPreviews else PreviewManager
+            self._pv = pvClass(display=self._conf.args.show, nnSource=self._conf.getModelSource(), colorMap=self._conf.getColorMap(),
+                               dispMultiplier=self._conf.dispMultiplier, mouseTracker=True, decode=self._conf.lowBandwidth and not self._conf.lowCapabilities,
+                               fpsHandler=self._fps, createWindows=self._displayFrames, depthConfig=self._pm._depthConfig)
 
             if self._conf.leftCameraEnabled:
                 self._pm.createLeftCam(self._monoRes, self._conf.args.monoFps,
                                  orientation=self._conf.args.cameraOrientation.get(Previews.left.name),
-                                 xout=Previews.left.name in self._conf.args.show and (self._conf.getModelSource() != "left" or not self._conf.args.sync))
+                                 xout=Previews.left.name in self._conf.args.show)
             if self._conf.rightCameraEnabled:
                 self._pm.createRightCam(self._monoRes, self._conf.args.monoFps,
                                   orientation=self._conf.args.cameraOrientation.get(Previews.right.name),
-                                  xout=Previews.right.name in self._conf.args.show and (self._conf.getModelSource() != "right" or not self._conf.args.sync))
+                                  xout=Previews.right.name in self._conf.args.show)
             if self._conf.rgbCameraEnabled:
-                self._pm.createColorCam(self._nnManager.inputSize if self._conf.useNN else self._conf.previewSize, self._rgbRes, self._conf.args.rgbFps,
+                self._pm.createColorCam(previewSize=self._conf.previewSize, res=self._rgbRes, fps=self._conf.args.rgbFps,
                                   orientation=self._conf.args.cameraOrientation.get(Previews.color.name),
                                   fullFov=not self._conf.args.disableFullFovNn,
-                                  xout=Previews.color.name in self._conf.args.show and (self._conf.getModelSource() != "color" or not self._conf.args.sync))
+                                  xout=Previews.color.name in self._conf.args.show)
 
             if self._conf.useDepth:
                 self._pm.createDepth(
@@ -247,10 +247,8 @@ def setup(self, conf: ConfigManager):
                     self._conf.args.subpixel,
                     useDepth=Previews.depth.name in self._conf.args.show or Previews.depthRaw.name in self._conf.args.show,
                     useDisparity=Previews.disparity.name in self._conf.args.show or Previews.disparityColor.name in self._conf.args.show,
-                    useRectifiedLeft=Previews.rectifiedLeft.name in self._conf.args.show and (
-                                self._conf.getModelSource() != "rectifiedLeft" or not self._conf.args.sync),
-                    useRectifiedRight=Previews.rectifiedRight.name in self._conf.args.show and (
-                                self._conf.getModelSource() != "rectifiedRight" or not self._conf.args.sync),
+                    useRectifiedLeft=Previews.rectifiedLeft.name in self._conf.args.show,
+                    useRectifiedRight=Previews.rectifiedRight.name in self._conf.args.show,
                 )
 
             self._encManager = None
@@ -269,10 +267,8 @@ def setup(self, conf: ConfigManager):
                 sbbScaleFactor=self._conf.args.sbbScaleFactor, fullFov=not self._conf.args.disableFullFovNn,
             )
 
-            self._pm.addNn(
-                nn=self._nn, sync=self._conf.args.sync, xoutNnInput=Previews.nnInput.name in self._conf.args.show,
-                useDepth=self._conf.useDepth, xoutSbb=self._conf.args.spatialBoundingBox and self._conf.useDepth
-            )
+            self._pm.addNn(nn=self._nn, xoutNnInput=Previews.nnInput.name in self._conf.args.show,
+                           xoutSbb=self._conf.args.spatialBoundingBox and self._conf.useDepth)
 
     def run(self):
         self._device.startPipeline(self._pm.pipeline)
@@ -312,8 +308,6 @@ def run(self):
             self._pv.createQueues(self._device, self._createQueueCallback)
             if self._encManager is not None:
                 self._encManager.createDefaultQueues(self._device)
-        elif self._conf.args.sync:
-            self._hostOut = self._device.getOutputQueue(Previews.nnInput.name, maxSize=1, blocking=False)
 
         self._seqNum = 0
         self._hostFrame = None
@@ -383,19 +377,16 @@ def loop(self):
 
             self._nnManager.sendInputFrame(rawHostFrame, self._seqNum)
             self._seqNum += 1
-
-            if not self._conf.args.sync:
-                self._hostFrame = rawHostFrame
+            self._hostFrame = rawHostFrame
             self._fps.tick('host')
 
         if self._nnManager is not None:
-            inNn = self._nnManager.outputQueue.tryGet()
+            newData, inNn = self._nnManager.parse()
             if inNn is not None:
-                self.onNn(inNn)
-                if not self._conf.useCamera and self._conf.args.sync:
-                    self._hostFrame = Previews.nnInput.value(self._hostOut.get())
-                self._nnData = self._nnManager.decode(inNn)
+                self.onNn(inNn, newData)
                 self._fps.tick('nn')
+            if newData is not None:
+                self._nnData = newData
 
         if self._conf.useCamera:
             if self._nnManager is not None:
@@ -875,6 +866,9 @@ def guiOnStaticticsConsent(self, value):
                 pass
             self.worker.signals.setDataSignal.emit(["restartRequired", True])
 
+        def guiOnToggleSyncPreview(self, value):
+            self.updateArg("syncPreviews", value)
+
         def guiOnToggleColorEncoding(self, enabled, fps):
             oldConfig = self.confManager.args.encode or {}
             if enabled:

depthai_helpers/arg_manager.py

@@ -107,8 +107,6 @@ def parseArgs():
     parser.add_argument('-s', '--show', default=[], nargs="+", choices=_streamChoices, help="Choose which previews to show. Default: %(default)s")
     parser.add_argument('--report', nargs="+", default=[], choices=["temp", "cpu", "memory"], help="Display device utilization data")
     parser.add_argument('--reportFile', help="Save report data to specified target file in CSV format")
-    parser.add_argument('-sync', '--sync', action="store_true",
-                        help="Enable NN/camera synchronization. If enabled, camera source will be from the NN's passthrough attribute")
     parser.add_argument("-monor", "--monoResolution", default=400, type=int, choices=[400,720,800],
                         help="Mono cam res height: (1280x)720, (1280x)800 or (640x)400. Default: %(default)s")
     parser.add_argument("-monof", "--monoFps", default=30.0, type=float,
@@ -148,4 +146,5 @@ def parseArgs():
     parser.add_argument("--cameraSharpness", type=_comaSeparated("all", int), nargs="+", help="Specify image sharpness")
     parser.add_argument('--skipVersionCheck', action="store_true", help="Disable libraries version check")
     parser.add_argument('--noSupervisor', action="store_true", help="Disable supervisor check")
+    parser.add_argument('--syncPreviews', action="store_true", help="Enable frame synchronization. If enabled, all frames will be synced before preview (same sequence number)")
     return parser.parse_args()

depthai_helpers/config_manager.py

@@ -24,11 +24,9 @@ def __init__(self, args):
         self.args.encode = dict(self.args.encode)
         self.args.cameraOrientation = dict(self.args.cameraOrientation)
         if self.args.scale is None:
-            self.args.scale = {"color": 0.37 if not self.args.sync else 1}
+            self.args.scale = {"color": 0.37}
         else:
             self.args.scale = dict(self.args.scale)
-        if not self.useCamera and not self.args.sync:
-            print("[WARNING] When using video file as an input, it's highly recommended to run the demo with \"--sync\" flag")
         if (Previews.left.name in self.args.cameraOrientation or Previews.right.name in self.args.cameraOrientation) and self.useDepth:
             print("[WARNING] Changing mono cameras orientation may result in incorrect depth/disparity maps")
 
@@ -247,17 +245,21 @@ def inputSize(self):
 
     @property
     def previewSize(self):
-        return self.inputSize or (576, 324)
+        return (576, 320)
 
     @property
     def lowBandwidth(self):
         return self.args.bandwidth == "low"
 
+    @property
+    def lowCapabilities(self):
+        return platform.machine().startswith("arm") or platform.machine().startswith("aarch")
+
     @property
     def shaves(self):
         if self.args.shaves is not None:
             return self.args.shaves
-        if not self.useCamera and not self.args.sync:
+        if not self.useCamera:
             return 8
         if self.args.rgbResolution > 1080:
             return 5

depthai_sdk/requirements.txt

@@ -5,3 +5,4 @@ opencv-contrib-python>4
 blobconverter>=1.2.8
 pytube>=11.0.1
 depthai>2
+PyTurboJPEG==1.6.4

depthai_sdk/src/depthai_sdk/managers/__init__.py

@@ -1,5 +1,5 @@
-from .blob_manager import BlobManager
-from .encoding_manager import EncodingManager
-from .nnet_manager import NNetManager
-from .pipeline_manager import PipelineManager
-from .preview_manager import PreviewManager
+from .blob_manager import *
+from .encoding_manager import *
+from .nnet_manager import *
+from .pipeline_manager import *
+from .preview_manager import *

depthai_sdk/src/depthai_sdk/managers/nnet_manager.py

@@ -4,7 +4,7 @@
 import cv2
 import numpy as np
 
-from .preview_manager import PreviewManager
+from .preview_manager import PreviewManager, SyncedPreviewManager
 from ..previews import Previews
 from ..utils import loadModule, toTensorResult, frameNorm, toPlanar
 
@@ -15,20 +15,22 @@ class NNetManager:
     decoding neural network output automatically or by using external handler file.
     """
 
-    def __init__(self, inputSize, nnFamily=None, labels=[], confidence=0.5):
+    def __init__(self, inputSize, nnFamily=None, labels=[], confidence=0.5, bufferSize=0):
         """
         Args:
             inputSize (tuple): Desired NN input size, should match the input size defined in the network itself (width, height)
             nnFamily (str, Optional): type of NeuralNetwork to be processed. Supported: :code:`"YOLO"` and :code:`mobilenet`
             labels (list, Optional): Allows to display class label instead of ID when drawing nn detections.
             confidence (float, Optional): Specify detection nn's confidence threshold
+            bufferSize (int, Optional): Specify how many nn data items to store in :attr:`buffer`
         """
         self.inputSize = inputSize
         self._nnFamily = nnFamily
         if nnFamily in ("YOLO", "mobilenet"):
             self._outputFormat = "detection"
         self._labels = labels
         self._confidence = confidence
+        self._bufferSize = bufferSize
 
     #: list: List of available neural network inputs
     sourceChoices = ("color", "left", "right", "rectifiedLeft", "rectifiedRight", "host")
@@ -42,6 +44,8 @@ def __init__(self, inputSize, nnFamily=None, labels=[], confidence=0.5):
     inputQueue = None
     #: depthai.DataOutputQueue: DepthAI output queue object that allows to receive NN results from the device.
     outputQueue = None
+    #: dict: nn data buffer, disabled by default. Stores parsed nn data with packet sequence number as dict key
+    buffer = {}
 
 
     _bboxColors = np.random.random(size=(256, 3)) * 256  # Random Colors for bounding boxes
@@ -165,14 +169,12 @@ def createNN(self, pipeline, nodes, blobPath, source="color", useDepth=False, mi
         nodes.xoutNn.setStreamName("nnOut")
         nodes.nn.out.link(nodes.xoutNn.input)
 
-        if self.source == "color":
-            nodes.camRgb.preview.link(nodes.nn.input)
-        elif self.source == "host":
+        if self.source == "host":
             nodes.xinNn = pipeline.createXLinkIn()
             nodes.xinNn.setMaxDataSize(self.inputSize[0] * self.inputSize[1] * 3)
             nodes.xinNn.setStreamName("nnIn")
             nodes.xinNn.out.link(nodes.nn.input)
-        elif self.source in ("left", "right", "rectifiedLeft", "rectifiedRight"):
+        else:
             nodes.manipNn = pipeline.createImageManip()
             nodes.manipNn.initialConfig.setResize(*self.inputSize)
             # The NN model expects BGR input. By default ImageManip output type would be same as input (gray in this case)
@@ -181,6 +183,8 @@ def createNN(self, pipeline, nodes, blobPath, source="color", useDepth=False, mi
             nodes.manipNn.out.link(nodes.nn.input)
             nodes.manipNn.setKeepAspectRatio(not self._fullFov)
 
+            if self.source == "color":
+                nodes.camRgb.preview.link(nodes.manipNn.inputImage)
             if self.source == "left":
                 nodes.monoLeft.out.link(nodes.manipNn.inputImage)
             elif self.source == "right":
@@ -219,6 +223,21 @@ def getLabelText(self, label):
             print(f"Label of ouf bounds (label index: {label}, available labels: {len(self._labels)}")
             return str(label)
 
+    def parse(self, blocking=False):
+        if blocking:
+            inNn = self.outputQueue.get()
+        else:
+            inNn = self.outputQueue.tryGet()
+        if inNn is not None:
+            data = self.decode(inNn)
+            if self._bufferSize > 0:
+                if len(self.buffer) == self._bufferSize:
+                    del self.buffer[min(self.buffer.keys())]
+                self.buffer[inNn.getSequenceNum()] = data
+            return data, inNn
+        else:
+            return None, None
+
     def decode(self, inNn):
         """
         Decodes NN output. Performs generic handling for supported detection networks or calls custom handler methods
@@ -304,12 +323,19 @@ def drawDetection(frame, detection):
                                 self._textType, 0.5, self._textBgColor, 4, self._lineType)
                     cv2.putText(frame, "Z: {:.2f} m".format(zMeters), (bbox[0] + 10, bbox[1] + 90),
                                 self._textType, 0.5, self._textColor, 1, self._lineType)
-            for detection in decodedData:
-                if isinstance(source, PreviewManager):
-                    for name, frame in source.frames.items():
-                        drawDetection(frame, detection)
-                else:
-                    drawDetection(source, detection)
+            if isinstance(source, SyncedPreviewManager):
+                if len(self.buffer) > 0:
+                    data = self.buffer.get(source.nnSyncSeq, self.buffer[max(self.buffer.keys())])
+                    for detection in data:
+                        for name, frame in source.frames.items():
+                            drawDetection(frame, detection)
+            else:
+                for detection in decodedData:
+                    if isinstance(source, PreviewManager):
+                        for name, frame in source.frames.items():
+                            drawDetection(frame, detection)
+                    else:
+                        drawDetection(source, detection)
 
             if self._countLabel is not None:
                 self._drawCount(source, decodedData)