diff --git a/callbacks.py b/callbacks.py index 527901602..c51e2609c 100644 --- a/callbacks.py +++ b/callbacks.py @@ -6,7 +6,7 @@ def onShowFrame(frame, source): pass -def onNn(nn_packet): +def onNn(nn_packet, decoded_data): pass diff --git a/depthai_demo.py b/depthai_demo.py index 04c130a1a..8eaa9b7ec 100755 --- a/depthai_demo.py +++ b/depthai_demo.py @@ -44,7 +44,7 @@ from depthai_helpers.metrics import MetricManager from depthai_helpers.version_check import checkRequirementsVersion from depthai_sdk import FPSHandler, loadModule, getDeviceInfo, downloadYTVideo, Previews, createBlankFrame -from depthai_sdk.managers import NNetManager, PreviewManager, PipelineManager, EncodingManager, BlobManager +from depthai_sdk.managers import NNetManager, SyncedPreviewManager, PreviewManager, PipelineManager, EncodingManager, BlobManager args = parseArgs() @@ -177,7 +177,7 @@ def setup(self, conf: ConfigManager): reportFileP = Path(self._conf.args.reportFile).with_suffix('.csv') reportFileP.parent.mkdir(parents=True, exist_ok=True) self._reportFile = reportFileP.open('a') - self._pm = PipelineManager(openvinoVersion=self._openvinoVersion) + self._pm = PipelineManager(openvinoVersion=self._openvinoVersion, lowCapabilities=self._conf.lowCapabilities) if self._conf.args.xlinkChunkSize is not None: self._pm.setXlinkChunkSize(self._conf.args.xlinkChunkSize) @@ -189,7 +189,7 @@ def setup(self, conf: ConfigManager): zooName=self._conf.getModelName(), progressFunc=self.showDownloadProgress ) - self._nnManager = NNetManager(inputSize=self._conf.inputSize) + self._nnManager = NNetManager(inputSize=self._conf.inputSize, bufferSize=10 if self._conf.args.syncPreviews else 0) if self._conf.getModelDir() is not None: configPath = self._conf.getModelDir() / Path(self._conf.getModelName()).with_suffix(f".json") @@ -216,25 +216,25 @@ def setup(self, conf: ConfigManager): self._cap = cv2.VideoCapture(self._conf.args.video) if not self._conf.useCamera else None self._fps = FPSHandler() if self._conf.useCamera else FPSHandler(self._cap) - if self._conf.useCamera or self._conf.args.sync: - self._pv = PreviewManager(display=self._conf.args.show, nnSource=self._conf.getModelSource(), colorMap=self._conf.getColorMap(), - dispMultiplier=self._conf.dispMultiplier, mouseTracker=True, lowBandwidth=self._conf.lowBandwidth, - scale=self._conf.args.scale, sync=self._conf.args.sync, fpsHandler=self._fps, createWindows=self._displayFrames, - depthConfig=self._pm._depthConfig) + if self._conf.useCamera: + pvClass = SyncedPreviewManager if self._conf.args.syncPreviews else PreviewManager + self._pv = pvClass(display=self._conf.args.show, nnSource=self._conf.getModelSource(), colorMap=self._conf.getColorMap(), + dispMultiplier=self._conf.dispMultiplier, mouseTracker=True, decode=self._conf.lowBandwidth and not self._conf.lowCapabilities, + fpsHandler=self._fps, createWindows=self._displayFrames, depthConfig=self._pm._depthConfig) if self._conf.leftCameraEnabled: self._pm.createLeftCam(self._monoRes, self._conf.args.monoFps, orientation=self._conf.args.cameraOrientation.get(Previews.left.name), - xout=Previews.left.name in self._conf.args.show and (self._conf.getModelSource() != "left" or not self._conf.args.sync)) + xout=Previews.left.name in self._conf.args.show) if self._conf.rightCameraEnabled: self._pm.createRightCam(self._monoRes, self._conf.args.monoFps, orientation=self._conf.args.cameraOrientation.get(Previews.right.name), - xout=Previews.right.name in self._conf.args.show and (self._conf.getModelSource() != "right" or not self._conf.args.sync)) + xout=Previews.right.name in self._conf.args.show) if self._conf.rgbCameraEnabled: - self._pm.createColorCam(self._nnManager.inputSize if self._conf.useNN else self._conf.previewSize, self._rgbRes, self._conf.args.rgbFps, + self._pm.createColorCam(previewSize=self._conf.previewSize, res=self._rgbRes, fps=self._conf.args.rgbFps, orientation=self._conf.args.cameraOrientation.get(Previews.color.name), fullFov=not self._conf.args.disableFullFovNn, - xout=Previews.color.name in self._conf.args.show and (self._conf.getModelSource() != "color" or not self._conf.args.sync)) + xout=Previews.color.name in self._conf.args.show) if self._conf.useDepth: self._pm.createDepth( @@ -247,10 +247,8 @@ def setup(self, conf: ConfigManager): self._conf.args.subpixel, useDepth=Previews.depth.name in self._conf.args.show or Previews.depthRaw.name in self._conf.args.show, useDisparity=Previews.disparity.name in self._conf.args.show or Previews.disparityColor.name in self._conf.args.show, - useRectifiedLeft=Previews.rectifiedLeft.name in self._conf.args.show and ( - self._conf.getModelSource() != "rectifiedLeft" or not self._conf.args.sync), - useRectifiedRight=Previews.rectifiedRight.name in self._conf.args.show and ( - self._conf.getModelSource() != "rectifiedRight" or not self._conf.args.sync), + useRectifiedLeft=Previews.rectifiedLeft.name in self._conf.args.show, + useRectifiedRight=Previews.rectifiedRight.name in self._conf.args.show, ) self._encManager = None @@ -269,10 +267,8 @@ def setup(self, conf: ConfigManager): sbbScaleFactor=self._conf.args.sbbScaleFactor, fullFov=not self._conf.args.disableFullFovNn, ) - self._pm.addNn( - nn=self._nn, sync=self._conf.args.sync, xoutNnInput=Previews.nnInput.name in self._conf.args.show, - useDepth=self._conf.useDepth, xoutSbb=self._conf.args.spatialBoundingBox and self._conf.useDepth - ) + self._pm.addNn(nn=self._nn, xoutNnInput=Previews.nnInput.name in self._conf.args.show, + xoutSbb=self._conf.args.spatialBoundingBox and self._conf.useDepth) def run(self): self._device.startPipeline(self._pm.pipeline) @@ -312,8 +308,6 @@ def run(self): self._pv.createQueues(self._device, self._createQueueCallback) if self._encManager is not None: self._encManager.createDefaultQueues(self._device) - elif self._conf.args.sync: - self._hostOut = self._device.getOutputQueue(Previews.nnInput.name, maxSize=1, blocking=False) self._seqNum = 0 self._hostFrame = None @@ -383,19 +377,16 @@ def loop(self): self._nnManager.sendInputFrame(rawHostFrame, self._seqNum) self._seqNum += 1 - - if not self._conf.args.sync: - self._hostFrame = rawHostFrame + self._hostFrame = rawHostFrame self._fps.tick('host') if self._nnManager is not None: - inNn = self._nnManager.outputQueue.tryGet() + newData, inNn = self._nnManager.parse() if inNn is not None: - self.onNn(inNn) - if not self._conf.useCamera and self._conf.args.sync: - self._hostFrame = Previews.nnInput.value(self._hostOut.get()) - self._nnData = self._nnManager.decode(inNn) + self.onNn(inNn, newData) self._fps.tick('nn') + if newData is not None: + self._nnData = newData if self._conf.useCamera: if self._nnManager is not None: @@ -875,6 +866,9 @@ def guiOnStaticticsConsent(self, value): pass self.worker.signals.setDataSignal.emit(["restartRequired", True]) + def guiOnToggleSyncPreview(self, value): + self.updateArg("syncPreviews", value) + def guiOnToggleColorEncoding(self, enabled, fps): oldConfig = self.confManager.args.encode or {} if enabled: diff --git a/depthai_helpers/arg_manager.py b/depthai_helpers/arg_manager.py index cacf12147..4a707c9ca 100644 --- a/depthai_helpers/arg_manager.py +++ b/depthai_helpers/arg_manager.py @@ -107,8 +107,6 @@ def parseArgs(): parser.add_argument('-s', '--show', default=[], nargs="+", choices=_streamChoices, help="Choose which previews to show. Default: %(default)s") parser.add_argument('--report', nargs="+", default=[], choices=["temp", "cpu", "memory"], help="Display device utilization data") parser.add_argument('--reportFile', help="Save report data to specified target file in CSV format") - parser.add_argument('-sync', '--sync', action="store_true", - help="Enable NN/camera synchronization. If enabled, camera source will be from the NN's passthrough attribute") parser.add_argument("-monor", "--monoResolution", default=400, type=int, choices=[400,720,800], help="Mono cam res height: (1280x)720, (1280x)800 or (640x)400. Default: %(default)s") parser.add_argument("-monof", "--monoFps", default=30.0, type=float, @@ -148,4 +146,5 @@ def parseArgs(): parser.add_argument("--cameraSharpness", type=_comaSeparated("all", int), nargs="+", help="Specify image sharpness") parser.add_argument('--skipVersionCheck', action="store_true", help="Disable libraries version check") parser.add_argument('--noSupervisor', action="store_true", help="Disable supervisor check") + parser.add_argument('--syncPreviews', action="store_true", help="Enable frame synchronization. If enabled, all frames will be synced before preview (same sequence number)") return parser.parse_args() diff --git a/depthai_helpers/config_manager.py b/depthai_helpers/config_manager.py index c52ffe839..e93a91072 100644 --- a/depthai_helpers/config_manager.py +++ b/depthai_helpers/config_manager.py @@ -24,11 +24,9 @@ def __init__(self, args): self.args.encode = dict(self.args.encode) self.args.cameraOrientation = dict(self.args.cameraOrientation) if self.args.scale is None: - self.args.scale = {"color": 0.37 if not self.args.sync else 1} + self.args.scale = {"color": 0.37} else: self.args.scale = dict(self.args.scale) - if not self.useCamera and not self.args.sync: - print("[WARNING] When using video file as an input, it's highly recommended to run the demo with \"--sync\" flag") if (Previews.left.name in self.args.cameraOrientation or Previews.right.name in self.args.cameraOrientation) and self.useDepth: print("[WARNING] Changing mono cameras orientation may result in incorrect depth/disparity maps") @@ -247,17 +245,21 @@ def inputSize(self): @property def previewSize(self): - return self.inputSize or (576, 324) + return (576, 320) @property def lowBandwidth(self): return self.args.bandwidth == "low" + @property + def lowCapabilities(self): + return platform.machine().startswith("arm") or platform.machine().startswith("aarch") + @property def shaves(self): if self.args.shaves is not None: return self.args.shaves - if not self.useCamera and not self.args.sync: + if not self.useCamera: return 8 if self.args.rgbResolution > 1080: return 5 diff --git a/depthai_sdk/requirements.txt b/depthai_sdk/requirements.txt index a91e67ec0..dc82d2323 100644 --- a/depthai_sdk/requirements.txt +++ b/depthai_sdk/requirements.txt @@ -5,3 +5,4 @@ opencv-contrib-python>4 blobconverter>=1.2.8 pytube>=11.0.1 depthai>2 +PyTurboJPEG==1.6.4 diff --git a/depthai_sdk/src/depthai_sdk/managers/__init__.py b/depthai_sdk/src/depthai_sdk/managers/__init__.py index af1d9ae0e..d82f31238 100644 --- a/depthai_sdk/src/depthai_sdk/managers/__init__.py +++ b/depthai_sdk/src/depthai_sdk/managers/__init__.py @@ -1,5 +1,5 @@ -from .blob_manager import BlobManager -from .encoding_manager import EncodingManager -from .nnet_manager import NNetManager -from .pipeline_manager import PipelineManager -from .preview_manager import PreviewManager \ No newline at end of file +from .blob_manager import * +from .encoding_manager import * +from .nnet_manager import * +from .pipeline_manager import * +from .preview_manager import * \ No newline at end of file diff --git a/depthai_sdk/src/depthai_sdk/managers/nnet_manager.py b/depthai_sdk/src/depthai_sdk/managers/nnet_manager.py index 121bba578..86116e318 100644 --- a/depthai_sdk/src/depthai_sdk/managers/nnet_manager.py +++ b/depthai_sdk/src/depthai_sdk/managers/nnet_manager.py @@ -4,7 +4,7 @@ import cv2 import numpy as np -from .preview_manager import PreviewManager +from .preview_manager import PreviewManager, SyncedPreviewManager from ..previews import Previews from ..utils import loadModule, toTensorResult, frameNorm, toPlanar @@ -15,13 +15,14 @@ class NNetManager: decoding neural network output automatically or by using external handler file. """ - def __init__(self, inputSize, nnFamily=None, labels=[], confidence=0.5): + def __init__(self, inputSize, nnFamily=None, labels=[], confidence=0.5, bufferSize=0): """ Args: inputSize (tuple): Desired NN input size, should match the input size defined in the network itself (width, height) nnFamily (str, Optional): type of NeuralNetwork to be processed. Supported: :code:`"YOLO"` and :code:`mobilenet` labels (list, Optional): Allows to display class label instead of ID when drawing nn detections. confidence (float, Optional): Specify detection nn's confidence threshold + bufferSize (int, Optional): Specify how many nn data items to store in :attr:`buffer` """ self.inputSize = inputSize self._nnFamily = nnFamily @@ -29,6 +30,7 @@ def __init__(self, inputSize, nnFamily=None, labels=[], confidence=0.5): self._outputFormat = "detection" self._labels = labels self._confidence = confidence + self._bufferSize = bufferSize #: list: List of available neural network inputs sourceChoices = ("color", "left", "right", "rectifiedLeft", "rectifiedRight", "host") @@ -42,6 +44,8 @@ def __init__(self, inputSize, nnFamily=None, labels=[], confidence=0.5): inputQueue = None #: depthai.DataOutputQueue: DepthAI output queue object that allows to receive NN results from the device. outputQueue = None + #: dict: nn data buffer, disabled by default. Stores parsed nn data with packet sequence number as dict key + buffer = {} _bboxColors = np.random.random(size=(256, 3)) * 256 # Random Colors for bounding boxes @@ -165,14 +169,12 @@ def createNN(self, pipeline, nodes, blobPath, source="color", useDepth=False, mi nodes.xoutNn.setStreamName("nnOut") nodes.nn.out.link(nodes.xoutNn.input) - if self.source == "color": - nodes.camRgb.preview.link(nodes.nn.input) - elif self.source == "host": + if self.source == "host": nodes.xinNn = pipeline.createXLinkIn() nodes.xinNn.setMaxDataSize(self.inputSize[0] * self.inputSize[1] * 3) nodes.xinNn.setStreamName("nnIn") nodes.xinNn.out.link(nodes.nn.input) - elif self.source in ("left", "right", "rectifiedLeft", "rectifiedRight"): + else: nodes.manipNn = pipeline.createImageManip() nodes.manipNn.initialConfig.setResize(*self.inputSize) # The NN model expects BGR input. By default ImageManip output type would be same as input (gray in this case) @@ -181,6 +183,8 @@ def createNN(self, pipeline, nodes, blobPath, source="color", useDepth=False, mi nodes.manipNn.out.link(nodes.nn.input) nodes.manipNn.setKeepAspectRatio(not self._fullFov) + if self.source == "color": + nodes.camRgb.preview.link(nodes.manipNn.inputImage) if self.source == "left": nodes.monoLeft.out.link(nodes.manipNn.inputImage) elif self.source == "right": @@ -219,6 +223,21 @@ def getLabelText(self, label): print(f"Label of ouf bounds (label index: {label}, available labels: {len(self._labels)}") return str(label) + def parse(self, blocking=False): + if blocking: + inNn = self.outputQueue.get() + else: + inNn = self.outputQueue.tryGet() + if inNn is not None: + data = self.decode(inNn) + if self._bufferSize > 0: + if len(self.buffer) == self._bufferSize: + del self.buffer[min(self.buffer.keys())] + self.buffer[inNn.getSequenceNum()] = data + return data, inNn + else: + return None, None + def decode(self, inNn): """ Decodes NN output. Performs generic handling for supported detection networks or calls custom handler methods @@ -304,12 +323,19 @@ def drawDetection(frame, detection): self._textType, 0.5, self._textBgColor, 4, self._lineType) cv2.putText(frame, "Z: {:.2f} m".format(zMeters), (bbox[0] + 10, bbox[1] + 90), self._textType, 0.5, self._textColor, 1, self._lineType) - for detection in decodedData: - if isinstance(source, PreviewManager): - for name, frame in source.frames.items(): - drawDetection(frame, detection) - else: - drawDetection(source, detection) + if isinstance(source, SyncedPreviewManager): + if len(self.buffer) > 0: + data = self.buffer.get(source.nnSyncSeq, self.buffer[max(self.buffer.keys())]) + for detection in data: + for name, frame in source.frames.items(): + drawDetection(frame, detection) + else: + for detection in decodedData: + if isinstance(source, PreviewManager): + for name, frame in source.frames.items(): + drawDetection(frame, detection) + else: + drawDetection(source, detection) if self._countLabel is not None: self._drawCount(source, decodedData) diff --git a/depthai_sdk/src/depthai_sdk/managers/pipeline_manager.py b/depthai_sdk/src/depthai_sdk/managers/pipeline_manager.py index 088257f52..cfb4dcebc 100644 --- a/depthai_sdk/src/depthai_sdk/managers/pipeline_manager.py +++ b/depthai_sdk/src/depthai_sdk/managers/pipeline_manager.py @@ -10,9 +10,10 @@ class PipelineManager: and connection logic onto a set of convenience functions. """ - def __init__(self, openvinoVersion=None, poeQuality=100): + def __init__(self, openvinoVersion=None, poeQuality=100, lowCapabilities=False): self.openvinoVersion=openvinoVersion self.poeQuality = poeQuality + self.lowCapabilities = lowCapabilities #: depthai.Pipeline: Ready to use requested pipeline. Can be passed to :obj:`depthai.Device` to start execution self.pipeline = dai.Pipeline() @@ -28,6 +29,8 @@ def __init__(self, openvinoVersion=None, poeQuality=100): poeQuality = None #: bool: If set to :code:`True`, manager will MJPEG-encode the packets sent from device to host to lower the bandwidth usage. **Can break** if more than 3 encoded outputs requested lowBandwidth = False + #: bool: If set to :code:`True`, manager will try to optimize the pipeline to reduce the amount of host-side calculations (useful for RPi or other embedded systems) + lowCapabilities = False _depthConfig = dai.StereoDepthConfig() _rgbConfig = dai.CameraControl() @@ -136,12 +139,12 @@ def _mjpegLink(self, node, xout, nodeOutput): videnc.setQuality(self.poeQuality) videnc.bitstream.link(xout.input) - def createColorCam(self, previewSize=None, res=dai.ColorCameraProperties.SensorResolution.THE_1080_P, fps=30, fullFov=True, orientation: dai.CameraImageOrientation=None, colorOrder=dai.ColorCameraProperties.ColorOrder.BGR, xout=False): + def createColorCam(self, previewSize=None, res=dai.ColorCameraProperties.SensorResolution.THE_1080_P, fps=30, fullFov=True, orientation: dai.CameraImageOrientation=None, colorOrder=dai.ColorCameraProperties.ColorOrder.BGR, xout=False, frameSize=None): """ Creates :obj:`depthai.node.ColorCamera` node based on specified attributes Args: - previewSize (tuple, Optional): Size of the preview output - :code:`(width, height)`. Usually same as NN input + previewSize (tuple, Optional): Size of the preview - :code:`(width, height)` res (depthai.ColorCameraProperties.SensorResolution, Optional): Camera resolution to be used fps (int, Optional): Camera FPS set on the device. Can limit / increase the amount of frames produced by the camera fullFov (bool, Optional): If set to :code:`True`, full frame will be scaled down to nn size. If to :code:`False`, @@ -163,17 +166,17 @@ def createColorCam(self, previewSize=None, res=dai.ColorCameraProperties.SensorR self.nodes.xoutRgb = self.pipeline.createXLinkOut() self.nodes.xoutRgb.setStreamName(Previews.color.name) if xout: - if self.lowBandwidth: + if self.lowBandwidth and not self.lowCapabilities: self._mjpegLink(self.nodes.camRgb, self.nodes.xoutRgb, self.nodes.camRgb.video) else: - self.nodes.camRgb.video.link(self.nodes.xoutRgb.input) + self.nodes.camRgb.preview.link(self.nodes.xoutRgb.input) self.nodes.xinRgbControl = self.pipeline.createXLinkIn() self.nodes.xinRgbControl.setMaxDataSize(1024) self.nodes.xinRgbControl.setStreamName(Previews.color.name + "_control") self.nodes.xinRgbControl.out.link(self.nodes.camRgb.inputControl) - def createLeftCam(self, res=dai.MonoCameraProperties.SensorResolution.THE_720_P, fps=30, orientation: dai.CameraImageOrientation=None, xout=False): + def createLeftCam(self, res=None, fps=30, orientation: dai.CameraImageOrientation=None, xout=False): """ Creates :obj:`depthai.node.MonoCamera` node based on specified attributes, assigned to :obj:`depthai.CameraBoardSocket.LEFT` @@ -187,13 +190,16 @@ def createLeftCam(self, res=dai.MonoCameraProperties.SensorResolution.THE_720_P, self.nodes.monoLeft.setBoardSocket(dai.CameraBoardSocket.LEFT) if orientation is not None: self.nodes.monoLeft.setImageOrientation(orientation) - self.nodes.monoLeft.setResolution(res) + if res is None: + self.nodes.monoLeft.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P if self.lowBandwidth and self.lowCapabilities else dai.MonoCameraProperties.SensorResolution.THE_720_P) + else: + self.nodes.monoLeft.setResolution(res) self.nodes.monoLeft.setFps(fps) self.nodes.xoutLeft = self.pipeline.createXLinkOut() self.nodes.xoutLeft.setStreamName(Previews.left.name) if xout: - if self.lowBandwidth: + if self.lowBandwidth and not self.lowCapabilities: self._mjpegLink(self.nodes.monoLeft, self.nodes.xoutLeft, self.nodes.monoLeft.out) else: self.nodes.monoLeft.out.link(self.nodes.xoutLeft.input) @@ -202,7 +208,7 @@ def createLeftCam(self, res=dai.MonoCameraProperties.SensorResolution.THE_720_P, self.nodes.xinLeftControl.setStreamName(Previews.left.name + "_control") self.nodes.xinLeftControl.out.link(self.nodes.monoLeft.inputControl) - def createRightCam(self, res=dai.MonoCameraProperties.SensorResolution.THE_720_P, fps=30, orientation: dai.CameraImageOrientation=None, xout=False): + def createRightCam(self, res=None, fps=30, orientation: dai.CameraImageOrientation=None, xout=False): """ Creates :obj:`depthai.node.MonoCamera` node based on specified attributes, assigned to :obj:`depthai.CameraBoardSocket.RIGHT` @@ -216,13 +222,16 @@ def createRightCam(self, res=dai.MonoCameraProperties.SensorResolution.THE_720_P self.nodes.monoRight.setBoardSocket(dai.CameraBoardSocket.RIGHT) if orientation is not None: self.nodes.monoRight.setImageOrientation(orientation) - self.nodes.monoRight.setResolution(res) + if res is None: + self.nodes.monoRight.setResolution(dai.MonoCameraProperties.SensorResolution.THE_400_P if self.lowBandwidth and self.lowCapabilities else dai.MonoCameraProperties.SensorResolution.THE_720_P) + else: + self.nodes.monoRight.setResolution(res) self.nodes.monoRight.setFps(fps) self.nodes.xoutRight = self.pipeline.createXLinkOut() self.nodes.xoutRight.setStreamName(Previews.right.name) if xout: - if self.lowBandwidth: + if self.lowBandwidth and not self.lowCapabilities: self._mjpegLink(self.nodes.monoRight, self.nodes.xoutRight, self.nodes.monoRight.out) else: self.nodes.monoRight.out.link(self.nodes.xoutRight.input) @@ -283,7 +292,7 @@ def createDepth(self, dct=245, median=dai.MedianFilter.KERNEL_7x7, sigma=0, lr=F if useDepth: self.nodes.xoutDepth = self.pipeline.createXLinkOut() self.nodes.xoutDepth.setStreamName(Previews.depthRaw.name) - # if self.lowBandwidth: TODO change once depth frame type (14) is supported by VideoEncoder + # if self.lowBandwidth and not self.lowCapabilities: TODO change once depth frame type (14) is supported by VideoEncoder if False: self._mjpegLink(self.nodes.stereo, self.nodes.xoutDepth, self.nodes.stereo.depth) else: @@ -402,20 +411,18 @@ def updateDepthConfig(self, device, dct=None, sigma=None, median=None, lrc=None, self._depthConfig.algorithmControl.enableLeftRightCheck = lrc self._depthConfigInputQueue.send(self._depthConfig) - def addNn(self, nn, sync=False, useDepth=False, xoutNnInput=False, xoutSbb=False): + def addNn(self, nn, xoutNnInput=False, xoutSbb=False): """ Adds NN node to current pipeline. Usually obtained by calling :obj:`depthai_sdk.managers.NNetManager.createNN` method first Args: nn (depthai.node.NeuralNetwork): prepared NeuralNetwork node to be attached to the pipeline - sync (bool): Will attach NN's passthough output to source XLinkOut, making the frame appear in the output queue same time as NN-results packet - useDepth (bool): If used together with :code:`sync` flag, will attach NN's passthoughDepth output to depth XLinkOut, making the depth frame appear in the output queue same time as NN-results packet xoutNnInput (bool): Set to :code:`True` to create output queue for NN's passthough frames xoutSbb (bool): Set to :code:`True` to create output queue for Spatial Bounding Boxes (area that is used to calculate spatial location) """ # TODO adjust this function once passthrough frame type (8) is supported by VideoEncoder (for self.MjpegLink) - if xoutNnInput or (sync and self.nnManager.source == "host"): + if xoutNnInput: self.nodes.xoutNnInput = self.pipeline.createXLinkOut() self.nodes.xoutNnInput.setStreamName(Previews.nnInput.name) nn.passthrough.link(self.nodes.xoutNnInput.input) @@ -425,39 +432,6 @@ def addNn(self, nn, sync=False, useDepth=False, xoutNnInput=False, xoutSbb=False self.nodes.xoutSbb.setStreamName("sbb") nn.boundingBoxMapping.link(self.nodes.xoutSbb.input) - if sync: - if self.nnManager.source == "color": - if not hasattr(self.nodes, "xoutRgb"): - self.nodes.xoutRgb = self.pipeline.createXLinkOut() - self.nodes.xoutRgb.setStreamName(Previews.color.name) - nn.passthrough.link(self.nodes.xoutRgb.input) - elif self.nnManager.source == "left": - if not hasattr(self.nodes, "xoutLeft"): - self.nodes.xoutLeft = self.pipeline.createXLinkOut() - self.nodes.xoutLeft.setStreamName(Previews.left.name) - nn.passthrough.link(self.nodes.xoutLeft.input) - elif self.nnManager.source == "right": - if not hasattr(self.nodes, "xoutRight"): - self.nodes.xoutRight = self.pipeline.createXLinkOut() - self.nodes.xoutRight.setStreamName(Previews.right.name) - nn.passthrough.link(self.nodes.xoutRight.input) - elif self.nnManager.source == "rectifiedLeft": - if not hasattr(self.nodes, "xoutRectLeft"): - self.nodes.xoutRectLeft = self.pipeline.createXLinkOut() - self.nodes.xoutRectLeft.setStreamName(Previews.rectifiedLeft.name) - nn.passthrough.link(self.nodes.xoutRectLeft.input) - elif self.nnManager.source == "rectifiedRight": - if not hasattr(self.nodes, "xoutRectRight"): - self.nodes.xoutRectRight = self.pipeline.createXLinkOut() - self.nodes.xoutRectRight.setStreamName(Previews.rectifiedRight.name) - nn.passthrough.link(self.nodes.xoutRectRight.input) - - if self.nnManager._nnFamily in ("YOLO", "mobilenet") and useDepth: - if not hasattr(self.nodes, "xoutDepth"): - self.nodes.xoutDepth = self.pipeline.createXLinkOut() - self.nodes.xoutDepth.setStreamName(Previews.depth.name) - nn.passthroughDepth.link(self.nodes.xoutDepth.input) - def createSystemLogger(self, rate=1): """ Creates :obj:`depthai.node.SystemLogger` node together with XLinkOut diff --git a/depthai_sdk/src/depthai_sdk/managers/preview_manager.py b/depthai_sdk/src/depthai_sdk/managers/preview_manager.py index 5eaea6504..23ae537a6 100644 --- a/depthai_sdk/src/depthai_sdk/managers/preview_manager.py +++ b/depthai_sdk/src/depthai_sdk/managers/preview_manager.py @@ -1,7 +1,10 @@ import math +from datetime import timedelta import cv2 import depthai as dai +from depthai_sdk import DelayQueue + from ..previews import Previews, MouseClickTracker import numpy as np @@ -14,30 +17,26 @@ class PreviewManager: #: dict: Contains name -> frame mapping that can be used to modify specific frames directly frames = {} - def __init__(self, display=[], nnSource=None, colorMap=None, depthConfig=None, dispMultiplier=255/96, mouseTracker=False, lowBandwidth=False, scale=None, sync=False, fpsHandler=None, createWindows=True): + def __init__(self, display=[], nnSource=None, colorMap=None, depthConfig=None, dispMultiplier=255/96, mouseTracker=False, decode=False, fpsHandler=None, createWindows=True): """ Args: display (list, Optional): List of :obj:`depthai_sdk.Previews` objects representing the streams to display mouseTracker (bool, Optional): If set to :code:`True`, will enable mouse tracker on the preview windows that will display selected pixel value fpsHandler (depthai_sdk.fps.FPSHandler, Optional): if provided, will use fps handler to modify stream FPS and display it - sync (bool, Optional): If set to :code:`True`, will assume that neural network source camera will not contain raw frame but scaled frame used by NN nnSource (str, Optional): Specifies NN source camera colorMap (cv2 color map, Optional): Color map applied on the depth frames - lowBandwidth (bool, Optional): If set to :code:`True`, will decode the received frames assuming they were encoded with MJPEG encoding - scale (dict, Optional): Allows to scale down frames before preview. Useful when previewing e.g. 4K frames + decode (bool, Optional): If set to :code:`True`, will decode the received frames assuming they were encoded with MJPEG encoding dispMultiplier (float, Optional): Multiplier used for depth <-> disparity calculations (calculated on baseline and focal) depthConfig (depthai.StereoDepthConfig, optional): Configuration used for depth <-> disparity calculations createWindows (bool, Optional): If True, will create preview windows using OpenCV (enabled by default) """ - self.sync = sync self.nnSource = nnSource if colorMap is not None: self.colorMap = colorMap else: self.colorMap = cv2.applyColorMap(np.arange(256, dtype=np.uint8), cv2.COLORMAP_JET) self.colorMap[0] = [0, 0, 0] - self.lowBandwidth = lowBandwidth - self.scale = scale + self.decode = decode self.dispMultiplier = dispMultiplier self._depthConfig = depthConfig self._fpsHandler = fpsHandler @@ -101,6 +100,37 @@ def closeQueues(self): for queue in self.outputQueues: queue.close() + def _processFrame(self, frame, queueName): + if self._fpsHandler is not None: + self._fpsHandler.tick(queueName) + return frame + + def _addRawFrame(self, frame, packet, name): + if name in self._display: + self._rawFrames[name] = frame + + if self._mouseTracker is not None: + if name == Previews.disparity.name: + rawFrame = packet.getFrame() if not self.decode else cv2.imdecode(packet.getData(), cv2.IMREAD_GRAYSCALE) + self._mouseTracker.extractValue(Previews.disparity.name, rawFrame) + self._mouseTracker.extractValue(Previews.disparityColor.name, rawFrame) + if name == Previews.depthRaw.name: + rawFrame = packet.getFrame() # if not self.decode else cv2.imdecode(packet.getData(), cv2.IMREAD_UNCHANGED) TODO uncomment once depth encoding is possible + self._mouseTracker.extractValue(Previews.depthRaw.name, rawFrame) + self._mouseTracker.extractValue(Previews.depth.name, rawFrame) + else: + self._mouseTracker.extractValue(name, frame) + + if name == Previews.disparity.name and Previews.disparityColor.name in self._display: + if self._fpsHandler is not None: + self._fpsHandler.tick(Previews.disparityColor.name) + self._rawFrames[Previews.disparityColor.name] = Previews.disparityColor.value(frame, self) + + if name == Previews.depthRaw.name and Previews.depth.name in self._display: + if self._fpsHandler is not None: + self._fpsHandler.tick(Previews.depth.name) + self._rawFrames[Previews.depth.name] = Previews.depth.value(frame, self) + def prepareFrames(self, blocking=False, callback=None): """ @@ -117,46 +147,21 @@ def prepareFrames(self, blocking=False, callback=None): else: packet = queue.tryGet() if packet is not None: - if self._fpsHandler is not None: - self._fpsHandler.tick(queue.getName()) frame = getattr(Previews, queue.getName()).value(packet, self) if frame is None: print("[WARNING] Conversion of the {} frame has failed! (None value detected)".format(queue.getName())) continue - if self.scale is not None and queue.getName() in self.scale: - h, w = frame.shape[0:2] - frame = cv2.resize(frame, (int(w * self.scale[queue.getName()]), int(h * self.scale[queue.getName()])), interpolation=cv2.INTER_AREA) + frame = self._processFrame(frame, queue.getName()) if queue.getName() in self._display: if callback is not None: callback(frame, queue.getName()) - self._rawFrames[queue.getName()] = frame - if self._mouseTracker is not None: - if queue.getName() == Previews.disparity.name: - rawFrame = packet.getFrame() if not self.lowBandwidth else cv2.imdecode(packet.getData(), cv2.IMREAD_GRAYSCALE) - self._mouseTracker.extractValue(Previews.disparity.name, rawFrame) - self._mouseTracker.extractValue(Previews.disparityColor.name, rawFrame) - if queue.getName() == Previews.depthRaw.name: - rawFrame = packet.getFrame() # if not self.lowBandwidth else cv2.imdecode(packet.getData(), cv2.IMREAD_UNCHANGED) TODO uncomment once depth encoding is possible - self._mouseTracker.extractValue(Previews.depthRaw.name, rawFrame) - self._mouseTracker.extractValue(Previews.depth.name, rawFrame) - else: - self._mouseTracker.extractValue(queue.getName(), frame) - - if queue.getName() == Previews.disparity.name and Previews.disparityColor.name in self._display: - if self._fpsHandler is not None: - self._fpsHandler.tick(Previews.disparityColor.name) - self._rawFrames[Previews.disparityColor.name] = Previews.disparityColor.value(frame, self) - - if queue.getName() == Previews.depthRaw.name and Previews.depth.name in self._display: - if self._fpsHandler is not None: - self._fpsHandler.tick(Previews.depth.name) - self._rawFrames[Previews.depth.name] = Previews.depth.value(frame, self) - - for name in self._rawFrames: - newFrame = self._rawFrames[name].copy() - if name == Previews.depthRaw.name: - newFrame = cv2.normalize(newFrame, None, 255, 0, cv2.NORM_INF, cv2.CV_8UC1) - self.frames[name] = newFrame + self._addRawFrame(frame, packet, queue.getName()) + + for name in self._rawFrames: + newFrame = self._rawFrames[name].copy() + if name == Previews.depthRaw.name: + newFrame = cv2.normalize(newFrame, None, 255, 0, cv2.NORM_INF, cv2.CV_8UC1) + self.frames[name] = newFrame def showFrames(self, callback=None): """ @@ -199,3 +204,90 @@ def get(self, name): numpy.ndarray: Resolved frame, will default to :code:`None` if not present """ return self.frames.get(name, None) + + +class SyncedPreviewManager(PreviewManager): + """ + Extension of the regular PreviewManager that allows to display all of the frames in sync + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._seqPackets = {} + self._packetsQ = None + self._lastSeqs = {} + self._syncedPackets = {} + self.nnSyncSeq = None + + def __get_next_seq_packet(self, seqKey, name, defaultPacket): + if seqKey not in self._seqPackets: + return defaultPacket + elif name not in self._seqPackets: + return self.__get_next_seq_packet(seqKey + 1, name, defaultPacket) + else: + return self._seqPackets[seqKey][name] + + def prepareFrames(self, blocking=False, callback=None): + """ + This overridden function serves the same purpose - to prepare ready to use frames - but before it provide any data + it will sync all of the packets first, using their sequence number. So any frames retrievable from this class, after + this method is called, will be in sync with each other. + + Args: + blocking (bool, Optional): If set to :code:`True`, will wait for a packet in each queue to be available + callback (func, Optional): Function that will be executed once packet with frame has arrived + """ + + for queue in self.outputQueues: + if blocking: + packet = queue.get() + else: + packet = queue.tryGet() + if packet is not None: + seq = packet.getSequenceNum() + packets = self._seqPackets.get(seq, {}) + packets[queue.getName()] = packet + self._seqPackets[seq] = packets + self._lastSeqs[queue.getName()] = seq + + if len(packets) == len(self.outputQueues): + self._packetsQ = DelayQueue(maxsize=100) + prevTimestamp = None + prevDelay = timedelta() + unsyncedSeq = sorted(list(filter(lambda itemSeq: itemSeq < seq, self._seqPackets.keys()))) + for seqKey in unsyncedSeq: + unsynced = { + synced_name: self.__get_next_seq_packet(seqKey, synced_name, synced_packet) + for synced_name, synced_packet in packets.items() + } + ts = next(iter(unsynced.values())).getTimestamp() + if prevTimestamp is None: + delay = timedelta() + else: + delta = ts - prevTimestamp + delay = delta + prevDelay + prevDelay += delta + + prevTimestamp = ts + self._packetsQ.put(unsynced, delay.microseconds) + del self._seqPackets[seqKey] + + if self._packetsQ is not None: + packets = self._packetsQ.get() + if packets is not None: + self.nnSyncSeq = min(map(lambda packet: packet.getSequenceNum(), packets.values())) + for name, packet in packets.items(): + frame = getattr(Previews, name).value(packet, self) + if frame is None: + print("[WARNING] Conversion of the {} frame has failed! (None value detected)".format(name)) + continue + frame = self._processFrame(frame, name) + if callback is not None: + callback(frame, name) + self._addRawFrame(frame, packet, name) + + for name in self._rawFrames: + newFrame = self._rawFrames[name].copy() + if name == Previews.depthRaw.name: + newFrame = cv2.normalize(newFrame, None, 255, 0, cv2.NORM_INF, cv2.CV_8UC1) + self.frames[name] = newFrame diff --git a/depthai_sdk/src/depthai_sdk/previews.py b/depthai_sdk/src/depthai_sdk/previews.py index cff9bd013..0ec602983 100644 --- a/depthai_sdk/src/depthai_sdk/previews.py +++ b/depthai_sdk/src/depthai_sdk/previews.py @@ -4,9 +4,27 @@ import cv2 import numpy as np +try: + from turbojpeg import TurboJPEG, TJFLAG_FASTUPSAMPLE, TJFLAG_FASTDCT, TJPF_GRAY + turbo = TurboJPEG() +except: + turbo = None class PreviewDecoder: + + @staticmethod + def __jpegDecode(data, type): + if turbo is not None: + if type == cv2.IMREAD_GRAYSCALE: + return turbo.decode(data, flags=TJFLAG_FASTUPSAMPLE | TJFLAG_FASTDCT, pixel_format=TJPF_GRAY) + if type == cv2.IMREAD_UNCHANGED: + return turbo.decode_to_yuv(data, flags=TJFLAG_FASTUPSAMPLE | TJFLAG_FASTDCT) + else: + return turbo.decode(data, flags=TJFLAG_FASTUPSAMPLE | TJFLAG_FASTDCT) + else: + return cv2.imdecode(data, type) + @staticmethod def nnInput(packet, manager=None): """ @@ -19,9 +37,9 @@ def nnInput(packet, manager=None): Returns: numpy.ndarray: Ready to use OpenCV frame """ - # if manager is not None and manager.lowBandwidth: TODO change once passthrough frame type (8) is supported by VideoEncoder + # if manager is not None and manager.decode: TODO change once passthrough frame type (8) is supported by VideoEncoder if False: - frame = cv2.imdecode(packet.getData(), cv2.IMREAD_COLOR) + frame = PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_COLOR) else: frame = packet.getCvFrame() if hasattr(manager, "nnSource") and manager.nnSource in (Previews.rectifiedLeft.name, Previews.rectifiedRight.name): @@ -40,8 +58,8 @@ def color(packet, manager=None): Returns: numpy.ndarray: Ready to use OpenCV frame """ - if manager is not None and manager.lowBandwidth and not manager.sync: # TODO remove sync check once passthrough is supported for MJPEG encoding - return cv2.imdecode(packet.getData(), cv2.IMREAD_COLOR) + if manager is not None and manager.decode: + return PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_COLOR) else: return packet.getCvFrame() @@ -57,8 +75,8 @@ def left(packet, manager=None): Returns: numpy.ndarray: Ready to use OpenCV frame """ - if manager is not None and manager.lowBandwidth and not manager.sync: # TODO remove sync check once passthrough is supported for MJPEG encoding - return cv2.imdecode(packet.getData(), cv2.IMREAD_GRAYSCALE) + if manager is not None and manager.decode: + return PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_GRAYSCALE) else: return packet.getCvFrame() @@ -74,8 +92,8 @@ def right(packet, manager=None): Returns: numpy.ndarray: Ready to use OpenCV frame """ - if manager is not None and manager.lowBandwidth and not manager.sync: # TODO remove sync check once passthrough is supported for MJPEG encoding - return cv2.imdecode(packet.getData(), cv2.IMREAD_GRAYSCALE) + if manager is not None and manager.decode: + return PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_GRAYSCALE) else: return packet.getCvFrame() @@ -91,9 +109,9 @@ def rectifiedLeft(packet, manager=None): Returns: numpy.ndarray: Ready to use OpenCV frame """ - # if manager is not None and manager.lowBandwidth: # disabled to limit the memory usage + # if manager is not None and manager.decode: # disabled to limit the memory usage if False: - return cv2.imdecode(packet.getData(), cv2.IMREAD_GRAYSCALE) + return PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_GRAYSCALE) else: return packet.getCvFrame() @@ -109,9 +127,9 @@ def rectifiedRight(packet, manager=None): Returns: numpy.ndarray: Ready to use OpenCV frame """ - # if manager is not None and manager.lowBandwidth: # disabled to limit the memory usage + # if manager is not None and manager.decode: # disabled to limit the memory usage if False: - return cv2.imdecode(packet.getData(), cv2.IMREAD_GRAYSCALE) + return PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_GRAYSCALE) else: return packet.getCvFrame() @@ -127,9 +145,9 @@ def depthRaw(packet, manager=None): Returns: numpy.ndarray: Ready to use OpenCV frame """ - # if manager is not None and manager.lowBandwidth: TODO change once depth frame type (14) is supported by VideoEncoder + # if manager is not None and manager.decode: TODO change once depth frame type (14) is supported by VideoEncoder if False: - return cv2.imdecode(packet.getData(), cv2.IMREAD_UNCHANGED) + return PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_UNCHANGED) else: return packet.getFrame() @@ -180,7 +198,7 @@ def disparity(packet, manager=None): numpy.ndarray: Ready to use OpenCV frame """ if False: - rawFrame = cv2.imdecode(packet.getData(), cv2.IMREAD_GRAYSCALE) + rawFrame = PreviewDecoder.__jpegDecode(packet.getData(), cv2.IMREAD_GRAYSCALE) else: rawFrame = packet.getFrame() return (rawFrame*(manager.dispMultiplier if manager is not None else 255/96)).astype(np.uint8) diff --git a/depthai_sdk/src/depthai_sdk/utils.py b/depthai_sdk/src/depthai_sdk/utils.py index b9c08e325..367afe987 100644 --- a/depthai_sdk/src/depthai_sdk/utils.py +++ b/depthai_sdk/src/depthai_sdk/utils.py @@ -6,6 +6,9 @@ import cv2 import numpy as np import depthai as dai +import datetime as dt +from heapq import heappop, heappush +import threading def cosDist(a, b): @@ -287,4 +290,94 @@ def createBlankFrame(width, height, rgb_color=(0, 0, 0)): # Fill image with color image[:] = color - return image \ No newline at end of file + return image + + +""" +.. Copyright (c) 2016 Marshall Farrier + license http://opensource.org/licenses/MIT +Synchronized delay queue. +Notes +----- +`DelayQueue` dispenses with the block and timeout +features available in the `queue` library. Otherwise, +`DelayQueue` is built in a similar manner to `queue.PriorityQueue`. +The client can track retries by wrapping each item +with a parameter that counts retries: + >>> queue.put([n_retries, item]) +""" + + +class DelayQueue(object): + class Empty(Exception): + # raised by `get()` if queue is empty + pass + + + class NotReady(Exception): + # raised by `get()` if queue is not empty, but delay for head + # of queue has not yet expired + pass + + + class Full(Exception): + # raised by `put()` if queue is full + pass + + def __init__(self, maxsize=0): + self.maxsize = maxsize + self.queue = [] + self.mutex = threading.Lock() + self.not_empty = threading.Condition(self.mutex) + self.not_full = threading.Condition(self.mutex) + self.ready = threading.Condition(self.mutex) + + def ask(self): + """ + Return the wait time in seconds required to retrieve the + item currently at the head of the queue. + + Note that there is no guarantee that a call to `get()` will + succeed even if `ask()` returns 0. By the time the calling + thread reacts, other threads may have caused a different + item to be at the head of the queue. + """ + with self.mutex: + if not len(self.queue): + raise self.Empty + utcnow = dt.datetime.utcnow() + if self.queue[0][0] <= utcnow: + self.ready.notify() + return 0 + return (self.queue[0][0] - utcnow).total_seconds() + + def put(self, item, delay=0): + if delay < 0: + raise ValueError("'delay' must be a non-negative number") + with self.not_full: + if len(self.queue) >= self.maxsize > 0: + raise self.Full + heappush(self.queue, (dt.datetime.utcnow() + dt.timedelta(seconds=delay), item)) + self.not_empty.notify() + if not delay: + self.ready.notify() + + def get(self): + with self.ready: + if not len(self.queue): + return None + utcnow = dt.datetime.utcnow() + if utcnow < self.queue[0][0]: + return None + item = heappop(self.queue)[1] + self.not_full.notify() + return item + + def qsize(self): + """ + Return the approximate size of the queue. + The answer will not be reliable, as producers and consumers + can change the queue size before the result can be used. + """ + with self.mutex: + return len(self.queue) \ No newline at end of file diff --git a/gui/main.py b/gui/main.py index cf98e114b..884434d50 100644 --- a/gui/main.py +++ b/gui/main.py @@ -58,6 +58,10 @@ def reloadDevices(self): def toggleStatisticsConsent(self, value): instance.guiOnStaticticsConsent(value) + @pyqtSlot(bool) + def toggleSyncPreview(self, value): + instance.guiOnToggleSyncPreview(value) + @pyqtSlot(str) def runApp(self, appName): instance.guiOnRunApp(appName) diff --git a/gui/views/CameraProperties.qml b/gui/views/CameraProperties.qml index d89c77aa4..d2752ac5a 100644 --- a/gui/views/CameraProperties.qml +++ b/gui/views/CameraProperties.qml @@ -580,6 +580,18 @@ ListView { font.family: "Courier" checked: false } + + Switch { + id: syncPreviewsSwitch + x: 203 + y: 158 + width: 164 + height: 28 + text: qsTr("Sync Previews") + onToggled: { + appBridge.toggleSyncPreview(syncPreviewsSwitch.checked) + } + } } } /*##^## diff --git a/requirements.txt b/requirements.txt index d9359b351..41c40d102 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,4 @@ opencv-contrib-python==4.4.0.46 ; platform_machine == "armv6l" or platform_machi --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-depthai-data-local/wheels/ pyqt5>5,<5.15.6 ; platform_machine != "armv6l" and platform_machine != "armv7l" and platform_machine != "aarch64" --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/ -depthai==2.13.3.0 +depthai==2.13.3.0.dev+c0d306db16bd76a98ca9b96297e21668c3176277