Jupiter_OpenCV_ObjDet_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tests for Object detection with OpenCV\n",
    "the following notebook contains some test with different model/images for object detection. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OpenCV: 4.5.3\n",
      "IMPORTS OK\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import cv2\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "print(\"OpenCV: \"+ str(cv2.__version__))\n",
    "print(\"IMPORTS OK\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_path = \"./img/obj_top.png\"\n",
    "show_image = False #show preview image in notebook\n",
    "WIDTH_ROBOT = 960\n",
    "HEIGHT_ROBOT = 600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dimensions image upload: (600, 960)\n",
      "dimensions robot camera: (600, 960)\n",
      "Same dimensions: True\n"
     ]
    }
   ],
   "source": [
    "#Read image\n",
    "img_original = cv2.imread(image_path,cv2.IMREAD_COLOR )\n",
    "#calculate h and w\n",
    "(h, w) = img_original.shape[:2]\n",
    "WIDTH = w\n",
    "HEIGHT = h\n",
    "print(\"dimensions image upload: \"+ str((h,w)) )\n",
    "print(\"dimensions robot camera: \"+ str((HEIGHT_ROBOT, WIDTH_ROBOT)) )\n",
    "print(\"Same dimensions: \"+str((h,w)==(HEIGHT, WIDTH)))\n",
    "\n",
    "def imshow(image_passed, save=False):\n",
    "    if show_image:\n",
    "        #cv2.imshow(\"image\",image_passed) #not working in Jupiter notebook\n",
    "        plt.imshow(cv2.cvtColor(image_passed, cv2.COLOR_BGR2RGB))\n",
    "        plt.show()\n",
    "    if save:\n",
    "        #SAVE\n",
    "        cv2.imwrite(\"./img/save.png\",image_passed)\n",
    "        print(\"Saved\")\n",
    "\n",
    "#SHOW IMAGE        \n",
    "imshow(img_original)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MobileNet SSD Object Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "PROTO = \"./models/MobileNetSSD_deploy.prototxt\"\n",
    "MODEL = \"./models/MobileNetSSD_deploy.caffemodel\"\n",
    "net = cv2.dnn.readNetFromCaffe(PROTO, MODEL)\n",
    "print(\"Net Loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction took 2.29557 seconds\n",
      "Detections: 4\n",
      "Saved\n"
     ]
    }
   ],
   "source": [
    "img_obj = img_original.copy() #copy img\n",
    "\n",
    "#SHOW IMAGE        \n",
    "imshow(img_obj)\n",
    "\n",
    "RESIZED_DIMENSIONS = (300, 300) # Dimensions net was trained on. \n",
    "IMG_NORM_RATIO = 0.007843 # In grayscale a pixel can range between 0 and 255\n",
    " \n",
    "#pascal voc classes\n",
    "classes =  [\"background\", \"aeroplane\", \"bicycle\", \"bird\", \"boat\", \"bottle\", \n",
    "            \"bus\", \"car\", \"cat\", \"chair\", \"cow\", \n",
    "           \"diningtable\",  \"dog\", \"horse\", \"motorbike\", \"person\", \n",
    "           \"pottedplant\", \"sheep\", \"sofa\", \"train\", \"tvmonitor\"]\n",
    "\n",
    "\n",
    "#FORWARD image\n",
    "# Create a blob. A blob is a group of connected pixels in a binary \n",
    "# frame that share some common property (e.g. grayscale value)\n",
    "# Preprocess the frame to prepare it for deep learning classification\n",
    "blob = cv2.dnn.blobFromImage(cv2.resize(img_obj, RESIZED_DIMENSIONS), \n",
    "             IMG_NORM_RATIO, RESIZED_DIMENSIONS, 127.5)\n",
    "\n",
    "# Set the input for the neural network\n",
    "net.setInput(blob)\n",
    "# Predict the objects in the image\n",
    "start = time.time()\n",
    "neural_network_output = net.forward()\n",
    "print('Prediction took {:.5f} seconds'.format(time.time() - start))\n",
    "print(\"Detections: \" + str(neural_network_output.shape[2])) if len(neural_network_output)!=0 else print(\"No Detections\")\n",
    "\n",
    "conf_threshold = 0.15\n",
    "\n",
    "#draw bounding box\n",
    "# Put the bounding boxes around the detected objects\n",
    "for i in np.arange(0, neural_network_output.shape[2]):\n",
    "    confidence = neural_network_output[0, 0, i, 2]\n",
    "    # Confidence must be at least x%       \n",
    "    if confidence > conf_threshold:\n",
    "        idx = int(neural_network_output[0, 0, i, 1])\n",
    "\n",
    "        bounding_box = neural_network_output[0, 0, i, 3:7] * np.array(\n",
    "            [WIDTH, HEIGHT, WIDTH, HEIGHT])\n",
    "\n",
    "        (startX, startY, endX, endY) = bounding_box.astype(\"int\")\n",
    "\n",
    "        label = \"{}: {:.2f}%\".format(classes[idx], confidence * 100) \n",
    "\n",
    "        cv2.rectangle(img_obj, (startX, startY), (\n",
    "            endX, endY), (255,0,0), 2)     \n",
    "\n",
    "        y = startY - 15 if startY > 30 else startY + 15    \n",
    "\n",
    "        cv2.putText(img_obj, label, (startX, y),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,0,0), 2)\n",
    "\n",
    "#SHOW IMAGE        \n",
    "imshow(img_obj, save=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## YOLO V3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### COCO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n",
      "Classes: 80\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "modelConfig  = \"./models/yolov3.cfg\"\n",
    "modelWeigths = \"./models/yolov3.weights\"\n",
    "net = cv2.dnn.readNetFromDarknet(modelConfig, modelWeigths)\n",
    "print(\"Net Loaded\")\n",
    "\n",
    "with open('./models/coco.names', 'r') as f:\n",
    "    classes = f.read().splitlines()\n",
    "print(\"Classes: {}\".format(len(classes)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OPEN IMAGES (OFFICIAL)\n",
    "https://github.com/AlexeyAB/darknet#pre-trained-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n",
      "Classes: 601\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "modelConfig  = \"./models/yolov3-openimages.cfg\"\n",
    "modelWeigths = \"./models/yolov3-openimages.weights\"\n",
    "net = cv2.dnn.readNetFromDarknet(modelConfig, modelWeigths)\n",
    "print(\"Net Loaded\")\n",
    "\n",
    "with open('./models/open_images_yolo.names', 'r') as f:\n",
    "    classes = f.read().splitlines()\n",
    "print(\"Classes: {}\".format(len(classes)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OPEN IMAGES (SPP TRAINED)\n",
    "credits: https://github.com/radekosmulski/yolo_open_images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n",
      "Classes: 500\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "modelConfig  = \"./models/yolov3-openimages-spp.cfg\"\n",
    "modelWeigths = \"./models/yolov3-openimages-spp.weights\"\n",
    "net = cv2.dnn.readNetFromDarknet(modelConfig, modelWeigths)\n",
    "print(\"Net Loaded\")\n",
    "\n",
    "with open('./models/open_images.names', 'r') as f:\n",
    "    classes = f.read().splitlines()\n",
    "print(\"Classes: {}\".format(len(classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Colors generated: 500\n",
      "Prediction took 1.48988 seconds\n",
      "Detections: 2\n",
      "Saved\n"
     ]
    }
   ],
   "source": [
    "img_yolo = img_original.copy() #copy img\n",
    "\n",
    "#SHOW IMAGE        \n",
    "imshow(img_yolo)\n",
    "\n",
    "# create input blob \n",
    "blob = cv2.dnn.blobFromImage(img_yolo, 1/255, (416,416), (0,0,0), True, crop=False)\n",
    "\n",
    "# set input blob for the network\n",
    "net.setInput(blob)\n",
    "\n",
    "# function to get the output layer names \n",
    "# in the architecture\n",
    "def get_output_layers(net):\n",
    "    layer_names = net.getLayerNames()\n",
    "    output_layers = [layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()]\n",
    "    return output_layers\n",
    "\n",
    "# function to draw bounding box on the detected object with class name\n",
    "def draw_bounding_box(img_yolo, class_id, confidence, x, y, x_plus_w, y_plus_h):\n",
    "    label = str(classes[class_id])\n",
    "    # Preparing colour for current bounding box\n",
    "    color = [int(j) for j in colors[class_id]]\n",
    "    cv2.rectangle(img_yolo, (x,y), (x_plus_w,y_plus_h), color, 2)\n",
    "    text_box_current = '{}: {:.2f}'.format(label, confidence)\n",
    "    if y<5:(x,y)=(x+15, y+30) #label position not out of the image\n",
    "    cv2.putText(img_yolo, text_box_current, (x-6,y-6), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,0), 2) \n",
    "    cv2.putText(img_yolo, text_box_current, (x-5,y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2) \n",
    "    \n",
    "\n",
    "# Seed the generator - every time we run the code it will generate by the same rules\n",
    "# In this way we can keep specific colour the same for every class\n",
    "np.random.seed(42)\n",
    "colors = np.random.randint(0, 255, size=(len(classes), 3), dtype='uint8')\n",
    "print(\"Colors generated: \"+str(colors.shape[0]))\n",
    "    \n",
    "    \n",
    "# run inference through the network\n",
    "# and gather predictions from output layers\n",
    "start = time.time()\n",
    "outs = net.forward(get_output_layers(net))\n",
    "print('Prediction took {:.5f} seconds'.format(time.time() - start))\n",
    "\n",
    "# initialization\n",
    "class_ids = []\n",
    "confidences = []\n",
    "boxes = []\n",
    "conf_threshold = 0.1\n",
    "nms_threshold = 0.6 #lower=stronger\n",
    "\n",
    "# for each detetion from each output layer \n",
    "# get the confidence, class id, bounding box params\n",
    "# and ignore weak detections (confidence < conf_threshold)\n",
    "for out in outs:\n",
    "    for detection in out:\n",
    "        scores = detection[5:]\n",
    "        class_id = np.argmax(scores)\n",
    "        confidence = scores[class_id]\n",
    "        if confidence > conf_threshold:\n",
    "            center_x = int(detection[0] * WIDTH)\n",
    "            center_y = int(detection[1] * HEIGHT)\n",
    "            w = int(detection[2] * WIDTH)\n",
    "            h = int(detection[3] * HEIGHT)\n",
    "            x = center_x - w / 2\n",
    "            y = center_y - h / 2\n",
    "            class_ids.append(class_id)\n",
    "            confidences.append(float(confidence))\n",
    "            boxes.append([x, y, w, h])\n",
    "\n",
    "#apply non-max suppression\n",
    "indices = cv2.dnn.NMSBoxes(boxes, confidences, conf_threshold, nms_threshold)\n",
    "print(\"Detections: \"+str(indices.shape[0])) if len(indices)!=0 else print(\"No Detections\")\n",
    "\n",
    "# go through the detections remaining\n",
    "# after nms and draw bounding box\n",
    "for i in indices:\n",
    "    i = i[0]\n",
    "    box = boxes[i]\n",
    "    x = box[0]\n",
    "    y = box[1]\n",
    "    w = box[2]\n",
    "    h = box[3]\n",
    "    draw_bounding_box(img_yolo, class_ids[i], confidences[i], round(x), round(y), round(x+w), round(y+h))\n",
    "\n",
    "\n",
    "#SHOW IMAGE        \n",
    "imshow(img_yolo, save=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Yolo V4 (requires OpenCV>4.3)\n",
    "https://github.com/AlexeyAB/darknet#pre-trained-models\n",
    "\n",
    "[YoloCSP](https://github.com/WongKinYiu/ScaledYOLOv4/tree/yolov4-csp) \n",
    "\n",
    "[Yolo9000](https://github.com/philipperemy/yolo-9000) is currently not supported by OpenCV [link](https://github.com/AlexeyAB/darknet/wiki/Using-Yolo9000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n",
      "Classes: 80\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "modelConfig  = \"./models/yolov4.cfg\"\n",
    "modelWeigths = \"./models/yolov4.weights\"\n",
    "net = cv2.dnn.readNetFromDarknet(modelConfig, modelWeigths)\n",
    "print(\"Net Loaded\")\n",
    "\n",
    "with open('./models/coco.names', 'r') as f:\n",
    "    classes = f.read().splitlines()\n",
    "print(\"Classes: {}\".format(len(classes)))\n",
    "\n",
    "conf_threshold = 0.1\n",
    "nms_threshold = 0.7 #lower=stronger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n",
      "Classes: 80\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "modelConfig  = \"./models/yolov4_new.cfg\"\n",
    "modelWeigths = \"./models/yolov4_new.weights\"\n",
    "net = cv2.dnn.readNetFromDarknet(modelConfig, modelWeigths)\n",
    "print(\"Net Loaded\")\n",
    "\n",
    "with open('./models/coco.names', 'r') as f:\n",
    "    classes = f.read().splitlines()\n",
    "print(\"Classes: {}\".format(len(classes)))\n",
    "\n",
    "#suggested\n",
    "conf_threshold = 0.35\n",
    "nms_threshold = 0.05 #lower=stronger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n",
      "Classes: 80\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "modelConfig  = \"./models/yolov4-csp.cfg\"\n",
    "modelWeigths = \"./models/yolov4-csp.weights\"\n",
    "net = cv2.dnn.readNetFromDarknet(modelConfig, modelWeigths)\n",
    "print(\"Net Loaded\")\n",
    "\n",
    "with open('./models/coco.names', 'r') as f:\n",
    "    classes = f.read().splitlines()\n",
    "print(\"Classes: {}\".format(len(classes)))\n",
    "\n",
    "#suggested\n",
    "conf_threshold = 0.3\n",
    "nms_threshold = 0.01 #lower=stronger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net Loaded\n",
      "Classes: 80\n"
     ]
    }
   ],
   "source": [
    "#Load net\n",
    "modelConfig  = \"./models/yolov4x-mish.cfg\"\n",
    "modelWeigths = \"./models/yolov4x-mish.weights\"\n",
    "net = cv2.dnn.readNetFromDarknet(modelConfig, modelWeigths)\n",
    "print(\"Net Loaded\")\n",
    "\n",
    "with open('./models/coco.names', 'r') as f:\n",
    "    classes = f.read().splitlines()\n",
    "print(\"Classes: {}\".format(len(classes)))\n",
    "\n",
    "#suggested\n",
    "conf_threshold = 0.3\n",
    "nms_threshold = 0.01 #lower=stronger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Colors generated: 80\n",
      "Prediction took 1.35407 seconds\n",
      "Detections: 19\n",
      "Saved\n"
     ]
    }
   ],
   "source": [
    "img_yolo = img_original.copy() #copy img\n",
    "\n",
    "#SHOW IMAGE        \n",
    "imshow(img_yolo)\n",
    "\n",
    "# create input blob \n",
    "blob = cv2.dnn.blobFromImage(img_yolo, 1/255, (416,416), (0,0,0), True, crop=False)\n",
    "\n",
    "# set input blob for the network\n",
    "net.setInput(blob)\n",
    "\n",
    "# function to get the output layer names \n",
    "# in the architecture\n",
    "def get_output_layers(net):\n",
    "    layer_names = net.getLayerNames()\n",
    "    output_layers = [layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()]\n",
    "    return output_layers\n",
    "\n",
    "# function to draw bounding box on the detected object with class name\n",
    "def draw_bounding_box(img_yolo, class_id, confidence, x, y, x_plus_w, y_plus_h):\n",
    "    label = str(classes[class_id])\n",
    "    # Preparing colour for current bounding box\n",
    "    color = [int(j) for j in colors[class_id]]\n",
    "    cv2.rectangle(img_yolo, (x,y), (x_plus_w,y_plus_h), color, 2)\n",
    "    text_box_current = '{}: {:.2f}'.format(label, confidence)\n",
    "    if y<5:(x,y)=(x+15, y+30) #label position not out of the image\n",
    "    cv2.putText(img_yolo, text_box_current, (x-6,y-6), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,0), 2) \n",
    "    cv2.putText(img_yolo, text_box_current, (x-5,y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2) \n",
    "    \n",
    "\n",
    "# Seed the generator - every time we run the code it will generate by the same rules\n",
    "# In this way we can keep specific colour the same for every class\n",
    "np.random.seed(42)\n",
    "colors = np.random.randint(0, 255, size=(len(classes), 3), dtype='uint8')\n",
    "print(\"Colors generated: \"+str(colors.shape[0]))\n",
    "    \n",
    "    \n",
    "# run inference through the network\n",
    "# and gather predictions from output layers\n",
    "start = time.time()\n",
    "outs = net.forward(get_output_layers(net))\n",
    "print('Prediction took {:.5f} seconds'.format(time.time() - start))\n",
    "\n",
    "# initialization\n",
    "class_ids = []\n",
    "confidences = []\n",
    "boxes = []\n",
    "\n",
    "# for each detetion from each output layer \n",
    "# get the confidence, class id, bounding box params\n",
    "# and ignore weak detections (confidence < conf_threshold)\n",
    "for out in outs:\n",
    "    for detection in out:\n",
    "        scores = detection[5:]\n",
    "        class_id = np.argmax(scores)\n",
    "        confidence = scores[class_id]\n",
    "        if confidence > conf_threshold:\n",
    "            center_x = int(detection[0] * WIDTH)\n",
    "            center_y = int(detection[1] * HEIGHT)\n",
    "            w = int(detection[2] * WIDTH)\n",
    "            h = int(detection[3] * HEIGHT)\n",
    "            x = center_x - w / 2\n",
    "            y = center_y - h / 2\n",
    "            class_ids.append(class_id)\n",
    "            confidences.append(float(confidence))\n",
    "            boxes.append([x, y, w, h])\n",
    "\n",
    "#apply non-max suppression\n",
    "indices = cv2.dnn.NMSBoxes(boxes, confidences, conf_threshold, nms_threshold)\n",
    "print(\"Detections: \"+str(indices.shape[0])) if len(indices)!=0 else print(\"No Detections\")\n",
    "\n",
    "# go through the detections remaining\n",
    "# after nms and draw bounding box\n",
    "for i in indices:\n",
    "    i = i[0]\n",
    "    box = boxes[i]\n",
    "    x = box[0]\n",
    "    y = box[1]\n",
    "    w = box[2]\n",
    "    h = box[3]\n",
    "    draw_bounding_box(img_yolo, class_ids[i], confidences[i], round(x), round(y), round(x+w), round(y+h))\n",
    "\n",
    "\n",
    "#SHOW IMAGE        \n",
    "imshow(img_yolo, save=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}