tierpsy_coco_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# COCO scores evaluator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculates the COCO evaluation metrics for the outputs of the tierpsy tracker for comparison against Mask RCNN. \n",
    "\n",
    "The TF object detection API containing Mask-RCNN has COCO evaulation built in, so this notebook is only needed for calculating baselines.\n",
    "\n",
    "Adapted from https://github.com/cocodataset/cocoapi.git\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-06T12:36:13.072733Z",
     "start_time": "2018-09-06T12:36:11.019804Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from pycocotools.coco import COCO\n",
    "from pycocotools.cocoeval import COCOeval\n",
    "import numpy as np\n",
    "import skimage.io as io\n",
    "import pylab\n",
    "import json\n",
    "import os\n",
    "pylab.rcParams['figure.figsize'] = (10.0, 8.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-06T12:36:13.089472Z",
     "start_time": "2018-09-06T12:36:13.077526Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Datasets:\n",
      "['N2_worms10_food1-10_Set1_Pos4_Ch5_20102017_125024', 'NIC199_worms10_food1-10_Set7_Pos4_Ch4_19052017_153012', 'N2_worms10_CSCD068947_10_Set2_Pos5_Ch1_08082017_212337', 'N2_worms10_CSCD068947_1_Set1_Pos4_Ch4_08082017_210418', 'ED3049_worms10_food1-10_Set6_Pos5_Ch4_19052017_151021', 'CX11271_worms10_food1-10_Set1_Pos4_Ch4_19052017_113042', 'JU360_worms10_food1-10_Set6_Pos5_Ch6_19052017_151012', 'VC2010_worms10_food1-10_Set1_Pos4_Ch6_20122017_150107', 'JU2234_worms10_food1-10_Set1_Pos4_Ch3_20102017_125033', 'JU2587_worms10_food1-10_Set1_Pos4_Ch1_20102017_125044', 'CB4856_worms10_food1-10_Set1_Pos4_Ch2_20102017_125044', 'JU2578_worms10_food1-10_Set1_Pos4_Ch4_20102017_125033']\n"
     ]
    }
   ],
   "source": [
    "annType = 'segm'\n",
    "\n",
    "dataDir='/Users/daniel/Documents/UCL/Project/Data/annotation-data/COCO_outputs/ground_truth'\n",
    "resultsDir='/Users/daniel/Documents/UCL/Project/Data/annotation-data/COCO_outputs/tierpsy_evaluation'\n",
    "datasets = [f[:-5] for f in os.listdir(resultsDir) if not f.startswith('.')]\n",
    "print(\"Datasets:\")\n",
    "print(datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-06T12:36:27.284465Z",
     "start_time": "2018-09-06T12:36:13.093131Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "COCO results for set N2_worms10_food1-10_Set1_Pos4_Ch5_20102017_125024\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.06s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "2330\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=1.18s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.16s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.969\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.969\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.969\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.969\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.614\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.985\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.985\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.985\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set NIC199_worms10_food1-10_Set7_Pos4_Ch4_19052017_153012\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.02s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "1078\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.48s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.06s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.773\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.773\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.773\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.773\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.643\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.787\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.787\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.787\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set N2_worms10_CSCD068947_10_Set2_Pos5_Ch1_08082017_212337\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.04s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "1131\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.76s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.07s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.630\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.630\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.630\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.630\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.331\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.635\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.635\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.635\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set N2_worms10_CSCD068947_1_Set1_Pos4_Ch4_08082017_210418\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.06s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "1281\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.61s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.08s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.908\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.908\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.908\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.908\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.519\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.912\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.912\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.912\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set ED3049_worms10_food1-10_Set6_Pos5_Ch4_19052017_151021\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.05s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "2060\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.99s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.14s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.961\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.961\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.961\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.961\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.635\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.980\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.980\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.980\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set CX11271_worms10_food1-10_Set1_Pos4_Ch4_19052017_113042\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.04s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.01s)\n",
      "creating index...\n",
      "index created!\n",
      "2156\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=1.23s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.17s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.628\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set JU360_worms10_food1-10_Set6_Pos5_Ch6_19052017_151012\n",
      "\n",
      "loading annotations into memory...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done (t=0.04s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "2248\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=1.13s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.15s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.628\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.990\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.990\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.990\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set VC2010_worms10_food1-10_Set1_Pos4_Ch6_20122017_150107\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.05s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "1983\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=1.18s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.13s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.509\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.989\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.989\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.989\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set JU2234_worms10_food1-10_Set1_Pos4_Ch3_20102017_125033\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.08s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "2370\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=1.23s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.23s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.971\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.634\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set JU2587_worms10_food1-10_Set1_Pos4_Ch1_20102017_125044\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.02s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "1258\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.59s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.08s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.612\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.612\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.612\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.612\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.663\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.461\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.619\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.619\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.619\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.667\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set CB4856_worms10_food1-10_Set1_Pos4_Ch2_20102017_125044\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.03s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "1747\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.72s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.11s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.985\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.985\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.985\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.985\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.793\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.994\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.994\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.994\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "COCO results for set JU2578_worms10_food1-10_Set1_Pos4_Ch4_20102017_125033\n",
      "\n",
      "loading annotations into memory...\n",
      "Done (t=0.05s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "2180\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=1.22s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.15s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.970\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.970\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.970\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.970\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.629\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.988\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = -1.000\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for dataset in datasets:\n",
    "\n",
    "    print(\"COCO results for set {}\".format(dataset))\n",
    "    print()\n",
    "    #select ground truth data\n",
    "    annFile = '{}/{}.json'.format(dataDir,dataset)\n",
    "    cocoGt=COCO(annFile)\n",
    "\n",
    "    #select tierpsy outputs for testing\n",
    "    resultsFilename = '{}/{}.json'.format(resultsDir,dataset)\n",
    "    resultsFile = json.load(open(resultsFilename))\n",
    "    resultsList = resultsFile['annotations']\n",
    "\n",
    "    cocoDt=cocoGt.loadRes(resultsList)\n",
    "\n",
    "    imgIds=sorted(cocoGt.getImgIds())\n",
    "    print(len(imgIds))\n",
    "#     imgIds=np.random.choice(len(imgIds), 300, replace=True)\n",
    "    # imgId = imgIds[np.random.randint(100)]\n",
    "\n",
    "    # running evaluation\n",
    "    cocoEval = COCOeval(cocoGt,cocoDt,annType)\n",
    "    cocoEval.params.imgIds  = imgIds\n",
    "    cocoEval.evaluate()\n",
    "    cocoEval.accumulate()\n",
    "    cocoEval.summarize()\n",
    "    print(\"----------------------------------------------------------------------------------------------------\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}