inference_script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dfb32a80-0e1b-4ddf-b2ff-1fe20611345b",
   "metadata": {},
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e496b5c5-b8f1-40b3-821c-ed62af8f54ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import ast\n",
    "import os\n",
    "import pathlib\n",
    "\n",
    "import neptune\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.models.detection.transform import GeneralizedRCNNTransform\n",
    "\n",
    "from pytorch_faster_rcnn_tutorial.backbone_resnet import ResNetBackbones\n",
    "from pytorch_faster_rcnn_tutorial.datasets import (\n",
    "    ObjectDetectionDataSet,\n",
    "    ObjectDetectionDatasetSingle,\n",
    ")\n",
    "from pytorch_faster_rcnn_tutorial.faster_RCNN import get_faster_rcnn_resnet\n",
    "from pytorch_faster_rcnn_tutorial.transformations import (\n",
    "    ComposeDouble,\n",
    "    ComposeSingle,\n",
    "    FunctionWrapperDouble,\n",
    "    FunctionWrapperSingle,\n",
    "    apply_nms,\n",
    "    apply_score_threshold,\n",
    "    normalize_01,\n",
    ")\n",
    "from pytorch_faster_rcnn_tutorial.utils import (\n",
    "    collate_single,\n",
    "    get_filenames_of_path,\n",
    "    save_json,\n",
    ")\n",
    "from pytorch_faster_rcnn_tutorial.viewers.object_detection_viewer import (\n",
    "    ObjectDetectionViewer,\n",
    "    ObjectDetectionViewerSingle,\n",
    ")\n",
    "from training_script import NeptuneSettings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "88f582ef-cf8b-4ddd-bfce-e986bfa6a87f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters\n",
    "params = {\n",
    "    \"EXPERIMENT\": \"HEAD-51\",  # experiment name, e.g. Head-42\n",
    "    \"OWNER\": \"johschmidt42\",  # e.g. johndoe55\n",
    "    \"INPUT_DIR\": \"src/pytorch_faster_rcnn_tutorial/data/heads/test\",  # files to predict\n",
    "    \"PREDICTIONS_PATH\": \"predictions\",  # where to save the predictions\n",
    "    \"MODEL_DIR\": \"heads\",  # load model from checkpoint\n",
    "    \"DOWNLOAD\": True,  # whether to download from neptune\n",
    "    \"DOWNLOAD_PATH\": \"model\",  # where to save the model if DOWNLOAD is True\n",
    "    \"PROJECT\": \"Heads\",  # Project name\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c0f8bab4-464b-480b-883e-3e8e5e267ee8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-08 18:16:13 - INFO - utils.py:27:get_filenames_of_path - Found 8 files in src/pytorch_faster_rcnn_tutorial/data/heads/test\n"
     ]
    }
   ],
   "source": [
    "# input files\n",
    "inputs = get_filenames_of_path(pathlib.Path(params[\"INPUT_DIR\"]))\n",
    "inputs.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "00c30ad2-9fd9-4090-b300-78a36fe83ed7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformations\n",
    "transforms = ComposeSingle(\n",
    "    [\n",
    "        FunctionWrapperSingle(np.moveaxis, source=-1, destination=0),\n",
    "        FunctionWrapperSingle(normalize_01),\n",
    "    ]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a5f296e5-971c-40bc-a7ae-cdb3f110ec2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dataset\n",
    "dataset = ObjectDetectionDatasetSingle(\n",
    "    inputs=inputs,\n",
    "    transform=transforms,\n",
    "    use_cache=False,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e02b562-62e3-42a1-abbd-892c3b77f3ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create dataloader\n",
    "dataloader_prediction = DataLoader(\n",
    "    dataset=dataset,\n",
    "    batch_size=1,\n",
    "    shuffle=False,\n",
    "    num_workers=0,\n",
    "    collate_fn=collate_single,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fe76faa4-35aa-4851-8a6d-91f601f425f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# environment variables (pydantic BaseSettings class)\n",
    "neptune_settings: NeptuneSettings = NeptuneSettings()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "618796d3-1009-42ce-8fbc-4e8197a652ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import experiment from neptune\n",
    "project_name = f'{params[\"OWNER\"]}/{params[\"PROJECT\"]}'\n",
    "project = neptune.init(\n",
    "    project_qualified_name=project_name, api_token=neptune_settings.api_key\n",
    ")  # get project\n",
    "experiment_id = params[\"EXPERIMENT\"]  # experiment id\n",
    "experiment = project.get_experiments(id=experiment_id)[0]\n",
    "parameters = experiment.get_parameters()\n",
    "properties = experiment.get_properties()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f455a449-db53-4e14-aa41-e2192ce719a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rcnn transform\n",
    "transform = GeneralizedRCNNTransform(\n",
    "    min_size=int(parameters[\"MIN_SIZE\"]),\n",
    "    max_size=int(parameters[\"MAX_SIZE\"]),\n",
    "    image_mean=ast.literal_eval(parameters[\"IMG_MEAN\"]),\n",
    "    image_std=ast.literal_eval(parameters[\"IMG_STD\"]),\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b4ada9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# color mapping\n",
    "color_mapping = {\n",
    "    1: \"red\",\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "abcea34a-aeb6-4ca2-9208-ac71074819b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# view dataset\n",
    "datasetviewer = ObjectDetectionViewerSingle(\n",
    "    dataset=dataset, color_mapping=color_mapping, rcnn_transform=transform\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8277d891-0926-4005-938f-a8caf9646ab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# download model from neptune or load from checkpoint\n",
    "if params[\"DOWNLOAD\"]:\n",
    "    download_path = pathlib.Path(os.getcwd()) / params[\"DOWNLOAD_PATH\"]\n",
    "    download_path.mkdir(parents=True, exist_ok=True)\n",
    "    model_name = \"best_model.pt\"  # that's how I called the best model\n",
    "    # model_name = properties['checkpoint_name']  # logged when called log_model_neptune()\n",
    "    if not (download_path / model_name).is_file():\n",
    "        experiment.download_artifact(\n",
    "            path=model_name, destination_dir=download_path\n",
    "        )  # download model\n",
    "\n",
    "    model_state_dict = torch.load(\n",
    "        download_path / model_name, map_location=torch.device(\"cpu\")\n",
    "    )\n",
    "else:\n",
    "    checkpoint = torch.load(params[\"MODEL_DIR\"], map_location=torch.device(\"cpu\"))\n",
    "    model_state_dict = checkpoint[\"hyper_parameters\"][\"model\"].state_dict()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "72a78b32-2283-45ad-83c2-11337e4c2505",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/johannes/learnspace/PyTorch-Object-Detection-Faster-RCNN-Tutorial/.venv/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
      "  warnings.warn(\n",
      "/Users/johannes/learnspace/PyTorch-Object-Detection-Faster-RCNN-Tutorial/.venv/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet34_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet34_Weights.DEFAULT` to get the most up-to-date weights.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "model = get_faster_rcnn_resnet(\n",
    "    num_classes=int(parameters[\"CLASSES\"]),\n",
    "    backbone_name=ResNetBackbones(parameters[\"BACKBONE\"]),  # reverse look-up enum\n",
    "    anchor_size=ast.literal_eval(parameters[\"ANCHOR_SIZE\"]),\n",
    "    aspect_ratios=ast.literal_eval(parameters[\"ASPECT_RATIOS\"]),\n",
    "    fpn=ast.literal_eval(parameters[\"FPN\"]),\n",
    "    min_size=int(parameters[\"MIN_SIZE\"]),\n",
    "    max_size=int(parameters[\"MAX_SIZE\"]),\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6d659f25-0d7d-4a97-be15-5795e24f2e71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load weights\n",
    "model.load_state_dict(model_state_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1e59d69e-b9ca-45b3-8320-2a7d4a0e4c6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# inference (cpu)\n",
    "model.eval()\n",
    "for sample in dataloader_prediction:\n",
    "    x, x_name = sample\n",
    "    with torch.no_grad():\n",
    "        pred = model(x)\n",
    "        pred = {key: value.numpy() for key, value in pred[0].items()}\n",
    "        name = pathlib.Path(x_name[0])\n",
    "        save_dir = pathlib.Path(os.getcwd()) / params[\"PREDICTIONS_PATH\"]\n",
    "        save_dir.mkdir(parents=True, exist_ok=True)\n",
    "        pred_list = {\n",
    "            key: value.tolist() for key, value in pred.items()\n",
    "        }  # numpy arrays are not serializable -> .tolist()\n",
    "        save_json(pred_list, path=save_dir / name.with_suffix(\".json\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "04a2e424-4308-4586-a2ca-d0349cffc594",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-08 18:16:23 - INFO - utils.py:27:get_filenames_of_path - Found 8 files in /Users/johannes/learnspace/PyTorch-Object-Detection-Faster-RCNN-Tutorial/predictions\n"
     ]
    }
   ],
   "source": [
    "# get prediction files\n",
    "predictions = get_filenames_of_path(\n",
    "    pathlib.Path(os.getcwd()) / params[\"PREDICTIONS_PATH\"]\n",
    ")\n",
    "predictions.sort()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "66697a0b-7989-4e44-8056-b62b70a49bc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create prediction dataset\n",
    "iou_threshold = 0.25\n",
    "score_threshold = 0.6\n",
    "\n",
    "transforms_prediction = ComposeDouble(\n",
    "    [\n",
    "        FunctionWrapperDouble(np.moveaxis, source=-1, destination=0),\n",
    "        FunctionWrapperDouble(normalize_01),\n",
    "        FunctionWrapperDouble(\n",
    "            apply_nms, input=False, target=True, iou_threshold=iou_threshold\n",
    "        ),\n",
    "        FunctionWrapperDouble(\n",
    "            apply_score_threshold,\n",
    "            input=False,\n",
    "            target=True,\n",
    "            score_threshold=score_threshold,\n",
    "        ),\n",
    "    ]\n",
    ")\n",
    "\n",
    "dataset_prediction = ObjectDetectionDataSet(\n",
    "    inputs=inputs, targets=predictions, transform=transforms_prediction, use_cache=False\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ccb62e53-0e06-4bb5-86a3-8c23d1556ddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# mapping\n",
    "color_mapping = {\n",
    "    1: \"red\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ff50e474-202d-48ac-b5d0-13d69c3b7de2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-06-08 18:16:25 - INFO - object_detection_viewer.py:42:get_data - Input sample: 001.jpg\n",
      "Shape: torch.Size([3, 300, 600])\n",
      "2023-06-08 18:16:25 - INFO - object_detection_viewer.py:64:get_target - Target sample: 001.json\n",
      "{'boxes': tensor([[334,  72, 425, 210],\n",
      "        [417,  17, 555, 172],\n",
      "        [ 53,   2, 168, 115],\n",
      "        [ 57, 106, 152, 233],\n",
      "        [186,  61, 253, 164],\n",
      "        [220,  43, 309, 152]]), 'labels': tensor([1, 1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:26 - INFO - object_detection_viewer.py:42:get_data - Input sample: 002.jpg\n",
      "Shape: torch.Size([3, 900, 1200])\n",
      "2023-06-08 18:16:26 - INFO - object_detection_viewer.py:64:get_target - Target sample: 002.json\n",
      "{'boxes': tensor([[ 148,    0,  411,  301],\n",
      "        [ 763,   88,  965,  298],\n",
      "        [ 545,   71,  770,  329],\n",
      "        [ 572,  329,  848,  670],\n",
      "        [ 854,  277, 1145,  633],\n",
      "        [ 214,  370,  604,  900],\n",
      "        [  36,  439,  267,  754]]), 'labels': tensor([1, 1, 1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:28 - INFO - object_detection_viewer.py:42:get_data - Input sample: 003.jpg\n",
      "Shape: torch.Size([3, 630, 1200])\n",
      "2023-06-08 18:16:28 - INFO - object_detection_viewer.py:64:get_target - Target sample: 003.json\n",
      "{'boxes': tensor([[ 297,  204,  549,  508],\n",
      "        [ 547,  181,  848,  515],\n",
      "        [ 303,    0,  557,  207],\n",
      "        [ 660,    0,  901,  173],\n",
      "        [ 738,  318, 1004,  558]]), 'labels': tensor([1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:29 - INFO - object_detection_viewer.py:42:get_data - Input sample: 004.jpg\n",
      "Shape: torch.Size([3, 770, 1024])\n",
      "2023-06-08 18:16:29 - INFO - object_detection_viewer.py:64:get_target - Target sample: 004.json\n",
      "{'boxes': tensor([[ 550,   96,  767,  356],\n",
      "        [  52,  174,  151,  302],\n",
      "        [ 217,  174,  355,  344],\n",
      "        [ 344,  147,  498,  310],\n",
      "        [ 810,   70, 1024,  502]]), 'labels': tensor([1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:30 - INFO - object_detection_viewer.py:42:get_data - Input sample: 005.jpg\n",
      "Shape: torch.Size([3, 480, 640])\n",
      "2023-06-08 18:16:30 - INFO - object_detection_viewer.py:64:get_target - Target sample: 005.json\n",
      "{'boxes': tensor([[ 51, 202, 203, 380],\n",
      "        [232,  80, 316, 169],\n",
      "        [420, 204, 555, 341],\n",
      "        [219, 202, 395, 450],\n",
      "        [ 13,  37, 152, 207],\n",
      "        [370, 101, 458, 180]]), 'labels': tensor([1, 1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:30 - INFO - object_detection_viewer.py:42:get_data - Input sample: 006.jpg\n",
      "Shape: torch.Size([3, 912, 1368])\n",
      "2023-06-08 18:16:30 - INFO - object_detection_viewer.py:64:get_target - Target sample: 006.json\n",
      "{'boxes': tensor([[323, 111, 712, 665]]), 'labels': tensor([1]), 'scores': tensor([0])}\n",
      "2023-06-08 18:16:31 - INFO - object_detection_viewer.py:42:get_data - Input sample: 007.jpg\n",
      "Shape: torch.Size([3, 1500, 1085])\n",
      "2023-06-08 18:16:31 - INFO - object_detection_viewer.py:64:get_target - Target sample: 007.json\n",
      "{'boxes': tensor([], dtype=torch.int64), 'labels': tensor([], dtype=torch.int64), 'scores': tensor([], dtype=torch.int64)}\n",
      "2023-06-08 18:16:32 - INFO - object_detection_viewer.py:42:get_data - Input sample: 000.jpg\n",
      "Shape: torch.Size([3, 408, 612])\n",
      "2023-06-08 18:16:32 - INFO - object_detection_viewer.py:64:get_target - Target sample: 000.json\n",
      "{'boxes': tensor([[137,  74, 269, 231],\n",
      "        [512,  78, 612, 195],\n",
      "        [233,  65, 368, 196],\n",
      "        [  0, 129, 144, 294],\n",
      "        [380,  41, 497, 182]]), 'labels': tensor([1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:33 - INFO - object_detection_viewer.py:42:get_data - Input sample: 007.jpg\n",
      "Shape: torch.Size([3, 1500, 1085])\n",
      "2023-06-08 18:16:33 - INFO - object_detection_viewer.py:64:get_target - Target sample: 007.json\n",
      "{'boxes': tensor([], dtype=torch.int64), 'labels': tensor([], dtype=torch.int64), 'scores': tensor([], dtype=torch.int64)}\n",
      "2023-06-08 18:16:33 - INFO - object_detection_viewer.py:42:get_data - Input sample: 000.jpg\n",
      "Shape: torch.Size([3, 408, 612])\n",
      "2023-06-08 18:16:33 - INFO - object_detection_viewer.py:64:get_target - Target sample: 000.json\n",
      "{'boxes': tensor([[137,  74, 269, 231],\n",
      "        [512,  78, 612, 195],\n",
      "        [233,  65, 368, 196],\n",
      "        [  0, 129, 144, 294],\n",
      "        [380,  41, 497, 182]]), 'labels': tensor([1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:34 - INFO - object_detection_viewer.py:42:get_data - Input sample: 001.jpg\n",
      "Shape: torch.Size([3, 300, 600])\n",
      "2023-06-08 18:16:35 - INFO - object_detection_viewer.py:64:get_target - Target sample: 001.json\n",
      "{'boxes': tensor([[334,  72, 425, 210],\n",
      "        [417,  17, 555, 172],\n",
      "        [ 53,   2, 168, 115],\n",
      "        [ 57, 106, 152, 233],\n",
      "        [186,  61, 253, 164],\n",
      "        [220,  43, 309, 152]]), 'labels': tensor([1, 1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:35 - INFO - object_detection_viewer.py:42:get_data - Input sample: 000.jpg\n",
      "Shape: torch.Size([3, 408, 612])\n",
      "2023-06-08 18:16:35 - INFO - object_detection_viewer.py:64:get_target - Target sample: 000.json\n",
      "{'boxes': tensor([[137,  74, 269, 231],\n",
      "        [512,  78, 612, 195],\n",
      "        [233,  65, 368, 196],\n",
      "        [  0, 129, 144, 294],\n",
      "        [380,  41, 497, 182]]), 'labels': tensor([1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0])}\n",
      "2023-06-08 18:16:36 - INFO - object_detection_viewer.py:42:get_data - Input sample: 001.jpg\n",
      "Shape: torch.Size([3, 300, 600])\n",
      "2023-06-08 18:16:36 - INFO - object_detection_viewer.py:64:get_target - Target sample: 001.json\n",
      "{'boxes': tensor([[334,  72, 425, 210],\n",
      "        [417,  17, 555, 172],\n",
      "        [ 53,   2, 168, 115],\n",
      "        [ 57, 106, 152, 233],\n",
      "        [186,  61, 253, 164],\n",
      "        [220,  43, 309, 152]]), 'labels': tensor([1, 1, 1, 1, 1, 1]), 'scores': tensor([0, 0, 0, 0, 0, 0])}\n"
     ]
    }
   ],
   "source": [
    "# visualize predictions\n",
    "datasetviewer_prediction = ObjectDetectionViewer(\n",
    "    dataset=dataset_prediction, color_mapping=color_mapping\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3b26ebc-1df1-4829-9e72-1bd2452976e7",
   "metadata": {},
   "source": [
    "## Experiment with Non-maximum suppression (nms) and score-thresholding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b440cbf-94fa-439c-ac9f-134959617f4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "## currently not available"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}