Merge_predict.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict script for Merge model, predict the D and R matrice, and visualize resule."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.merge_modules import MergeModel\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch\n",
    "import json\n",
    "from dataset.dataset import ImageDataset\n",
    "from PIL import Image\n",
    "import cv2\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init the Merge model\n",
    "net = MergeModel(3).cuda()\n",
    "cudnn.benchmark = True\n",
    "cudnn.deterministic = True\n",
    "net = torch.nn.DataParallel(net).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load saved checkpoint \n",
    "net.load_state_dict(torch.load('Merge_model.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# change the model to eval mode\n",
    "net.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init dataset\n",
    "folder = 'validation'\n",
    "with open('D:/dataset/table/table_line/Split1/'+ folder+'_merge_dict.json', 'r') as f:\n",
    "    labels = json.load(f)\n",
    "dataset = ImageDataset('D:/dataset/table/table_line/Split1/'+ folder+'_input', labels, 8, scale=0.25,mode='merge')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "img, label, arc = dataset[index]\n",
    "index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# predict \n",
    "input_img = img.unsqueeze(0)\n",
    "arc_c = [[torch.Tensor([y]) for y in x] for x in arc]\n",
    "pred = net(input_img,arc_c)\n",
    "u,d,l,r = pred # up, down, left, right\n",
    "# calculate D and R matrice, \n",
    "D = 0.5 * u[:, :-1, :] * d[:, 1:, :] + 0.25 * (u[:, :-1, :] + d[:, 1:, :])\n",
    "R = 0.5 * r[:, :, :-1] * l[:, :, 1:] + 0.25 * (r[:, :, :-1] + l[:, :, 1:])\n",
    "D = D[0].detach().cpu().numpy()\n",
    "R = R[0].detach().cpu().numpy()\n",
    "D[D>0.5] = 1\n",
    "D[D<=0.5] = 0\n",
    "R[R>0.5] = 1\n",
    "R[R<=0.5] = 0\n",
    "\n",
    "rows, columns = arc\n",
    "h,w = img[2].shape\n",
    "rows = [round(h*x) for x in rows]\n",
    "columns = [round(w*x) for x in columns]\n",
    "rows = [0] + rows + [h]\n",
    "columns = [0] + columns +[w]\n",
    "\n",
    "# draw lines on the original image\n",
    "draw_img = img[2].numpy()*255.\n",
    "draw_img = cv2.cvtColor(draw_img, cv2.COLOR_GRAY2RGB)\n",
    "for i in range(R.shape[0]):\n",
    "    for j in range(R.shape[1]):\n",
    "        if R[i,j] == 0:\n",
    "            pts1 = (columns[j+1],rows[i])\n",
    "            pts2 = (columns[j+1],rows[i+1])\n",
    "            draw_img = cv2.line(draw_img, pts1,pts2,(255.,0,0),2)\n",
    "for i in range(D.shape[0]):\n",
    "    for j in range(D.shape[1]):\n",
    "        if D[i,j] == 0:\n",
    "            pts1 = (columns[j],rows[i+1])\n",
    "            pts2 = (columns[j+1],rows[i+1])\n",
    "            draw_img = cv2.line(draw_img, pts1,pts2,(255.,0,0),2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# visualize original image\n",
    "Image.fromarray(img[2].numpy()*255.).convert('L')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# visualize merged image\n",
    "Image.fromarray(np.array(draw_img,dtype=np.uint8))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}