switch to using parallel from fastai

resize.ipynb

@@ -9,11 +9,7 @@
     "%matplotlib inline\n",
     "import matplotlib.pyplot as plt\n",
     "from fastai.vision import *\n",
-    "from fastai.metrics import accuracy\n",
-    "from fastai.basic_data import *\n",
-    "from skimage.util import montage\n",
-    "import pandas as pd\n",
-    "from torch import optim\n",
+    "from fastai.core import parallel\n",
     "import re\n",
     "from pathlib import Path\n",
     "\n",
@@ -35,7 +31,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "SZ = 224"
+    "SZ = 224\n",
+    "NUM_WORKERS = 12"
    ]
   },
   {
@@ -63,42 +60,63 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from concurrent.futures import ThreadPoolExecutor"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def resize_img(path):\n",
+    "def resize_img(path, _):\n",
     "    PIL.Image.open(path).resize((SZ,SZ), resample=PIL.Image.BICUBIC).save((PATH/f'{path.parent.name}-{SZ}'/path.name))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "        <style>\n",
+       "            /* Turns off some styling */\n",
+       "            progress {\n",
+       "                /* gets rid of default border in Firefox and Opera. */\n",
+       "                border: none;\n",
+       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
+       "                background-size: auto;\n",
+       "            }\n",
+       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
+       "                background: #F44336;\n",
+       "            }\n",
+       "        </style>\n",
+       "      <progress value='33321' class='' max='33321', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      100.00% [33321/33321 00:26<00:00]\n",
+       "    </div>\n",
+       "    "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "CPU times: user 4min 38s, sys: 10.8 s, total: 4min 48s\n",
-      "Wall time: 34.3 s\n"
+      "CPU times: user 10.3 s, sys: 1.61 s, total: 11.9 s\n",
+      "Wall time: 28 s\n"
      ]
     }
    ],
    "source": [
     "%%time\n",
+    "\n",
     "files = list((PATH/'train').iterdir()) + list((PATH/'test').iterdir())\n",
-    "with ThreadPoolExecutor(12) as e: e.map(resize_img, files)"
+    "parallel(resize_img, files, NUM_WORKERS)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -107,7 +125,7 @@
        "(25361, 7960)"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -118,7 +136,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -127,7 +145,7 @@
        "(25361, 7960)"
       ]
      },
-     "execution_count": 13,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }