test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18/18 [00:06<00:00,  3.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Taken using multithreading:6.007270812988281\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor, as_completed\n",
    "from functools import partial\n",
    "import cv2\n",
    "import time\n",
    "\n",
    "img_list = [\n",
    "    \"apple.jpeg\",\n",
    "    \"banana.jpeg\",\n",
    "    \"pineapple.jpeg\",\n",
    "    \"mango.jpeg\",\n",
    "    \"cherry.jpeg\",\n",
    "    \"blueberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "    \"raspberry.jpeg\",\n",
    "]\n",
    "\n",
    "\n",
    "def read_image(img_name, init_path):\n",
    "    # img = cv2.imread(init_path + \"\\\\\" + img_name)\n",
    "    time.sleep(1)\n",
    "    # print(img_name)\n",
    "\n",
    "\n",
    "start = time.time()\n",
    "with ThreadPoolExecutor(max_workers=3) as executor:\n",
    "    list(tqdm(executor.map(\n",
    "        partial(\n",
    "            read_image,\n",
    "            init_path=\"C:\\code\\images_medium\",\n",
    "        ),\n",
    "        img_list,\n",
    "    ), total=len(img_list)))\n",
    "\n",
    "end = time.time()\n",
    "print(\"Time Taken using multithreading:{}\".format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:\tError downloading ./ok: unknown url type: './ok'. Moving on\n",
      "ERROR:root:\tError downloading ./oui: unknown url type: './oui'. Moving on\n",
      "ERROR:root:\tError downloading ./ok: unknown url type: './ok'. Cleaning up mp4\n",
      "ERROR:root:\tError downloading ./oui: unknown url type: './oui'. Cleaning up mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "coucoucoucou ok tmp ./\n",
      " oui tmp ./\n",
      "<generator object Executor.map.<locals>.result_iterator at 0x7f668e43c2e0>\n"
     ]
    }
   ],
   "source": [
    "from utils.download_dataset import download_file\n",
    "\n",
    "l = ['oui', 'ok']\n",
    "\n",
    "with ThreadPoolExecutor(max_workers=64) as executor:\n",
    "    results = executor.map(\n",
    "        partial(\n",
    "            download_file,\n",
    "            output_dir=\"tmp\",\n",
    "            base_dir=\"./\",\n",
    "        ),\n",
    "        l,\n",
    "    )\n",
    "    print(results)\n"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}