programming/py_multiprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Python Multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spawned 0 1\n",
      "Spawned 1 2\n",
      "Spawned 2 3\n",
      "Spawned 3 4\n",
      "Spawned 4 5\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing\n",
    "\n",
    "def spawn(num, num2):\n",
    "    print('Spawned {} {}'.format(num, num2))\n",
    "    \n",
    "for i in range(5):\n",
    "    p = multiprocessing.Process(target=spawn, args=(i,i+1))\n",
    "    p.start()\n",
    "    # wait until job finishes\n",
    "    p.join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38]\n"
     ]
    }
   ],
   "source": [
    "from multiprocessing import Pool\n",
    "\n",
    "def job(num):\n",
    "    return num * 2\n",
    "\n",
    "p = Pool(processes=20)\n",
    "data = p.map(job, range(20))\n",
    "p.close()\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "import time\n",
    "\n",
    "class ProcessHelper:\n",
    "    def __init__(self, num_processes=4):\n",
    "        self.num_processes = num_processes\n",
    "    \n",
    "    def split_list(self, data_list):\n",
    "        list_of_lists = []\n",
    "        for i in range(0, len(data_list), self.num_processes):\n",
    "            list_of_lists.append(data_list[i:i+self.num_processes])\n",
    "        return list_of_lists\n",
    "    \n",
    "    def map_reduce(self, function, data_list):\n",
    "        \n",
    "        processes = multiprocessing.Pool(processes=self.num_processes)\n",
    "        results_list = processes.map(function, data_list)\n",
    "        processes.close()\n",
    "        return results_list\n",
    "        '''\n",
    "        split_data = self.split_list(data_list)\n",
    "        processes = multiprocessing.Pool(processes=self.num_processes)\n",
    "        results_list_of_lists = processes.map(function, split_data)\n",
    "        processes.close()\n",
    "        results_list = [item for sublist in results_list_of_lists for item in sublist]\n",
    "        return results_list\n",
    "        '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.48835301399231\n",
      "[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]\n"
     ]
    }
   ],
   "source": [
    "def job(num_list):\n",
    "    # return [i*i for i in num_list]\n",
    "    return num_list * num_list\n",
    "\n",
    "data = range(100000000)\n",
    "\n",
    "'''\n",
    "p = ProcessHelper(6)\n",
    "time1 = time.time()\n",
    "result = p.map_reduce(job, data)\n",
    "time2 = time.time()\n",
    "print(time2-time1)\n",
    "print(result[:100])\n",
    "'''\n",
    "n=4\n",
    "time1 = time.time()\n",
    "processes = multiprocessing.Pool(processes=n)\n",
    "results_list = processes.map(func=job, iterable=data, chunksize=10000)\n",
    "processes.close()\n",
    "time2 = time.time()\n",
    "print(time2-time1)\n",
    "print(results_list[0:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results\n",
    "\n",
    "Wrote a helper class, then realised that I could automate this with the operable and chunk size argument. Timed function and realised that a chunk size of 10000 was optimal - 100 or 100k was slower."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}