Logistic Regression For Multi Class Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Took the dataset from Andrew NG machine learning course. Goal here to classify the handwritten digits from 0 - 10. Will use Logistic Regression influenced by concept given by Andrew NG in his machine learning course.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### There are 5000 training examples in ex3data1.mat, where each training example is a 20 pixel by 20 pixel grayscale image of the digit. The 20 by 20 grid of pixels is unrolled\" into a 400-dimensional vector. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.io import loadmat\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 400)\n",
      "(5000, 1)\n"
     ]
    }
   ],
   "source": [
    "data = loadmat(\"data\\ex3data1.mat\")\n",
    "print(data['X'].shape)\n",
    "print(data['y'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Plot one image from input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def show(img, title:None):\n",
    "    plt.imshow(img, cmap='gray')\n",
    "    if(title is not None): plt.title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQgAAAEICAYAAACj9mr/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEdFJREFUeJzt3X+MVeWdx/H3Z66oBOlKly3+LnZLMKypsw2y2nU3Wlv5\nsVqqaSh0q65rA23UqFnTsNts6z/VJsYaWQyWthNs0yqaXVqSEhGNWdu0taJLEVTWkWhlFmGtWfw1\nqAzf/WMOZhzu4zz3nvtrrp9XQub8+N5znpMZPnPOvc88jyICM7NqetrdADPrXA4IM0tyQJhZkgPC\nzJIcEGaW5IAwsyQHhJklOSAsi6SQ9Iakb2fWXynp9eJ1H292+6w5HBBWizMi4huHViR9WtITkl6V\ntFPS0kP7IuKHEXFMe5ppjeKAsLpImgCsA74H/AnwReC7ks5oa8OsoRwQVq8PAx8CfhzDHgOeBma1\nt1nWSA4Iq0tE7AHuBq6QVJF0NvBR4FftbZk10hHtboCNa3cDPwBuL9a/FhEvtrE91mC+g7C6SDoN\nWAtcBhwJ/AXwdUl/19aGWUM5IKxepwM7ImJjRByMiB3AL4D5bW6XNZADwur1X8DHi486JenPgQuB\nrW1ulzWQ34OwukTEc5KuBFYw/ObkPuAnDL8nYV1CHlHKckjaD7wFrIiIf82ovwK4DTgamBURO5vc\nRGsCB4SZJfk9CDNLckCYWVJHvkkpKXp6nF1mzXLw4EEiQmPVdWRA9PT0MHHixHY3w6xrDQ4OZtX5\n17SZJZUKCEnzJO2Q1C9peZX9krSi2L9V0ifLnM/MWqvugJBUAe5guGvtLGCJpNF/6jsfmFH8Wwqs\nqvd8ZtZ6Ze4g5gD9EbEzIt4G7gEWjqpZCPyoGC/gt8Cxko4vcU4za6EyAXEiMPJPe3cV22qtAUDS\nUkmbJW125y2zztAxn2JExGpgNUClUnFCmHWAMncQA8DJI9ZPKrbVWmNmHapMQDwGzJB0qqQjgcXA\n+lE164HLik8zzgL2RcTuEuc0sxaq+xEjIg5IuhrYCFSAvojYLumrxf47gQ3AAqAfeBO4onyTzaxV\nOvKvOSuVSrgnpVnzDA4OMjQ0NGZXa/ekNLMkB4SZJTkgzCzJAWFmSQ4IM0tyQJhZkgPCzJIcEGaW\n5IAwsyQHhJklOSDMLMkBYWZJDggzS3JAmFmSA8LMkhwQZpbkgDCzJAeEmSWVmVnrZEkPS3pK0nZJ\n11apOVfSPklbin/fLNdcM2ulMvNiHAD+KSKekDQZeFzSpoh4alTdLyPiwhLnMbM2qfsOIiJ2R8QT\nxfJrwNMkZs0ys/GpITNrSZoO/CXwaJXdn5K0leEJc26IiO2JYyxleIJfpDEH2zU7TCeO0N4I7fz/\nUHrYe0nHAP8JfDsi/mPUvg8BByPidUkLgNsjYsZYx/Sw91YPB0S+lgx7L2kC8O/AT0aHA0BEvBoR\nrxfLG4AJkqaWOaeZtU6ZTzEE/BB4OiK+m6g5rqhD0pzifH+s95xm1lpl3oP4a+BS4ElJW4pt/wKc\nAu9OvfcF4GuSDgCDwOLo1vtAsy7kqfesa3Tiz3IjjNv3IMysuzkgzCzJAWFmSQ4IM0tyQJhZUkO6\nWps1Sy2fTAwNDWXXViqVrLqenvzfoe1uazP4DsLMkhwQZpbkgDCzJAeEmSU5IMwsyQFhZkkOCDNL\nckCYWZIDwsyS3JPSWq5Z4zaceeaZ2bVf/vKXs+qmTJmSfcy33noru3bFihXZtdu2bcuubXSvS99B\nmFmSA8LMksqOav28pCeLafU2V9kvSSsk9UvaKumTZc5nZq3ViPcgzouIlxP75gMzin9/BawqvprZ\nONDsR4yFwI9i2G+BYyUd3+RzmlmDlA2IAB6U9Hgxdd5oJwIvjljfRWL+TklLJW2WtLlbRyc2G2/K\nPmKcExEDkj4CbJL0TEQ8Us+BImI1sBqGh70v2S4za4BSdxARMVB83QusA+aMKhkATh6xflKxzczG\ngTJT702SNPnQMnABMLpHx3rgsuLTjLOAfRGxu+7WmllLlXnEmAasK2b9OQL4aUTcL+mr8O7UexuA\nBUA/8CZwRbnmmlkreeq9FmrWoKa1HLeWadyaMeUbwDvvvJNde/HFF2fX3nzzzdm1kydPzqrbu3dv\n9jGPO+647Npf//rX2bWLFi3Krp0wYUJWnafeM7PSHBBmluSAMLMkB4SZJTkgzCzJAWFmSQ4IM0ty\nQJhZkgPCzJIcEGaW5FGtGyC3q/PBgwezj7lgwYLs2lpGc/7Zz36WXfvCCy9k19YyovNll12WXVtL\n9+lHH300u/aaa67JqjvttNOyj9nX15dd29MzPn43j49WmllbOCDMLMkBYWZJDggzS3JAmFmSA8LM\nkhwQZpZUZtDamcWUe4f+vSrpulE150raN6Lmm+WbbGatUndHqYjYAfQCSKowPJz9uiqlv4yIC+s9\nj5m1T6MeMc4HnouI/K53ZtbxGtXVejFwd2LfpyRtZfgO44aI2F6tqJi6b2mx3KBm1a8ZI0V/5Stf\nyT7mtddem117yimnZNdecskl2bV/+MMfsmv379+fXXvWWWdl17700kvZtTfccEN2be737Kqrrso+\nZi3dzVetWpVd287/D6XvICQdCXwOuK/K7ieAUyLiE8C/Ack/BIiI1RExOyJmd0JAmFljHjHmA09E\nxJ7ROyLi1Yh4vVjeAEyQNLUB5zSzFmhEQCwh8Xgh6TgVtwOS5hTn+2MDzmlmLVDqPYhiTs7PAstG\nbBs59d4XgK9JOgAMAoujE6fyMrOqSgVERLwB/OmobXeOWF4JrCxzDjNrH/ekNLMkB4SZJTkgzCzJ\nAWFmSQ4IM0vyqNYNUKlUsuoWLVqUfcxaRsC+7rrrxi4qfOlLX8quPfvss7NrJ06cmF1bi40bN2bX\nzp07N7v20ksvzao76qijso+5bNmysYsKDz/8cHZt7s9XM/gOwsySHBBmluSAMLMkB4SZJTkgzCzJ\nAWFmSQ4IM0tyQJhZkgPCzJIcEGaWpE4c4KlSqUSzuu7mqqWr86RJk7LqHnjggexjPvfcc9m1ud2G\nAXp7e7Nr16xZk107ffr07Np9+/Zl19by81nLyNq5I1DfeOON2cdcu3Ztdu3RRx+dXduMQZwHBwcZ\nGhoa88C+gzCzpDEDQlKfpL2Sto3Y9mFJmyQ9W3ydknjtPEk7JPVLWt7IhptZ8+XcQawB5o3athx4\nKCJmAA8V6+9RTMd3B8PD4s8ClkiaVaq1ZtZSYwZERDwCvDJq80LgrmL5LuDzVV46B+iPiJ0R8TZw\nT/E6Mxsn6n0PYlpE7C6WXwKmVak5EXhxxPquYpuZjROlB4yJiJBU+qOQTpub08zqv4PYI+l4gOLr\n3io1A8DJI9ZPKrZV5bk5zTpPvQGxHri8WL4c+HmVmseAGZJOLSb4XVy8zszGiZyPOe8GfgPMlLRL\n0pXAd4DPSnoW+EyxjqQTJG0AiIgDwNXARuBp4N6I2N6cyzCzZhjzPYiIWJLYdX6V2v8BFoxY3wBs\nqLt1ZtZWHtU6oZYuvieccEJW3dSpU7OPedNNN2XXHnPMMdm1119/fXbtzJkzs2vXrVuXXbt69ers\n2lqu7eWXX86ufeWV0Z/cV7djx47sY7a7+3QzuKu1mSU5IMwsyQFhZkkOCDNLckCYWZIDwsySHBBm\nluSAMLMkB4SZJTkgzCzJXa1bqJaRsk8//fTs2rlz52bXXnTRRdm19913X3ZtLaM/P/XUU9m1PT35\nv8OaUXvEEfn/RcZL9+la+A7CzJIcEGaW5IAwsyQHhJklOSDMLMkBYWZJDggzS6p3bs5bJD0jaauk\ndZKOTbz2eUlPStoiaXMjG25mzVfv3JybgNMj4hPAfwP//D6vPy8ieiNidn1NNLN2qWtuzoh4oBjW\nHuC3DE+KY2ZdphFdrf8RWJvYF8CDkoaA70VEcjjjTpt6r5Zu0UNDQ1l1mzfnP2UtX37YhOmlzw9w\n++23Z9euXLkyu7aWEaUnTZqUXWvtVSogJH0DOAD8JFFyTkQMSPoIsEnSM8UdyWGK8FgNUKlUSs/1\naWbl1f0phqR/AC4E/j4Sk0hExEDxdS+wDphT7/nMrPXqCghJ84CvA5+LiDcTNZMkTT60DFwAbKtW\na2adqd65OVcCkxl+bNgi6c6i9t25OYFpwK8k/R74HfCLiLi/KVdhZk1R79ycP0zUvjs3Z0TsBM4o\n1Tozayv3pDSzJAeEmSU5IMwsyQFhZkkOCDNL+kCNar1///7s2vPPPz+79qabbsqqmz59evYxX3vt\ntezavr6+7Npbbrklu/aNN97Irq1l9GcbP3wHYWZJDggzS3JAmFmSA8LMkhwQZpbkgDCzJAeEmSU5\nIMwsyQFhZkkfqO5viZHxqpo8eXJ27e7du7PqNmzYMHZR4d57782u7e/vz66thXtHmu8gzCzJAWFm\nSfVOvXejpIFiPMotkhYkXjtP0g5J/ZLyJ3ows45Q79R7ALcVU+r1RsRhD9eSKsAdwHxgFrBE0qwy\njTWz1qpr6r1Mc4D+iNgZEW8D9wAL6ziOmbVJmfcgrilm9+6TNKXK/hOBF0es7yq2VSVpqaTNkjbX\n8mmDmTVPvQGxCvgY0AvsBm4t25CIWB0RsyNidifMzWlmdQZEROyJiKGIOAh8n+pT6g0AJ49YP6nY\nZmbjRL1T7x0/YvViqk+p9xgwQ9Kpko4EFgPr6zmfmbXHmF3liqn3zgWmStoFfAs4V1IvEMDzwLKi\n9gTgBxGxICIOSLoa2AhUgL6I2N6UqzCzplAnviFYqVRi4sSJDT9uLddaqVSya4eGhhp+/lpqe3ry\nbwT9/o4BDA4OMjQ0NOYPg3tSmlmSA8LMkhwQZpbkgDCzJAeEmSU5IMwsyQFhZkkOCDNLckCYWZID\nwsySPlDDFtfSzTi3+3Szzl9L92mzZvFPoZklOSDMLMkBYWZJDggzS3JAmFmSA8LMkhwQZpaUMyZl\nH3AhsDciTi+2rQVmFiXHAv8XEb1VXvs88BowBByIiNkNareZtUBOR6k1wErgR4c2RMQXDy1LuhXY\n9z6vPy8iXq63gWbWPmMGREQ8Iml6tX0a7hq4CPh0Y5tlZp2gbFfrvwH2RMSzif0BPChpCPheRKxO\nHUjSUmBpsVyyWeV1QhvM2q1sQCwB7n6f/edExICkjwCbJD1TTAZ8mCI8VsPwsPcl22VmDVD3pxiS\njgAuAdamaiJioPi6F1hH9Sn6zKxDlfmY8zPAMxGxq9pOSZMkTT60DFxA9Sn6zKxDjRkQxdR7vwFm\nStol6cpi12JGPV5IOkHShmJ1GvArSb8Hfgf8IiLub1zTzazZPlBT75nZME+9Z2alOSDMLMkBYWZJ\nDggzS3JAmFmSA8LMkhwQZpbkgDCzJAeEmSU5IMwsyQFhZkkOCDNLckCYWZIDwsySHBBmluSAMLOk\njhwwRtL/Ai+M2jwV6Mb5Nbr1uqB7r60bruujEfFnYxV1ZEBUI2lzN87M1a3XBd17bd16XdX4EcPM\nkhwQZpY0ngIiOSvXONet1wXde23del2HGTfvQZhZ642nOwgzazEHhJkldXxASJonaYekfknL292e\nRpL0vKQnJW2RtLnd7amXpD5JeyVtG7Htw5I2SXq2+DqlnW2sV+LabpQ0UHzftkha0M42NlNHB4Sk\nCnAHMB+YBSyRNKu9rWq48yKid5x/rr4GmDdq23LgoYiYATxUrI9Hazj82gBuK75vvRGxocr+rtDR\nAcHwbOD9EbEzIt4G7gEWtrlNNkpEPAK8MmrzQuCuYvku4PMtbVSDJK7tA6PTA+JE4MUR67uKbd0i\ngAclPS5pabsb02DTImJ3sfwSw5M5d5NrJG0tHkHG5eNTjk4PiG53TkT0MvwIdZWkv213g5ohhj9L\n76bP01cBHwN6gd3Are1tTvN0ekAMACePWD+p2NYVImKg+LoXWMfwI1W32CPpeIDi6942t6dhImJP\nRAxFxEHg+3TX9+09Oj0gHgNmSDpV0pHAYmB9m9vUEJImSZp8aBm4ANj2/q8aV9YDlxfLlwM/b2Nb\nGupQ8BUupru+b+9xRLsb8H4i4oCkq4GNQAXoi4jtbW5Wo0wD1kmC4e/DTyPi/vY2qT6S7gbOBaZK\n2gV8C/gOcK+kKxn+0/1F7Wth/RLXdq6kXoYfm54HlrWtgU3mrtZmltTpjxhm1kYOCDNLckCYWZID\nwsySHBBmluSAMLMkB4SZJf0/ZE4Quqx+he0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2bfe47b9518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#reshape by 20*20\n",
    "image = np.reshape(data['X'][4000],(20,20))\n",
    "image.shape\n",
    "show(image, data['y'][4000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Let   me copy the Logisitc Regression and gradient desent Class which i done in my previous session and extend the class to support multi classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "class LogisticRegression():\n",
    "    def __init__(self, X, y , X_valid, y_valid, lr,epochs, reg = False,useoptimizer = False,ismultiClass = False):\n",
    "        self.X,self.y,  self.X_valid,self.y_valid,self.lr ,self.epochs,self.reg,self.useoptimizer, self.ismultiClass= X, y , X_valid, y_valid, lr, epochs, reg, useoptimizer,ismultiClass\n",
    "        if(not ismultiClass):            \n",
    "            self.optim = GradientDesent( X, y , X_valid, y_valid, lr,epochs, reg,useoptimizer, self.sigmoid)\n",
    "   \n",
    "    def sigmoid(self, z):    \n",
    "        return 1/(1+np.exp(-z))  \n",
    "    \n",
    "    def fit(self):\n",
    "        if(self.ismultiClass):\n",
    "            self.fitMulti()\n",
    "        else:\n",
    "            w = self.optim.optimize()\n",
    "            h = sigmoid(np.dot(self.X, w))\n",
    "            h_valid = sigmoid(np.dot(self.X_valid, w))\n",
    "            train_preds = [1 if i > 0.5 else 0 for i in h]\n",
    "            val_preds = [1 if i > 0.5 else 0 for i in h_valid]\n",
    "            train_accuracy =  accuracy_score(self.y , train_preds)\n",
    "            val_accuracy =  accuracy_score(self.y_valid , val_preds)\n",
    "\n",
    "            print(f'Train Accuracy: {train_accuracy} Val Accuracy: {val_accuracy}')   \n",
    "        \n",
    "        \n",
    "    def fitMulti(self):\n",
    "        labelCount = len(np.unique(self.y))\n",
    "        print(f'Total label Count: {labelCount}') \n",
    "        w = np.zeros((labelCount, self.X.shape[1]))\n",
    "                     \n",
    "        for i in range(1, labelCount+1):\n",
    "            print(f'Train LG for class : {i}')\n",
    "            y_change = np.array([1 if i == label else 0 for label in self.y])\n",
    "            y_valid_change = np.array([1 if i == label else 0 for label in self.y_valid])            \n",
    "            optim = GradientDesent( self.X, y_change , self.X_valid, y_valid_change, self.lr, self.epochs, self.reg, self.useoptimizer, self.sigmoid)\n",
    "            w[i-1:] = optim.optimize()                     \n",
    "        \n",
    "        train_preds = np.argmax(self.sigmoid(np.dot(self.X, w.T)) ,axis = 1) + 1\n",
    "        val_preds = np.argmax(self.sigmoid(np.dot(self.X_valid, w.T)) ,axis = 1) + 1         \n",
    "        \n",
    "        train_accuracy =  accuracy_score(self.y , train_preds)\n",
    "        val_accuracy =  accuracy_score(self.y_valid , val_preds)        \n",
    "        print(f'Train Accuracy: {train_accuracy} Val Accuracy: {val_accuracy}')\n",
    "                           \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy.optimize as opt\n",
    "class GradientDesent():\n",
    "    \n",
    "    def __init__(self, X, y , X_valid, y_valid, lr, epochs, reg = False, useoptimizer = False, activate_fn = None):\n",
    "        self.X,self.y,  self.X_valid,self.y_valid,self.lr ,self.epochs, self.reg, self.useoptimizer, self.activate_fn  = X, y , X_valid, y_valid, lr,epochs, reg,useoptimizer,activate_fn\n",
    "            \n",
    "    def optimize(self):\n",
    "        w = np.zeros(X.shape[1])\n",
    "              \n",
    "        \n",
    "        if(self.useoptimizer):\n",
    "            result = opt.fmin_tnc(func=self.cost, x0=w, fprime=self.gradient,args=(self.X, self.y)) \n",
    "            train_loss= self.cost(result[0], self.X, self.y)\n",
    "            val_loss = self.cost(result[0], self.X_valid, self.y_valid)  \n",
    "            print(f'Runned Spicy Optimizer > Train Loss: {train_loss} Val Loss: {val_loss}')          \n",
    "            return result[0]             \n",
    "            \n",
    "        train_cost = np.zeros(self.epochs)\n",
    "        val_cost = np.zeros(self.epochs)      \n",
    "\n",
    "        \n",
    "        for i in range(self.epochs):\n",
    "            grad = self.gradient(w, self.X, self.y)\n",
    "            w = np.subtract(w , np.multiply(grad , self.lr))\n",
    "            train_loss= self.cost(w, self.X, self.y)\n",
    "            val_loss = self.cost(w, self.X_valid, self.y_valid)         \n",
    "            train_cost[i] = train_loss\n",
    "            val_cost[i] = val_loss\n",
    "            print(f'Epochs: {i} Train Loss: {train_loss} Val Loss: {val_loss}')   \n",
    "            \n",
    "        \n",
    "        return w\n",
    "        \n",
    "    def cost(self,w, X, y):\n",
    "        h = np.dot(X, w)\n",
    "        m = len(X)       \n",
    "        if(self.activate_fn != None):  h = self.activate_fn(h)\n",
    "        \n",
    "        loss= (np.dot(-y, np.log(h))) - (np.dot(1-y, np.log(1-h)))\n",
    "        loss = np.sum(loss)/m\n",
    "        \n",
    "        if(self.reg):           \n",
    "            loss = loss + ((self.lr/2*m) * np.sum(np.power(w,2)))\n",
    "        return loss\n",
    "            \n",
    "    \n",
    "    def gradient(self,w,X, y):\n",
    "        h = np.dot(X, w)      \n",
    "        if(self.activate_fn != None):  h = self.activate_fn(h)     \n",
    "        m = len(X)\n",
    "        loss= h - y\n",
    "        gradient = np.dot(X.T,loss)/m\n",
    "        if(self.reg): gradient = gradient + np.dot((self.lr/m), w)      \n",
    "        return gradient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = np.insert(data['X'], 0,  np.ones(data['X'].shape[0]), axis =1)\n",
    "y =  data['y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4250, 401)\n",
      "(750, 401)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "            X,y, test_size=0.15, random_state=42)\n",
    "print(X_train.shape)\n",
    "print(X_valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total label Count: 10\n",
      "Train LG for class : 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Program Files\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:39: RuntimeWarning: divide by zero encountered in log\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Runned Spicy Optimizer > Train Loss: 0.002923041733598084 Val Loss: 0.138538817864986\n",
      "Train LG for class : 2\n",
      "Runned Spicy Optimizer > Train Loss: 0.028470023386103632 Val Loss: 0.2696736429306643\n",
      "Train LG for class : 3\n",
      "Runned Spicy Optimizer > Train Loss: 0.03293049777668425 Val Loss: nan\n",
      "Train LG for class : 4\n",
      "Runned Spicy Optimizer > Train Loss: 0.02734732825590251 Val Loss: 0.0907335536576923\n",
      "Train LG for class : 5\n",
      "Runned Spicy Optimizer > Train Loss: 0.027994746815151718 Val Loss: 0.23885494175167288\n",
      "Train LG for class : 6\n",
      "Runned Spicy Optimizer > Train Loss: 0.0055219151298216635 Val Loss: 0.17925622430139207\n",
      "Train LG for class : 7\n",
      "Runned Spicy Optimizer > Train Loss: 0.019568046980762508 Val Loss: 0.1346285309284556\n",
      "Train LG for class : 8\n",
      "Runned Spicy Optimizer > Train Loss: 0.055570550206426934 Val Loss: 0.24359708820847892\n",
      "Train LG for class : 9\n",
      "Runned Spicy Optimizer > Train Loss: 0.04223319830224293 Val Loss: 0.32487084597910837\n",
      "Train LG for class : 10\n",
      "Runned Spicy Optimizer > Train Loss: 0.007414052581755591 Val Loss: 0.017317162538430998\n",
      "Train Accuracy: 0.9788235294117648 Val Accuracy: 0.8706666666666667\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegression(X_train, y_train,X_valid,y_valid,  lr = 0.001, epochs =5000,useoptimizer=True,ismultiClass=True)\n",
    "model.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total label Count: 10\n",
      "Train LG for class : 1\n",
      "Runned Spicy Optimizer > Train Loss: 0.03250798310363381 Val Loss: 0.05026684314405355\n",
      "Train LG for class : 2\n",
      "Runned Spicy Optimizer > Train Loss: 0.07681252224323201 Val Loss: 0.11384103048925709\n",
      "Train LG for class : 3\n",
      "Runned Spicy Optimizer > Train Loss: 0.08235614851997286 Val Loss: 0.09411560987136036\n",
      "Train LG for class : 4\n",
      "Runned Spicy Optimizer > Train Loss: 0.060484883155183224 Val Loss: 0.07343524436019651\n",
      "Train LG for class : 5\n",
      "Runned Spicy Optimizer > Train Loss: 0.09115962198398103 Val Loss: 0.08995227763724033\n",
      "Train LG for class : 6\n",
      "Runned Spicy Optimizer > Train Loss: 0.04251527798702612 Val Loss: 0.06923898052032325\n",
      "Train LG for class : 7\n",
      "Runned Spicy Optimizer > Train Loss: 0.05246759450811332 Val Loss: 0.07253550930589386\n",
      "Train LG for class : 8\n",
      "Runned Spicy Optimizer > Train Loss: 0.1135066041276439 Val Loss: 0.10061835531918789\n",
      "Train LG for class : 9\n",
      "Runned Spicy Optimizer > Train Loss: 0.10266840457205065 Val Loss: 0.10701855986621917\n",
      "Train LG for class : 10\n",
      "Runned Spicy Optimizer > Train Loss: 0.030533733181829247 Val Loss: 0.020734170291084708\n",
      "Train Accuracy: 0.9418823529411765 Val Accuracy: 0.904\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegression(X_train, y_train,X_valid,y_valid,  lr = 0.0000001, epochs =5000,reg = True, useoptimizer=True,ismultiClass=True)\n",
    "model.fit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}