Linear_Classifiers_and_Generalizations.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Answers for Automatic Review Analyser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function for Hinge Loss Single ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hinge_loss_single(feature_vector, label, theta, theta_0):\n",
    "    \"\"\"\n",
    "    Finds the hinge loss on a single data point given specific classification\n",
    "    parameters.\n",
    "\n",
    "    Args:\n",
    "        feature_vector - A numpy array describing the given data point.\n",
    "        label - A real valued number, the correct classification of the data\n",
    "            point.\n",
    "        theta - A numpy array describing the linear classifier.\n",
    "        theta_0 - A real valued number representing the offset parameter.\n",
    "\n",
    "\n",
    "    Returns: A real number representing the hinge loss associated with the\n",
    "    given data point and parameters.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    import numpy as np\n",
    "    z=(np.dot(feature_vector,theta) + theta_0)*label\n",
    "    if z>=1:\n",
    "        return 0\n",
    "    else:\n",
    "        return 1-z\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function for Perceptron Algorithm ## Single Update ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def perceptron_single_step_update(\n",
    "        feature_vector,\n",
    "        label,\n",
    "        current_theta,\n",
    "        current_theta_0):\n",
    "    \"\"\"\n",
    "    Properly updates the classification parameter, theta and theta_0, on a\n",
    "    single step of the perceptron algorithm.\n",
    "\n",
    "    Args:\n",
    "        feature_vector - A numpy array describing a single data point.\n",
    "        label - The correct classification of the feature vector.\n",
    "        current_theta - The current theta being used by the perceptron\n",
    "            algorithm before this update.\n",
    "        current_theta_0 - The current theta_0 being used by the perceptron\n",
    "            algorithm before this update.\n",
    "\n",
    "    Returns: A tuple where the first element is a numpy array with the value of\n",
    "    theta after the current update has completed and the second element is a\n",
    "    real valued number with the value of theta_0 after the current updated has\n",
    "    completed.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    import numpy as np\n",
    "    if (np.dot(feature_vector,current_theta)+current_theta_0)*label <=0:\n",
    "        new_theta=current_theta+np.dot(feature_vector,label)\n",
    "        new_theta_not=current_theta_0+label\n",
    "        return(new_theta,new_theta_not)\n",
    "    else:\n",
    "        return(current_theta,current_theta_0)\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function for Full Perceptron Algorithm ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def perceptron(feature_matrix, labels, T):\n",
    "    \"\"\"\n",
    "    Runs the full perceptron algorithm on a given set of data. Runs T\n",
    "    iterations through the data set, there is no need to worry about\n",
    "    stopping early.\n",
    "\n",
    "    NOTE: Please use the previously implemented functions when applicable.\n",
    "    Do not copy paste code from previous parts.\n",
    "\n",
    "    NOTE: Iterate the data matrix by the orders returned by get_order(feature_matrix.shape[0])\n",
    "\n",
    "    Args:\n",
    "        feature_matrix -  A numpy matrix describing the given data. Each row\n",
    "            represents a single data point.\n",
    "        labels - A numpy array where the kth element of the array is the\n",
    "            correct classification of the kth row of the feature matrix.\n",
    "        T - An integer indicating how many times the perceptron algorithm\n",
    "            should iterate through the feature matrix.\n",
    "\n",
    "    Returns: A tuple where the first element is a numpy array with the value of\n",
    "    theta, the linear classification parameter, after T iterations through the\n",
    "    feature matrix and the second element is a real number with the value of\n",
    "    theta_0, the offset classification parameter, after T iterations through\n",
    "    the feature matrix.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    (nshape,nzeros)=feature_matrix.shape\n",
    "    theta=np.zeros(nzeros)\n",
    "    theta_0=0.0\n",
    "    for t in range(T):\n",
    "        for i in get_order(nshape):\n",
    "            theta, theta_0 = perceptron_single_step_update(feature_matrix[i], labels[i], theta, theta_0)\n",
    "    return (theta,theta_0)\n",
    "    pass\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Average Perceptron Algorhithm ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_perceptron(feature_matrix, labels, T):\n",
    "    \"\"\"\n",
    "    Runs the average perceptron algorithm on a given set of data. Runs T\n",
    "    iterations through the data set, there is no need to worry about\n",
    "    stopping early.\n",
    "\n",
    "    NOTE: Please use the previously implemented functions when applicable.\n",
    "    Do not copy paste code from previous parts.\n",
    "\n",
    "    NOTE: Iterate the data matrix by the orders returned by get_order(feature_matrix.shape[0])\n",
    "\n",
    "\n",
    "    Args:\n",
    "        feature_matrix -  A numpy matrix describing the given data. Each row\n",
    "            represents a single data point.\n",
    "        labels - A numpy array where the kth element of the array is the\n",
    "            correct classification of the kth row of the feature matrix.\n",
    "        T - An integer indicating how many times the perceptron algorithm\n",
    "            should iterate through the feature matrix.\n",
    "\n",
    "    Returns: A tuple where the first element is a numpy array with the value of\n",
    "    the average theta, the linear classification parameter, found after T\n",
    "    iterations through the feature matrix and the second element is a real\n",
    "    number with the value of the average theta_0, the offset classification\n",
    "    parameter, found after T iterations through the feature matrix.\n",
    "\n",
    "    Hint: It is difficult to keep a running average; however, it is simple to\n",
    "    find a sum and divide.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    (nsamples, nfeatures) = feature_matrix.shape\n",
    "    theta = np.zeros(nfeatures)\n",
    "    theta_0 = 0.0\n",
    "    theta_sum=0.0\n",
    "    theta_0_sum=0.0\n",
    "    for t in range(T):\n",
    "        for i in get_order(nsamples):\n",
    "            theta, theta_0 = perceptron_single_step_update(feature_matrix[i], labels[i], theta, theta_0)\n",
    "            theta_sum+=theta\n",
    "            theta_0_sum+=theta_0\n",
    "    return (theta_sum/(nsamples*T), theta_0_sum/(nsamples*T))\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function for Pegasos Algorithm ## Single Update Rule ##\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pegasos_single_step_update(\n",
    "        feature_vector,\n",
    "        label,\n",
    "        L,\n",
    "        eta,\n",
    "        current_theta,\n",
    "        current_theta_0):\n",
    "    \"\"\"\n",
    "    Properly updates the classification parameter, theta and theta_0, on a\n",
    "    single step of the Pegasos algorithm\n",
    "\n",
    "    Args:\n",
    "        feature_vector - A numpy array describing a single data point.\n",
    "        label - The correct classification of the feature vector.\n",
    "        L - The lamba value being used to update the parameters.\n",
    "        eta - Learning rate to update parameters.\n",
    "        current_theta - The current theta being used by the Pegasos\n",
    "            algorithm before this update.\n",
    "        current_theta_0 - The current theta_0 being used by the\n",
    "            Pegasos algorithm before this update.\n",
    "\n",
    "    Returns: A tuple where the first element is a numpy array with the value of\n",
    "    theta after the current update has completed and the second element is a\n",
    "    real valued number with the value of theta_0 after the current updated has\n",
    "    completed.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    import numpy as np\n",
    "    if (label*(np.dot(feature_vector,current_theta)+current_theta_0))<=1.0 :\n",
    "        new_theta= (1-eta*L)*current_theta + eta*label*feature_vector\n",
    "        new_theta_0= current_theta_0+ eta*label\n",
    "        return new_theta, new_theta_0\n",
    "    else :\n",
    "        new_theta=(1-eta*L)*current_theta\n",
    "        new_theta_0=current_theta_0\n",
    "        return new_theta,new_theta_0 \n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function for Full Pegasos Algorhithm ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pegasos(feature_matrix, labels, T, L):\n",
    "    \"\"\"\n",
    "    Runs the Pegasos algorithm on a given set of data. Runs T\n",
    "    iterations through the data set, there is no need to worry about\n",
    "    stopping early.\n",
    "\n",
    "    For each update, set learning rate = 1/sqrt(t),\n",
    "    where t is a counter for the number of updates performed so far (between 1\n",
    "    and nT inclusive).\n",
    "\n",
    "    NOTE: Please use the previously implemented functions when applicable.\n",
    "    Do not copy paste code from previous parts.\n",
    "\n",
    "    Args:\n",
    "        feature_matrix - A numpy matrix describing the given data. Each row\n",
    "            represents a single data point.\n",
    "        labels - A numpy array where the kth element of the array is the\n",
    "            correct classification of the kth row of the feature matrix.\n",
    "        T - An integer indicating how many times the algorithm\n",
    "            should iterate through the feature matrix.\n",
    "        L - The lamba value being used to update the Pegasos\n",
    "            algorithm parameters.\n",
    "\n",
    "    Returns: A tuple where the first element is a numpy array with the value of\n",
    "    the theta, the linear classification parameter, found after T\n",
    "    iterations through the feature matrix and the second element is a real\n",
    "    number with the value of the theta_0, the offset classification\n",
    "    parameter, found after T iterations through the feature matrix.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    (nsamples, nfeatures) = feature_matrix.shape\n",
    "    theta = np.zeros(nfeatures)\n",
    "    theta_0 = 0\n",
    "    count = 0\n",
    "    for t in range(T):\n",
    "        for i in get_order(nsamples):\n",
    "            count += 1\n",
    "            eta = 1.0 / np.sqrt(count)\n",
    "            (theta, theta_0) = pegasos_single_step_update(\n",
    "                feature_matrix[i], labels[i], L, eta, theta, theta_0)\n",
    "    return (theta, theta_0)\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Function for Classification and Accuracy for Automatic ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Classification Function ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify(feature_matrix, theta, theta_0):\n",
    "    \"\"\"\n",
    "    A classification function that uses theta and theta_0 to classify a set of\n",
    "    data points.\n",
    "\n",
    "    Args:\n",
    "        feature_matrix - A numpy matrix describing the given data. Each row\n",
    "            represents a single data point.\n",
    "                theta - A numpy array describing the linear classifier.\n",
    "        theta - A numpy array describing the linear classifier.\n",
    "        theta_0 - A real valued number representing the offset parameter.\n",
    "\n",
    "    Returns: A numpy array of 1s and -1s where the kth element of the array is\n",
    "    the predicted classification of the kth row of the feature matrix using the\n",
    "    given theta and theta_0. If a prediction is GREATER THAN zero, it should\n",
    "    be considered a positive classification.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    import numpy as np\n",
    "    feature=([])\n",
    "    for i in feature_matrix:\n",
    "        if (np.dot(i,theta)+theta_0) >0.0:\n",
    "            feature.append(1)\n",
    "        else:\n",
    "            feature.append(-1)\n",
    "    return np.array(feature)\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Accuracy Function ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classifier_accuracy(\n",
    "        classifier,\n",
    "        train_feature_matrix,\n",
    "        val_feature_matrix,\n",
    "        train_labels,\n",
    "        val_labels,\n",
    "        **kwargs):\n",
    "    \"\"\"\n",
    "    Trains a linear classifier and computes accuracy.\n",
    "    The classifier is trained on the train data. The classifier's\n",
    "    accuracy on the train and validation data is then returned.\n",
    "\n",
    "    Args:\n",
    "        classifier - A classifier function that takes arguments\n",
    "            (feature matrix, labels, **kwargs) and returns (theta, theta_0)\n",
    "        train_feature_matrix - A numpy matrix describing the training\n",
    "            data. Each row represents a single data point.\n",
    "        val_feature_matrix - A numpy matrix describing the training\n",
    "            data. Each row represents a single data point.\n",
    "        train_labels - A numpy array where the kth element of the array\n",
    "            is the correct classification of the kth row of the training\n",
    "            feature matrix.\n",
    "        val_labels - A numpy array where the kth element of the array\n",
    "            is the correct classification of the kth row of the validation\n",
    "            feature matrix.\n",
    "        **kwargs - Additional named arguments to pass to the classifier\n",
    "            (e.g. T or L)\n",
    "\n",
    "    Returns: A tuple in which the first element is the (scalar) accuracy of the\n",
    "    trained classifier on the training data and the second element is the\n",
    "    accuracy of the trained classifier on the validation data.\n",
    "    \"\"\"\n",
    "    # Your code here\n",
    "    theta, theta_0 = classifier(train_feature_matrix, train_labels, **kwargs)\n",
    "    train_predictions = classify(train_feature_matrix, theta, theta_0)\n",
    "    val_predictions = classify(val_feature_matrix, theta, theta_0)\n",
    "    train_accuracy = accuracy(train_predictions, train_labels)\n",
    "    validation_accuracy = accuracy(val_predictions, val_labels)\n",
    "    return (train_accuracy, validation_accuracy)\n",
    "    raise NotImplementedError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "## END OF NOTEBOOK ##\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}