session3-unsolved.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The task\n",
    "\n",
    "Ok, so let's do some fake news detection.\n",
    "\n",
    "Given a headline and a body of text, we want to say whether these two are:\n",
    "\n",
    "* unrelated\n",
    "* agree with each other\n",
    "* disagree with each other\n",
    "* discuss each other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading dataset in dataset/\n",
      "Loading files train_bodies.csv, train_stances.csv\n",
      "Reading dataset in dataset/\n",
      "Loading files competition_test_bodies.csv, competition_test_stances.csv\n"
     ]
    }
   ],
   "source": [
    "from utils.data import DataSet\n",
    "from utils.features import *\n",
    "from utils.dependencies import *\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "training = DataSet()\n",
    "test = DataSet(name=\"competition_test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline_model():\n",
    "    # create model\n",
    "    #\n",
    "    #\n",
    "    #\n",
    "    #\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading dataset in dataset/\n",
      "Loading files train_bodies.csv, train_stances.csv\n",
      "Reading dataset in dataset/\n",
      "Loading files competition_test_bodies.csv, competition_test_stances.csv\n",
      "Reading dataset in dataset/\n",
      "Loading files train_bodies.csv, train_stances.csv\n",
      "Reading dataset in dataset/\n",
      "Loading files competition_test_bodies.csv, competition_test_stances.csv\n",
      "Score for fold 0 was - 0.8024025496445207\n",
      "Score for fold 1 was - 0.7927084788378265\n",
      "Score for fold 2 was - 0.7865658621692353\n",
      "Score for fold 3 was - 0.8108217514505465\n",
      "Score for fold 4 was - 0.7828015452854485\n",
      "Score for fold 5 was - 0.7606774668630338\n",
      "Score for fold 6 was - 0.7638609594943981\n",
      "Score for fold 7 was - 0.7690567105077075\n",
      "Score for fold 8 was - 0.804063492063492\n",
      "Score for fold 9 was - 0.7789561819704158\n"
     ]
    }
   ],
   "source": [
    "d = DataSet(\"train\")\n",
    "folds,hold_out = kfold_split(d,n_folds=10)\n",
    "fold_stances, hold_out_stances = get_stances_for_folds(d,folds,hold_out)\n",
    "\n",
    "# Load the competition dataset\n",
    "competition_dataset = DataSet(\"competition_test\")\n",
    "X_competition, y_competition = generate_features(competition_dataset.data, \"competition\")\n",
    "\n",
    "training = DataSet()\n",
    "test = DataSet(name=\"competition_test\")\n",
    "Xs = dict()\n",
    "ys = dict()\n",
    "\n",
    "# Load/Precompute all features now\n",
    "X_holdout,y_holdout = generate_features(hold_out_stances,\"holdout\")\n",
    "for fold in fold_stances:\n",
    "    Xs[fold],ys[fold] = generate_features(fold_stances[fold],str(fold))\n",
    "\n",
    "best_score = 0\n",
    "best_fold = None\n",
    "\n",
    "encoder = LabelEncoder()\n",
    "\n",
    "# Classifier for each fold\n",
    "for fold in fold_stances:\n",
    "    ids = list(range(len(folds)))\n",
    "    del ids[fold]\n",
    "\n",
    "    X_train = np.vstack(tuple([Xs[i] for i in ids]))\n",
    "    y_train = np.hstack(tuple([ys[i] for i in ids]))\n",
    "\n",
    "    X_test = Xs[fold]\n",
    "    y_test = ys[fold]\n",
    "    \n",
    "    encoder.fit(y_train)\n",
    "    encoded_Y = encoder.transform(y_train)\n",
    "    dummy_y = np_utils.to_categorical(encoded_Y)\n",
    "\n",
    "    estimator = # \n",
    "    estimator.fit(X_train,dummy_y)\n",
    "    \n",
    "    predicted = [LABELS[int(a)] for a in estimator.predict(X_test)]\n",
    "    actual = [LABELS[int(a)] for a in y_test]\n",
    "\n",
    "    fold_score, _ = score_submission(actual, predicted)\n",
    "    max_fold_score, _ = score_submission(actual, actual)\n",
    "\n",
    "    score = fold_score/float(max_fold_score)\n",
    "\n",
    "    print(\"Score for fold \"+ str(fold) + \" was - \" + str(score))\n",
    "    if score > best_score:\n",
    "        best_score = score\n",
    "        best_fold = estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scores on the dev set\n",
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |    153    |    32     |    497    |    80     |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    20     |    28     |    103    |    11     |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    161    |    54     |   1384    |    201    |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |    20     |     3     |    179    |   6696    |\n",
      "-------------------------------------------------------------\n",
      "Score: 3455.75 out of 4448.5\t(77.68348881645498%)\n",
      "\n",
      "\n",
      "Scores on the test set\n",
      "-------------------------------------------------------------\n",
      "|           |   agree   | disagree  |  discuss  | unrelated |\n",
      "-------------------------------------------------------------\n",
      "|   agree   |    326    |    43     |   1271    |    263    |\n",
      "-------------------------------------------------------------\n",
      "| disagree  |    81     |    67     |    327    |    222    |\n",
      "-------------------------------------------------------------\n",
      "|  discuss  |    498    |    116    |   3201    |    649    |\n",
      "-------------------------------------------------------------\n",
      "| unrelated |    73     |    24     |    471    |   17781   |\n",
      "-------------------------------------------------------------\n",
      "Score: 8623.25 out of 11651.25\t(74.01137217036799%)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "74.01137217036799"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Run on Holdout set and report the final score on the holdout set\n",
    "predicted = [LABELS[int(a)] for a in best_fold.predict(X_holdout)]\n",
    "actual = [LABELS[int(a)] for a in y_holdout]\n",
    "\n",
    "print(\"Scores on the dev set\")\n",
    "report_score(actual,predicted)\n",
    "print(\"\")\n",
    "print(\"\")\n",
    "\n",
    "#Run on competition dataset\n",
    "predicted = [LABELS[int(a)] for a in best_fold.predict(X_competition)]\n",
    "actual = [LABELS[int(a)] for a in y_competition]\n",
    "\n",
    "print(\"Scores on the test set\")\n",
    "report_score(actual,predicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Towards real stuff\n",
    "\n",
    "Let's try to check some real news data!  \n",
    "\n",
    "So you need a headline and a blob of text.\n",
    "\n",
    "Try out: https://www.bbc.com/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}