SMART2021_AT_Prediction_Task_v4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SMART2021_AT Prediction Task_v4.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOdRcCZMBYXSnVc9WSvC17y",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/chaeyoonyunakim/smart-2021-AT_Answer_Type_Prediction/blob/main/SMART2021_AT_Prediction_Task_v4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q1YCTaqmYYfV"
      },
      "source": [
        "## 0. Environment setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tMErWmmfkxqp",
        "outputId": "37e79bbf-3c7c-4df7-8490-b3a08b5e54c1"
      },
      "source": [
        "# mount google drive\n",
        "from google.colab import drive\n",
        "\n",
        "# authorization\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# locate dataset folder\n",
        "%ls -l '/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_train.json'"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "-rw------- 1 root root 9263124 Sep 29 08:07 '/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_train.json'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KqGDQCAWfPtF"
      },
      "source": [
        "# set working directory\n",
        "import os\n",
        "os.chdir(path = \"/content/drive/My Drive/2021_INM363_SMART/\")"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C1KBaWJJk4Qg"
      },
      "source": [
        "# import basics\n",
        "import pandas as pd\n",
        "import json\n",
        "import numpy as np\n",
        "import pickle\n",
        "seed = 20211001\n",
        "import regex as re"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qghbo96ZlCTN",
        "outputId": "8a99783c-9a07-4a52-d020-c394efeb209a"
      },
      "source": [
        "# import nlp relevants\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "from collections import defaultdict\n",
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "set(stopwords.words('english'))\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "nltk.download('wordnet')\n",
        "\n",
        "# for bag-of-words (bow)\n",
        "from sklearn import feature_extraction, model_selection, naive_bayes, pipeline, manifold, preprocessing"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGJtj4fPlFid"
      },
      "source": [
        "# import scikit-learn tools for modelling and evaluation\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApUkA-SzlD3c"
      },
      "source": [
        "# import algos\n",
        "from sklearn import svm\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.linear_model import LogisticRegression"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HUs8fZVzZnq2"
      },
      "source": [
        "## 1. Data Loading & Manipulation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qRL8r9it-bla"
      },
      "source": [
        "***Filter ''o'' in question out***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C1W4YPu2wBEo"
      },
      "source": [
        "ap_re = re.compile(\"(^\\')(.*)(\\'$)\")\n",
        "\n",
        "def remove_appostrophe(x):\n",
        "  matches = ap_re.findall(x)\n",
        "  if len(matches) == 1:\n",
        "    return matches[0][1]\n",
        "  else:\n",
        "    return x"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F7bJ0m9W-XH7"
      },
      "source": [
        "***Load Dataset*** "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XWVZEsmFwBLW"
      },
      "source": [
        "def load_data(path, train = True):\n",
        "  data = pd.read_json(path)\n",
        "  \n",
        "  # map na strings to nan\n",
        "  data.loc[:, \"question\"].replace(\"n/a\", np.nan, inplace = True)\n",
        "  \n",
        "  # drop na in data\n",
        "  if train:\n",
        "    data.dropna(subset=['id', 'question', 'category'], inplace=True)\n",
        "  else:\n",
        "    data.dropna(subset=['id', 'question'], inplace=True)\n",
        "\n",
        "  # remove apostrophes from the start and end of str\n",
        "  data.loc[:, \"question\"] = data[\"question\"].map(lambda x: remove_appostrophe(x))\n",
        "\n",
        "  # for the training data remove rows that have no types\n",
        "  if train:\n",
        "    data = data[data[\"type\"].map(lambda x : len(x) != 0)]\n",
        "\n",
        "  return data"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sw9SNMabwBRI"
      },
      "source": [
        "# load dataset\n",
        "train_data2020 = load_data('/content/drive/My Drive/2021_INM363_SMART/smarttask_dbpedia_train.json')\n",
        "test_data2020 = load_data('/content/drive/My Drive/2021_INM363_SMART/smarttask_dbpedia_test.json', train = False)\n",
        "\n",
        "train_data2021 = load_data('/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_train.json')\n",
        "test_data2021 = load_data('/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_test.json', train = False)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0t3x_V5ogknH",
        "outputId": "1accdce0-72dc-49e4-b359-6c4f058aa9ec"
      },
      "source": [
        "# check data size\n",
        "train_data2020.shape, test_data2020.shape, train_data2021.shape, test_data2021.shape"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((17482, 4), (4378, 4), (36670, 4), (9104, 2))"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RsgzXPEz-2ry"
      },
      "source": [
        "***Merge two training dataset***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9fzeOQLKrifd"
      },
      "source": [
        "# unify the id format\n",
        "train_data2020.loc[:, \"id\"] = train_data2020[\"id\"].map(lambda x: x.split('_')[1])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_dmL_aNrimT"
      },
      "source": [
        "# concat two dataset\n",
        "all_data = pd.concat([train_data2020, train_data2021], axis = 0)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dzfd0-sSsAAk"
      },
      "source": [
        "# pre processing to remove duplicates in type\n",
        "# transform array to string\n",
        "all_data.loc[:, \"type_str\"] = all_data[\"type\"].map(lambda x: \",\".join(x))"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQXhkU89_EVf"
      },
      "source": [
        "# remove duplicates when question, category, and type are same\n",
        "merged_data = all_data.drop_duplicates(subset=[\"question\", \"category\", \"type_str\"])"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d0khrcFZZqsS"
      },
      "source": [
        "***Tabular representation of the dataset***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "id": "mknRYgOTripL",
        "outputId": "47df4c24-4f21-4c43-b7bc-8336151e26d3"
      },
      "source": [
        "merged_data.head(3)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>question</th>\n",
              "      <th>category</th>\n",
              "      <th>type</th>\n",
              "      <th>type_str</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1177</td>\n",
              "      <td>Was Jacqueline Kennedy Onassis a follower of M...</td>\n",
              "      <td>boolean</td>\n",
              "      <td>[boolean]</td>\n",
              "      <td>boolean</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>14427</td>\n",
              "      <td>What is the name of the opera based on Twelfth...</td>\n",
              "      <td>resource</td>\n",
              "      <td>[dbo:Opera, dbo:MusicalWork, dbo:Work]</td>\n",
              "      <td>dbo:Opera,dbo:MusicalWork,dbo:Work</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16615</td>\n",
              "      <td>When did Lena Horne receive the Grammy Award f...</td>\n",
              "      <td>literal</td>\n",
              "      <td>[date]</td>\n",
              "      <td>date</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      id  ...                            type_str\n",
              "0   1177  ...                             boolean\n",
              "1  14427  ...  dbo:Opera,dbo:MusicalWork,dbo:Work\n",
              "2  16615  ...                                date\n",
              "\n",
              "[3 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "kvZtVjKP6eC6",
        "outputId": "8bfe21bd-c9bb-4172-c4e2-7bdecbca8dea"
      },
      "source": [
        "merged_data = merged_data.drop(['type_str'], axis =1)\n",
        "merged_data.head()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>question</th>\n",
              "      <th>category</th>\n",
              "      <th>type</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1177</td>\n",
              "      <td>Was Jacqueline Kennedy Onassis a follower of M...</td>\n",
              "      <td>boolean</td>\n",
              "      <td>[boolean]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>14427</td>\n",
              "      <td>What is the name of the opera based on Twelfth...</td>\n",
              "      <td>resource</td>\n",
              "      <td>[dbo:Opera, dbo:MusicalWork, dbo:Work]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16615</td>\n",
              "      <td>When did Lena Horne receive the Grammy Award f...</td>\n",
              "      <td>literal</td>\n",
              "      <td>[date]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>23480</td>\n",
              "      <td>Do Prince Harry and Prince William have the sa...</td>\n",
              "      <td>boolean</td>\n",
              "      <td>[boolean]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3681</td>\n",
              "      <td>What is the subsidiary company working for Leo...</td>\n",
              "      <td>resource</td>\n",
              "      <td>[dbo:EducationalInstitution, dbo:Organisation,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      id  ...                                               type\n",
              "0   1177  ...                                          [boolean]\n",
              "1  14427  ...             [dbo:Opera, dbo:MusicalWork, dbo:Work]\n",
              "2  16615  ...                                             [date]\n",
              "3  23480  ...                                          [boolean]\n",
              "4   3681  ...  [dbo:EducationalInstitution, dbo:Organisation,...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GQKlBsKeMkK9",
        "outputId": "fff95f81-617a-4d1d-a806-ea91f96e8a16"
      },
      "source": [
        "# check data size\n",
        "all_data.shape, merged_data.shape"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((54152, 5), (39556, 4))"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tc3FaIg_tHHC"
      },
      "source": [
        "## 2. Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G7LfNufdhaCV"
      },
      "source": [
        "***Extract, Transform, Load (ETL)***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Epe56ypal-qk"
      },
      "source": [
        "class ETL:\n",
        "\n",
        "###############################################################################\n",
        "#   Global Variables     \n",
        "###############################################################################\n",
        "\n",
        "  # text normalization - stemming, lemmatization, stopwords\n",
        "  ps = PorterStemmer()\n",
        "  wordnet_lemmatizer = WordNetLemmatizer() \n",
        "  s_words = stopwords.words()\n",
        "\n",
        "  # set default feature_extraction parameters\n",
        "  count_vectorizer = None \n",
        "  inv_count_vectorizer_vocab = None\n",
        "  tfidf_vectorizer = None\n",
        "  inv_tfidf_vectorizer_vocab = None\n",
        "\n",
        "  # category maps\n",
        "  category_map = {\"boolean\": 0, \"resource\": 1, \"literal\": 2}\n",
        "  inv_category_map = {}\n",
        "\n",
        "  for label, ind in category_map.items():\n",
        "    inv_category_map[ind] = label\n",
        "\n",
        "  # literal maps\n",
        "  literal_map = {\"date\": 0, \"string\": 1, \"number\": 2}\n",
        "  inv_literal_map = {}\n",
        "\n",
        "  for label, ind in literal_map.items():\n",
        "    inv_literal_map[ind] = label  \n",
        "\n",
        "  # resource maps\n",
        "  type_maps = {}\n",
        "  invtype_maps = {}\n",
        "\n",
        "\n",
        "###############################################################################\n",
        "#   Main\n",
        "###############################################################################\n",
        "\n",
        "  def __init__(self, path_to_type_maps = None, path_to_vectorizers = None):\n",
        "\n",
        "    # load type maps if requested\n",
        "    if path_to_type_maps != None:\n",
        "      base_dir = path_to_type_maps\n",
        "      paths = [fp for fp in os.listdir(base_dir) if \"type\" in fp]\n",
        "\n",
        "      for fp in paths:\n",
        "        with open(os.path.join(base_dir, fp), \"r\") as input_file:\n",
        "          type_name = fp.split(\"_\")[0]\n",
        "          self.invtype_maps[type_name] = {}\n",
        "          self.type_maps[type_name] = json.load(input_file)[type_name]\n",
        "          for ontology, ind in self.type_maps[type_name].items():\n",
        "            self.invtype_maps[type_name][ind] = ontology\n",
        "\n",
        "    # load data vectorizers if requested\n",
        "    if path_to_vectorizers != None:\n",
        "      base_dir = path_to_vectorizers\n",
        "      paths = [fp for fp in os.listdir(base_dir) if \"vectorizer\" in fp]\n",
        "\n",
        "      for fp in paths:\n",
        "        vectorizer_name = fp.split(\"_\")[0]\n",
        "        with open(os.path.join(base_dir, fp), \"rb\") as input_file:\n",
        "          if vectorizer_name == \"count\":\n",
        "            self.count_vectorizer = pickle.load(input_file)\n",
        "          elif vectorizer_name == \"tfidf\":\n",
        "            self.tfidf_vectorizer = pickle.load(input_file)\n",
        "          else:\n",
        "            NotImplementedError\n",
        "\n",
        "\n",
        "  # split training dataset to exclude validation dataset\n",
        "  # set train:val = 8:2\n",
        "  def split_data(self, data, val_size = 0.2):\n",
        "    df_train, df_test = model_selection.train_test_split(data, test_size = val_size, random_state = seed)\n",
        "    return df_train, df_test\n",
        "\n",
        "\n",
        "  # normalization of question sentences\n",
        "  def _norm_sent(self, sent, rm_stopwords = False, stemming = True, lemmatization = False):\n",
        "    # tokenize - sentence to word\n",
        "    words = word_tokenize(sent)\n",
        "    # take if all characters in the string are alphabets and then decapitalize\n",
        "    sent = [w.lower() for w in words if w.isalpha()] \n",
        "\n",
        "    # remove stopwords\n",
        "    if rm_stopwords:\n",
        "      sent = [w for w in sent if w not in self.s_words]    \n",
        "\n",
        "    # apply lemmatization \n",
        "    if lemmatization:\n",
        "      sent = [self.wordnet_lemmatizer.lemmatize(w, pos = \"n\") for w in sent]\n",
        "      sent = [self.wordnet_lemmatizer.lemmatize(w, pos = \"v\") for w in sent]\n",
        "      sent = [self.wordnet_lemmatizer.lemmatize(w, pos = (\"a\")) for w in sent]\n",
        "\n",
        "    # apply stemming \n",
        "    if stemming:\n",
        "      sent = [self.ps.stem(w) for w in sent]\n",
        "\n",
        "    sent = \" \".join(sent)\n",
        "    return sent  \n",
        "\n",
        "\n",
        "  # add a new column to show how question parsing has done through normalization above\n",
        "  # for Tabular representation of the dataset\n",
        "  def norm_data(self, data):   \n",
        "    data.loc[:, \"question_processed\"] = data[\"question\"].apply(lambda x: self._norm_sent(x, rm_stopwords = False, lemmatization = True, stemming = True))\n",
        "    return data\n",
        "\n",
        "\n",
        "\n",
        "  # vectorization - fit vectorizer to training data\n",
        "  def bow_fit(self, corpus, type = \"tf\", max_features = 10000, ngram_range = (1,2)):\n",
        "\n",
        "    if type == \"tf\":\n",
        "      self.count_vectorizer = feature_extraction.text.CountVectorizer(max_features = max_features, ngram_range = ngram_range)\n",
        "      self.count_vectorizer.fit(corpus[\"question_processed\"])\n",
        "\n",
        "      # create a reverse mapping for the vocab\n",
        "      self.inv_count_vectorizer_vocab = {}\n",
        "      for label, ind in self.count_vectorizer.vocabulary_.items():\n",
        "        self.inv_count_vectorizer_vocab[ind] = label\n",
        "\n",
        "    elif type == \"tfidf\": \n",
        "      self.tfidf_vectorizer = feature_extraction.text.TfidfVectorizer(max_features = max_features, ngram_range = ngram_range)\n",
        "      self.tfidf_vectorizer.fit(corpus[\"question_processed\"])\n",
        "      \n",
        "      # create a reverse mapping for the vocab\n",
        "      self.inv_tfidf_vectorizer_vocab = {}\n",
        "      for label, ind in self.tfidf_vectorizer.vocabulary_.items():\n",
        "        self.inv_tfidf_vectorizer_vocab[ind] = label\n",
        "\n",
        "    else:\n",
        "      return NotImplementedError\n",
        "\n",
        "\n",
        "  # transformation\n",
        "  def bow_transform(self, data, type = \"tf\"):\n",
        "    if type == \"tf\":\n",
        "      return self.count_vectorizer.transform(data[\"question_processed\"])\n",
        "    elif type == \"tfidf\":\n",
        "      return self.tfidf_vectorizer.transform(data[\"question_processed\"])\n",
        "    else:\n",
        "      return NotImplementedError\n",
        "\n",
        "\n",
        "  # category maps\n",
        "  def category_to_int(self, data):\n",
        "    return data.category.map(lambda x: self.category_map[x])\n",
        "\n",
        "  # literal maps\n",
        "  def literal_to_int(self, data):\n",
        "    return data.type.map(lambda x: self.literal_map[x[0]])\n",
        "\n",
        "\n",
        "  # distribute type by ontology class and encode missing if none\n",
        "  def type_to_int(self, data, type_no):\n",
        "    return data.type.map(\n",
        "        lambda x: self.type_maps[f\"type{type_no}\"][x[type_no - 1]] \n",
        "        if len(x) >= type_no \n",
        "        else self.type_maps[f\"type{type_no}\"][\"missing\"]\n",
        "        )\n",
        "\n",
        "\n",
        "  # resource maps\n",
        "  def add_type_maps(self, train_data, depth = 6, save = True, path = \"resource_types/\"):\n",
        "\n",
        "    levels = range(1, depth)\n",
        "    \n",
        "    if save:\n",
        "      os.makedirs(path, exist_ok = True)\n",
        "  \n",
        "    for l in levels:\n",
        "      type_name = f\"type{l}\"\n",
        "      self.type_maps[type_name] = {}\n",
        "      self.invtype_maps[type_name] = {}\n",
        "      ind = 0\n",
        "      temp_df = train_data[train_data[\"category\"] == \"resource\"][\"type\"].map(lambda x: x[l-1] if len(x) >= l else \"missing\").to_frame(type_name)\n",
        "      for ontology in temp_df[type_name]:\n",
        "        if (ontology not in self.type_maps[type_name]) and (ontology != \"missing\"):\n",
        "          self.type_maps[type_name][ontology] = ind \n",
        "          self.invtype_maps[type_name][ind] = ontology\n",
        "          ind += 1\n",
        "      if save:\n",
        "        with open(os.path.join(\"resource_types\", f\"type{l}_map.json\"), \"w\") as outfile:\n",
        "          temp_json_obj = json.dump(self.type_maps, outfile)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "  # save output\n",
        "  def save_vectorizers(self, path):\n",
        "\n",
        "    # make sure directory exists\n",
        "    os.makedirs(exist_ok= True, name=path)\n",
        "\n",
        "    if self.count_vectorizer != None:\n",
        "      with open(os.path.join(path, \"count_vectorizer.pkl\"), \"wb\") as count_file:\n",
        "        pickle.dump(self.count_vectorizer, count_file)\n",
        "    if self.tfidf_vectorizer != None:\n",
        "      with open(os.path.join(path, \"tfidf_vectorizer.pkl\"), \"wb\") as tfidf_file:\n",
        "        pickle.dump(self.tfidf_vectorizer, tfidf_file)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zRRJ_AlIwcCS"
      },
      "source": [
        "etl = ETL()"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eKbfRa2a80BP"
      },
      "source": [
        "# split dataset\n",
        "# for validation 8:2\n",
        "# df_train, df_val = etl.split_data(merged_data)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVWgYwQS80ZZ"
      },
      "source": [
        "# text normalization\n",
        "# df_train = etl.norm_data(df_train)   # training set without val\n",
        "df_train = etl.norm_data(merged_data) # training set including val (total)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_YJAsGNPC9Dv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "ad129177-b66a-494e-a1ce-335b08a44262"
      },
      "source": [
        "df_train[['question', 'question_processed']]"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>question_processed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Was Jacqueline Kennedy Onassis a follower of M...</td>\n",
              "      <td>wa jacquelin kennedi onassi a follow of melkit...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What is the name of the opera based on Twelfth...</td>\n",
              "      <td>what be the name of the opera base on twelfth ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>When did Lena Horne receive the Grammy Award f...</td>\n",
              "      <td>when do lena horn receiv the grammi award for ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Do Prince Harry and Prince William have the sa...</td>\n",
              "      <td>do princ harri and princ william have the same...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>What is the subsidiary company working for Leo...</td>\n",
              "      <td>what be the subsidiari compani work for leonar...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36665</th>\n",
              "      <td>what kinds of music is played by season's end</td>\n",
              "      <td>what kind of music be play by season end</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36666</th>\n",
              "      <td>which asteroid group is 6753 fursenko a member...</td>\n",
              "      <td>which asteroid group be fursenko a member of</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36667</th>\n",
              "      <td>What language is azhakiya ravanan filmed in?</td>\n",
              "      <td>what languag be azhakiya ravanan film in</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36668</th>\n",
              "      <td>which position did herby fortunat play in foot...</td>\n",
              "      <td>which posit do herbi fortunat play in footbal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36669</th>\n",
              "      <td>who is a person that was born in sao paulo</td>\n",
              "      <td>who be a person that wa bear in sao paulo</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>39556 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                question                                 question_processed\n",
              "0      Was Jacqueline Kennedy Onassis a follower of M...  wa jacquelin kennedi onassi a follow of melkit...\n",
              "1      What is the name of the opera based on Twelfth...  what be the name of the opera base on twelfth ...\n",
              "2      When did Lena Horne receive the Grammy Award f...  when do lena horn receiv the grammi award for ...\n",
              "3      Do Prince Harry and Prince William have the sa...  do princ harri and princ william have the same...\n",
              "4      What is the subsidiary company working for Leo...  what be the subsidiari compani work for leonar...\n",
              "...                                                  ...                                                ...\n",
              "36665      what kinds of music is played by season's end           what kind of music be play by season end\n",
              "36666  which asteroid group is 6753 fursenko a member...       which asteroid group be fursenko a member of\n",
              "36667       What language is azhakiya ravanan filmed in?           what languag be azhakiya ravanan film in\n",
              "36668  which position did herby fortunat play in foot...      which posit do herbi fortunat play in footbal\n",
              "36669         who is a person that was born in sao paulo          who be a person that wa bear in sao paulo\n",
              "\n",
              "[39556 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ku3cAUNXuzUG"
      },
      "source": [
        "# vectorization - bag of words model\n",
        "etl.bow_fit(corpus = df_train, type = \"tf\")\n",
        "etl.save_vectorizers(path=\"sklearn_objects\")"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VpGAtqqku-Mb"
      },
      "source": [
        "## 3. Category Prediction Task"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hNrKgk6Cu9b8"
      },
      "source": [
        "# set category prediction dataset\n",
        "X_train_category = etl.bow_transform(df_train)\n",
        "y_train_category = etl.category_to_int(df_train)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c-2QymUbxPNo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16a41877-b71d-476a-e840-5679ad6fef42"
      },
      "source": [
        "# model for category classification\n",
        "clf_category = LogisticRegression(\n",
        "    random_state=seed, penalty = 'elasticnet', solver = 'saga',\n",
        "    l1_ratio = 0.2, n_jobs = -1, verbose = 2, max_iter = 200)\\\n",
        "    .fit(X_train_category, y_train_category)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 2 concurrent workers.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max_iter reached after 913 seconds\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n",
            "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed: 15.2min finished\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XhcXhB1491gk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e6c1979-66a5-482d-a7ca-730c742b76ce"
      },
      "source": [
        "clf_category.score(X_train_category, y_train_category)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9836687228233391"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v6t3q77QvuOE"
      },
      "source": [
        "## 4-1. Type Prediction Task - Literal"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FjARPmJfv84w"
      },
      "source": [
        "***3 different model in total: 1 for category, 1 for literal, 1 for resource***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vi_yxC3C5nrx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fa05468e-dc9e-4fc0-a695-97b1a26ed743"
      },
      "source": [
        "# model for literal classification\n",
        "# get which rows are for literal only  \n",
        "train_literal_rows = (df_train[\"category\"] == \"literal\").values\n",
        "y_train_literal = etl.literal_to_int(df_train[train_literal_rows])\n",
        "\n",
        "clf_literal = LogisticRegression(\n",
        "    random_state=seed, penalty = 'elasticnet', solver = 'saga',\n",
        "    l1_ratio = 0.5, n_jobs = -1, verbose = 2, max_iter = 200)\\\n",
        "    .fit(X_train_category[train_literal_rows, :], y_train_literal)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 2 concurrent workers.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max_iter reached after 11 seconds\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n",
            "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:   10.9s finished\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ihsyVY143UPg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f9c3442-bde9-42d1-c5de-516cc1e06eb8"
      },
      "source": [
        "clf_literal.score(X_train_category[train_literal_rows], y_train_literal)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9766809172172561"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hr-4DSgGGYoF"
      },
      "source": [
        "## 4-2. Type Prediction Task - Resource\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1bLD1-vEVYWj"
      },
      "source": [
        "***Identify types of resources in the train data***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZe_y3EeIYvz"
      },
      "source": [
        "etl.add_type_maps(df_train)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZU7YplswisE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "61f8df47-ae02-47c3-bc49-dfd962fe4c30"
      },
      "source": [
        "etl.type_maps.keys()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['type1', 'type2', 'type3', 'type4', 'type5'])"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TPYDqPkqw6KD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d4c63b7f-4a25-40a7-8be5-a727129e5425"
      },
      "source": [
        "etl.type_maps.values()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_values([{'dbo:Opera': 0, 'dbo:EducationalInstitution': 1, 'dbo:State': 2, 'dbo:Country': 3, 'dbo:AcademicSubject': 4, 'dbo:Name': 5, 'dbo:Organisation': 6, 'dbo:Person': 7, 'dbo:WrittenWork': 8, 'dbo:EthnicGroup': 9, 'dbo:Museum': 10, 'dbo:Writer': 11, 'dbo:AmericanFootballPlayer': 12, 'dbo:ChemicalCompound': 13, 'dbo:OfficeHolder': 14, 'dbo:Magazine': 15, 'dbo:Award': 16, 'dbo:Animal': 17, 'dbo:Activity': 18, 'dbo:NobelPrize': 19, 'dbo:Single': 20, 'dbo:Work': 21, 'dbo:MetroStation': 22, 'dbo:Film': 23, 'dbo:Profession': 24, 'dbo:City': 25, 'dbo:Deity': 26, 'dbo:Galaxy': 27, 'dbo:Company': 28, 'dbo:Gene': 29, 'dbo:Contest': 30, 'dbo:TelevisionEpisode': 31, 'dbo:University': 32, 'dbo:MusicalArtist': 33, 'dbo:Scientist': 34, 'dbo:PoliticalParty': 35, 'dbo:MusicGenre': 36, 'dbo:Disease': 37, 'dbo:Taxon': 38, 'dbo:Book': 39, 'dbo:Settlement': 40, 'dbo:GovernmentAgency': 41, 'dbo:Ship': 42, 'dbo:Drug': 43, 'dbo:Island': 44, 'dbo:Mountain': 45, 'dbo:Village': 46, 'dbo:River': 47, 'dbo:Media': 48, 'dbo:Sport': 49, 'dbo:LegalCase': 50, 'dbo:HistoricalRegion': 51, 'dbo:MilitaryConflict': 52, 'dbo:SportsTeam': 53, 'dbo:TelevisionStation': 54, 'dbo:Archive': 55, 'dbo:Asteroid': 56, 'dbo:GivenName': 57, 'dbo:Software': 58, 'dbo:Monastery': 59, 'dbo:TelevisionShow': 60, 'dbo:MilitaryUnit': 61, 'dbo:Archipelago': 62, 'dbo:Saint': 63, 'dbo:RailwayLine': 64, 'dbo:Non-ProfitOrganisation': 65, 'dbo:Cemetery': 66, 'dbo:Tournament': 67, 'dbo:Aircraft': 68, 'dbo:SoccerPlayer': 69, 'dbo:Royalty': 70, 'dbo:Building': 71, 'dbo:Bridge': 72, 'dbo:Band': 73, 'dbo:Lake': 74, 'dbo:Family': 75, 'dbo:Spacecraft': 76, 'dbo:Church': 77, 'dbo:BoardGame': 78, 'dbo:Constellation': 79, 'dbo:Play': 80, 'dbo:Region': 81, 'dbo:VideoGame': 82, 'dbo:Castle': 83, 'dbo:BasketballPlayer': 84, 'dbo:InformationAppliance': 85, 'dbo:Election': 86, 'dbo:Protein': 87, 'dbo:Weapon': 88, 'dbo:ProgrammingLanguage': 89, 'dbo:Ideology': 90, 'dbo:BasketballTeam': 91, 'dbo:Album': 92, 'dbo:RailwayStation': 93, 'dbo:RecordLabel': 94, 'dbo:AdultActor': 95, 'dbo:Vein': 96, 'dbo:President': 97, 'dbo:SpaceMission': 98, 'dbo:Event': 99, 'dbo:SpaceStation': 100, 'dbo:Game': 101, 'dbo:Airport': 102, 'dbo:Parliament': 103, 'dbo:MedicalSpecialty': 104, 'dbo:Currency': 105, 'dbo:Planet': 106, 'dbo:FormulaOneRacer': 107, 'dbo:SportsClub': 108, 'dbo:RailwayTunnel': 109, 'dbo:WineRegion': 110, 'dbo:Newspaper': 111, 'dbo:Town': 112, 'dbo:HorseRace': 113, 'dbo:Manga': 114, 'dbo:Skyscraper': 115, 'dbo:BaseballTeam': 116, 'dbo:NobleFamily': 117, 'dbo:SportsLeague': 118, 'dbo:Novel': 119, 'dbo:Mammal': 120, 'dbo:FigureSkater': 121, 'dbo:NaturalRegion': 122, 'dbo:CyclingRace': 123, 'dbo:Hotel': 124, 'dbo:Road': 125, 'dbo:Stadium': 126, 'dbo:NationalAnthem': 127, 'dbo:Musical': 128, 'dbo:HockeyTeam': 129, 'dbo:Legislature': 130, 'dbo:MartialArtist': 131, 'dbo:Dam': 132, 'dbo:AcademicJournal': 133, 'dbo:MountainRange': 134, 'dbo:AnatomicalStructure': 135, 'dbo:Ocean': 136, 'dbo:SoccerClub': 137, 'dbo:FictionalCharacter': 138, 'dbo:Airline': 139, 'dbo:Language': 140, 'dbo:Locomotive': 141, 'dbo:Beverage': 142, 'dbo:MusicalWork': 143, 'dbo:Monarch': 144, 'dbo:PowerStation': 145, 'dbo:MountainPass': 146, 'dbo:AmericanFootballTeam': 147, 'dbo:Letter': 148, 'dbo:Mineral': 149, 'dbo:Cat': 150, 'dbo:CollegeCoach': 151, 'dbo:Boxer': 152, 'dbo:Architect': 153, 'dbo:SkiResort': 154, 'dbo:Philosopher': 155, 'dbo:Valley': 156, 'dbo:PopulatedPlace': 157, 'dbo:Senator': 158, 'dbo:Factory': 159, 'dbo:MilitaryPerson': 160, 'dbo:Gymnast': 161, 'dbo:Library': 162, 'dbo:Port': 163, 'dbo:Volcano': 164, 'dbo:Swimmer': 165, 'dbo:School': 166, 'dbo:Astronaut': 167, 'dbo:Food': 168, 'dbo:Venue': 169, 'dbo:Artist': 170, 'dbo:Politician': 171, 'dbo:Rocket': 172, 'dbo:Cave': 173, 'dbo:SportsManager': 174, 'dbo:GeologicalPeriod': 175, 'dbo:Judge': 176, 'dbo:AdministrativeRegion': 177, 'dbo:Square': 178, 'dbo:Monument': 179, 'dbo:Comedian': 180, 'dbo:SoccerManager': 181, 'dbo:Mollusca': 182, 'dbo:ComicsCreator': 183, 'dbo:Prison': 184, 'dbo:BeautyQueen': 185, 'dbo:Train': 186, 'dbo:Anime': 187, 'dbo:Enzyme': 188, 'dbo:Muscle': 189, 'dbo:Holiday': 190, 'dbo:RadioProgram': 191, 'dbo:BaseballLeague': 192, 'dbo:Ambassador': 193, 'dbo:Governor': 194, 'dbo:ComicsCharacter': 195, 'dbo:GolfCourse': 196, 'dbo:ProtectedArea': 197, 'dbo:PublicService': 198, 'dbo:Street': 199, 'dbo:LaunchPad': 200, 'dbo:RugbyPlayer': 201, 'dbo:Mosque': 202, 'dbo:Bird': 203, 'dbo:Broadcaster': 204, 'dbo:Plant': 205, 'dbo:BasketballLeague': 206, 'dbo:Tax': 207, 'dbo:Artery': 208, 'dbo:Bacteria': 209, 'dbo:Bank': 210, 'dbo:Restaurant': 211, 'dbo:PersonFunction': 212, 'dbo:TennisPlayer': 213, 'dbo:Bone': 214, 'dbo:ReligiousBuilding': 215, 'dbo:Publisher': 216, 'dbo:Motorcycle': 217, 'dbo:AutomobileEngine': 218, 'dbo:Poem': 219, 'dbo:Lighthouse': 220, 'dbo:IceHockeyLeague': 221, 'dbo:Tower': 222, 'dbo:HistoricBuilding': 223, 'dbo:Zoo': 224, 'dbo:Nerve': 225, 'dbo:HollywoodCartoon': 226, 'dbo:ArchitecturalStructure': 227, 'dbo:MusicFestival': 228, 'dbo:Brain': 229, 'dbo:Insect': 230, 'dbo:Song': 231, 'dbo:Agent': 232, 'dbo:ChessPlayer': 233, 'dbo:CultivatedVariety': 234, 'dbo:IceHockeyPlayer': 235, 'dbo:Hospital': 236, 'dbo:Flag': 237, 'dbo:FloweringPlant': 238, 'dbo:WorldHeritageSite': 239, 'dbo:HistoricalPeriod': 240, 'dbo:PublicTransitSystem': 241, 'dbo:DartsPlayer': 242, 'dbo:NaturalPlace': 243, 'dbo:Meeting': 244, 'dbo:BaseballPlayer': 245, 'dbo:AmericanFootballLeague': 246, 'dbo:TennisTournament': 247, 'dbo:Place': 248, 'dbo:Noble': 249, 'dbo:Sea': 250, 'dbo:Grape': 251, 'dbo:FilmFestival': 252, 'dbo:Continent': 253, 'dbo:PlayboyPlaymate': 254, 'dbo:BroadcastNetwork': 255, 'dbo:Priest': 256, 'dbo:HotSpring': 257, 'dbo:Actor': 258, 'dbo:MemberOfParliament': 259, 'dbo:Glacier': 260, 'dbo:Wrestler': 261, 'dbo:Beer': 262, 'dbo:CardGame': 263, 'dbo:Brewery': 264, 'dbo:RaceTrack': 265, 'dbo:FootballLeagueSeason': 266, 'dbo:Genre': 267, 'dbo:SocietalEvent': 268, 'dbo:SportsEvent': 269, 'dbo:MythologicalFigure': 270, 'dbo:Colour': 271, 'dbo:College': 272, 'dbo:Reptile': 273, 'dbo:PeriodicalLiterature': 274, 'dbo:Automobile': 275, 'dbo:OlympicEvent': 276, 'dbo:TelevisionSeason': 277, 'dbo:CanadianFootballTeam': 278, 'dbo:Eukaryote': 279, 'dbo:Artwork': 280, 'dbo:Star': 281, 'dbo:ArtificialSatellite': 282, 'dbo:Athlete': 283, 'dbo:MeanOfTransportation': 284, 'dbo:SoccerTournament': 285, 'dbo:Arachnid': 286, 'dbo:Fish': 287, 'dbo:CyclingTeam': 288, 'dbo:Comic': 289, 'dbo:Device': 290, 'dbo:ComicStrip': 291, 'dbo:SportFacility': 292, 'dbo:Surname': 293, 'dbo:Criminal': 294, 'dbo:GolfTournament': 295, 'dbo:Convention': 296, 'dbo:RadioStation': 297}, {'dbo:MusicalWork': 0, 'dbo:Organisation': 1, 'dbo:PopulatedPlace': 2, 'dbo:State': 3, 'dbo:TopicalConcept': 4, 'dbo:Agent': 5, 'dbo:Work': 6, 'dbo:Castle': 7, 'dbo:Person': 8, 'dbo:GridironFootballPlayer': 9, 'dbo:ChemicalSubstance': 10, 'dbo:President': 11, 'dbo:PeriodicalLiterature': 12, 'dbo:Eukaryote': 13, 'dbo:Award': 14, 'dbo:Station': 15, 'dbo:PersonFunction': 16, 'dbo:City': 17, 'dbo:Settlement': 18, 'dbo:CelestialBody': 19, 'dbo:Biomolecule': 20, 'dbo:Competition': 21, 'dbo:EducationalInstitution': 22, 'dbo:Artist': 23, 'dbo:Band': 24, 'dbo:Genre': 25, 'dbo:TelevisionShow': 26, 'dbo:WrittenWork': 27, 'dbo:MeanOfTransportation': 28, 'dbo:NaturalPlace': 29, 'dbo:Stream': 30, 'dbo:Game': 31, 'dbo:Case': 32, 'dbo:Region': 33, 'dbo:SocietalEvent': 34, 'dbo:BroadcastNetwork': 35, 'dbo:CollectionOfValuables': 36, 'dbo:Name': 37, 'dbo:ReligiousBuilding': 38, 'dbo:Cleric': 39, 'dbo:RouteOfTransportation': 40, 'dbo:Place': 41, 'dbo:SportsEvent': 42, 'dbo:Athlete': 43, 'dbo:OfficeHolder': 44, 'dbo:ArchitecturalStructure': 45, 'dbo:Country': 46, 'dbo:FormulaOneTeam': 47, 'dbo:Group': 48, 'dbo:Continent': 49, 'dbo:Film': 50, 'dbo:Sea': 51, 'dbo:Software': 52, 'dbo:Island': 53, 'dbo:Building': 54, 'dbo:Device': 55, 'dbo:Language': 56, 'dbo:SoccerClub': 57, 'dbo:Company': 58, 'dbo:Actor': 59, 'dbo:AnatomicalStructure': 60, 'dbo:Politician': 61, 'dbo:Activity': 62, 'dbo:Infrastructure': 63, 'dbo:RacingDriver': 64, 'dbo:PublicTransitSystem': 65, 'dbo:Comic': 66, 'dbo:SportsTeam': 67, 'dbo:Family': 68, 'dbo:Book': 69, 'dbo:Animal': 70, 'dbo:Race': 71, 'dbo:SportFacility': 72, 'dbo:Sport': 73, 'dbo:BodyOfWater': 74, 'dbo:BaseballTeam': 75, 'dbo:Deity': 76, 'dbo:AmericanFootballTeam': 77, 'dbo:MilitaryPerson': 78, 'dbo:Food': 79, 'dbo:Mammal': 80, 'dbo:Coach': 81, 'dbo:SkiArea': 82, 'dbo:Royalty': 83, 'dbo:SoccerPlayer': 84, 'dbo:TimePeriod': 85, 'dbo:Philosopher': 86, 'dbo:SportsManager': 87, 'dbo:SportsClub': 88, 'dbo:Cartoon': 89, 'dbo:SportsLeague': 90, 'dbo:Theatre': 91, 'dbo:FictionalCharacter': 92, 'dbo:Museum': 93, 'dbo:BasketballTeam': 94, 'dbo:SoccerLeague': 95, 'dbo:Species': 96, 'dbo:Engine': 97, 'dbo:Broadcaster': 98, 'dbo:Stadium': 99, 'dbo:Tower': 100, 'dbo:WinterSportPlayer': 101, 'dbo:Mountain': 102, 'dbo:TelevisionEpisode': 103, 'dbo:Writer': 104, 'dbo:Volcano': 105, 'dbo:PowerStation': 106, 'dbo:Insect': 107, 'dbo:ComicsCreator': 108, 'dbo:Plant': 109, 'dbo:Skyscraper': 110, 'dbo:School': 111, 'dbo:Airline': 112, 'dbo:Non-ProfitOrganisation': 113, 'dbo:Noble': 114, 'dbo:Church': 115, 'dbo:Monument': 116, 'dbo:Tournament': 117, 'dbo:Location': 118, 'dbo:River': 119, 'dbo:Town': 120, 'dbo:FloweringPlant': 121, 'dbo:RailwayStation': 122, 'dbo:MountainRange': 123, 'dbo:Beverage': 124, 'dbo:PoliticalParty': 125, 'dbo:MusicalArtist': 126, 'dbo:Pope': 127, 'dbo:MusicGenre': 128, 'dbo:AmericanFootballPlayer': 129, 'dbo:MilitaryConflict': 130, 'dbo:Saint': 131, 'dbo:Disease': 132, 'dbo:Grape': 133, 'dbo:SoccerTournament': 134, 'dbo:Website': 135, 'dbo:EthnicGroup': 136, 'dbo:Scientist': 137, 'dbo:FootballMatch': 138, 'dbo:Olympics': 139, 'dbo:TelevisionStation': 140, 'dbo:Event': 141, 'dbo:Album': 142, 'dbo:MilitaryUnit': 143, 'dbo:Election': 144, 'dbo:Song': 145, 'dbo:RugbyClub': 146, 'dbo:University': 147, 'dbo:ProgrammingLanguage': 148, 'dbo:RadioStation': 149, 'dbo:Library': 150, 'dbo:InformationAppliance': 151, 'dbo:VideoGame': 152, 'dbo:Musical': 153, 'dbo:Criminal': 154, 'dbo:Drug': 155, 'dbo:Religious': 156, 'dbo:Anime': 157, 'dbo:Publisher': 158, 'dbo:HollywoodCartoon': 159, 'dbo:Cardinal': 160, 'dbo:GovernmentAgency': 161, 'dbo:Architect': 162, 'dbo:PlayboyPlaymate': 163, 'dbo:Legislature': 164, 'dbo:HorseRace': 165, 'dbo:RecordLabel': 166, 'dbo:FormulaOneRacer': 167, 'dbo:Venue': 168, 'dbo:Star': 169, 'dbo:Automobile': 170, 'dbo:Presenter': 171}, {'dbo:Work': 0, 'dbo:Agent': 1, 'dbo:Place': 2, 'dbo:PopulatedPlace': 3, 'dbo:Building': 4, 'dbo:Athlete': 5, 'dbo:Politician': 6, 'dbo:WrittenWork': 7, 'dbo:Species': 8, 'dbo:Infrastructure': 9, 'dbo:Settlement': 10, 'dbo:Event': 11, 'dbo:Organisation': 12, 'dbo:Person': 13, 'dbo:TopicalConcept': 14, 'dbo:BodyOfWater': 15, 'dbo:Activity': 16, 'dbo:UnitOfWork': 17, 'dbo:State': 18, 'dbo:Broadcaster': 19, 'dbo:Location': 20, 'dbo:SocietalEvent': 21, 'dbo:SportsTeam': 22, 'dbo:AdministrativeRegion': 23, 'dbo:ArchitecturalStructure': 24, 'dbo:Artist': 25, 'dbo:OfficeHolder': 26, 'dbo:MotorsportRacer': 27, 'dbo:Plant': 28, 'dbo:SportsEvent': 29, 'dbo:Island': 30, 'dbo:ProtectedArea': 31, 'dbo:Group': 32, 'dbo:NaturalPlace': 33, 'dbo:Eukaryote': 34, 'dbo:River': 35, 'dbo:Animal': 36, 'dbo:SportFacility': 37, 'dbo:Cemetery': 38, 'dbo:Country': 39, 'dbo:MilitaryPerson': 40, 'dbo:President': 41, 'dbo:SoccerManager': 42, 'dbo:Venue': 43, 'dbo:ReligiousBuilding': 44, 'dbo:SportsLeague': 45, 'dbo:Company': 46, 'dbo:SoccerPlayer': 47, 'dbo:PublicTransitSystem': 48, 'dbo:Device': 49, 'dbo:MusicalArtist': 50, 'dbo:EducationalInstitution': 51, 'dbo:Band': 52, 'dbo:SoccerClub': 53, 'dbo:Station': 54, 'dbo:Volcano': 55, 'dbo:Food': 56, 'dbo:SportsManager': 57, 'dbo:FictionalCharacter': 58, 'dbo:City': 59, 'dbo:Scientist': 60, 'dbo:MusicGenre': 61, 'dbo:Writer': 62, 'dbo:Continent': 63, 'dbo:Sport': 64, 'dbo:BaseballTeam': 65, 'dbo:HistoricBuilding': 66, 'dbo:CityDistrict': 67, 'dbo:Mountain': 68, 'dbo:GridironFootballPlayer': 69, 'dbo:TelevisionShow': 70, 'dbo:SoapCharacter': 71, 'dbo:Sea': 72, 'dbo:RecordLabel': 73, 'dbo:MotorsportSeason': 74, 'dbo:HistoricPlace': 75, 'dbo:PersonFunction': 76, 'dbo:Model': 77, 'dbo:PeriodicalLiterature': 78, 'dbo:HumanGene': 79, 'dbo:Olympics': 80, 'dbo:Film': 81, 'dbo:ConcentrationCamp': 82, 'dbo:School': 83, 'dbo:TelevisionStation': 84, 'dbo:Royalty': 85, 'dbo:Bank': 86, 'dbo:AmericanFootballLeague': 87, 'dbo:MilitaryUnit': 88, 'dbo:RaceHorse': 89, 'dbo:AmericanFootballTeam': 90, 'dbo:BeautyQueen': 91, 'dbo:IceHockeyPlayer': 92, 'dbo:SoccerClubSeason': 93, 'dbo:Planet': 94, 'dbo:Bodybuilder': 95, 'dbo:Song': 96, 'dbo:Architect': 97, 'dbo:Constellation': 98, 'dbo:VideoGame': 99, 'dbo:RugbyPlayer': 100, 'dbo:Cartoon': 101, 'dbo:Star': 102, 'dbo:Pope': 103, 'dbo:Park': 104, 'dbo:Disease': 105, 'dbo:University': 106, 'dbo:CollegeCoach': 107, 'dbo:Book': 108, 'dbo:Astronaut': 109, 'dbo:Chef': 110, 'dbo:Economist': 111, 'dbo:Bridge': 112, 'dbo:MusicalWork': 113, 'dbo:Noble': 114, 'dbo:Manga': 115, 'dbo:MilitaryConflict': 116, 'dbo:GovernmentalAdministrativeRegion': 117, 'dbo:ChristianBishop': 118, 'dbo:Medician': 119, 'dbo:Saint': 120, 'dbo:PoliticalParty': 121, 'dbo:Cleric': 122, 'dbo:Album': 123, 'dbo:RacingDriver': 124, 'dbo:Insect': 125, 'dbo:Award': 126, 'dbo:Play': 127, 'dbo:RugbyClub': 128, 'dbo:WomensTennisAssociationTournament': 129, 'dbo:Senator': 130, 'dbo:EthnicGroup': 131, 'dbo:HorseTrainer': 132}, {'dbo:Location': 0, 'dbo:Place': 1, 'dbo:ArchitecturalStructure': 2, 'dbo:Person': 3, 'dbo:Work': 4, 'dbo:PopulatedPlace': 5, 'dbo:Agent': 6, 'dbo:NaturalPlace': 7, 'dbo:Organisation': 8, 'dbo:Event': 9, 'dbo:Athlete': 10, 'dbo:Eukaryote': 11, 'dbo:SocietalEvent': 12, 'dbo:CollegeCoach': 13, 'dbo:Species': 14, 'dbo:Building': 15, 'dbo:Bird': 16, 'dbo:Governor': 17, 'dbo:Settlement': 18, 'dbo:TelevisionShow': 19, 'dbo:Airport': 20, 'dbo:Band': 21, 'dbo:Insect': 22, 'dbo:Infrastructure': 23, 'dbo:Hospital': 24, 'dbo:Country': 25, 'dbo:SportsTeam': 26, 'dbo:OfficeHolder': 27, 'dbo:Entomologist': 28, 'dbo:University': 29, 'dbo:SportsClub': 30, 'dbo:AmericanFootballTeam': 31, 'dbo:Region': 32, 'dbo:SoccerPlayer': 33, 'dbo:Station': 34, 'dbo:Broadcaster': 35, 'dbo:Continent': 36, 'dbo:Town': 37, 'dbo:Cleric': 38, 'dbo:WorldHeritageSite': 39, 'dbo:Island': 40, 'dbo:Village': 41, 'dbo:City': 42, 'dbo:BasketballPlayer': 43, 'dbo:BaseballPlayer': 44, 'dbo:BodyOfWater': 45, 'dbo:Comedian': 46, 'dbo:MusicalArtist': 47, 'dbo:Bridge': 48, 'dbo:ProtectedArea': 49, 'dbo:CityDistrict': 50, 'dbo:Lake': 51, 'dbo:Group': 52, 'dbo:Road': 53, 'dbo:Airline': 54, 'dbo:Protein': 55, 'dbo:SportsEvent': 56, 'dbo:SoccerClub': 57, 'dbo:Museum': 58, 'dbo:Single': 59, 'dbo:MartialArtist': 60, 'dbo:HistoricPlace': 61, 'dbo:IceHockeyPlayer': 62, 'dbo:SportsLeague': 63, 'dbo:Castle': 64, 'dbo:TennisPlayer': 65, 'dbo:HandballTeam': 66, 'dbo:BaseballTeam': 67, 'dbo:GolfPlayer': 68, 'dbo:CelestialBody': 69, 'dbo:Bay': 70, 'dbo:Bank': 71, 'dbo:Genre': 72, 'dbo:Crater': 73, 'dbo:SoccerManager': 74, 'dbo:Wrestler': 75, 'dbo:MotorsportRacer': 76, 'dbo:Boxer': 77, 'dbo:MusicGenre': 78, 'dbo:Astronaut': 79, 'dbo:ComicsCreator': 80, 'dbo:SportsManager': 81, 'dbo:Company': 82, 'dbo:MusicalWork': 83, 'dbo:Scientist': 84, 'dbo:ComicStrip': 85, 'dbo:Coach': 86, 'dbo:Royalty': 87, 'dbo:PersonFunction': 88, 'dbo:RecordLabel': 89, 'dbo:Swimmer': 90, 'dbo:Cricketer': 91, 'dbo:Noble': 92, 'dbo:Book': 93, 'dbo:ScreenWriter': 94, 'dbo:AustralianFootballTeam': 95, 'dbo:RailwayLine': 96, 'dbo:WrittenWork': 97, 'dbo:Congressman': 98, 'dbo:Municipality': 99, 'dbo:Politician': 100, 'dbo:Painter': 101, 'dbo:ChessPlayer': 102, 'dbo:TelevisionStation': 103, 'dbo:President': 104, 'dbo:TableTennisPlayer': 105, 'dbo:Mayor': 106, 'dbo:AmusementParkAttraction': 107, 'dbo:NascarDriver': 108, 'dbo:RacingDriver': 109, 'dbo:MemberOfParliament': 110, 'dbo:PrimeMinister': 111, 'dbo:Poem': 112, 'dbo:Jockey': 113, 'dbo:MilitaryPerson': 114, 'dbo:FashionDesigner': 115, 'dbo:HorseTrainer': 116, 'dbo:MilitaryStructure': 117, 'dbo:Historian': 118, 'dbo:Automobile': 119, 'dbo:Skier': 120, 'dbo:Artist': 121, 'dbo:Monarch': 122, 'dbo:Cyclist': 123, 'dbo:FictionalCharacter': 124, 'dbo:River': 125, 'dbo:RadioHost': 126, 'dbo:PoliticalParty': 127, 'dbo:GridironFootballPlayer': 128}, {'dbo:Location': 0, 'dbo:Place': 1, 'dbo:Agent': 2, 'dbo:Settlement': 3, 'dbo:Person': 4, 'dbo:Species': 5, 'dbo:Event': 6, 'dbo:ArchitecturalStructure': 7, 'dbo:Insect': 8, 'dbo:PopulatedPlace': 9, 'dbo:Work': 10, 'dbo:Infrastructure': 11, 'dbo:Eukaryote': 12, 'dbo:Country': 13, 'dbo:Building': 14, 'dbo:Writer': 15, 'dbo:Venue': 16, 'dbo:Region': 17, 'dbo:Prison': 18, 'dbo:HistoricPlace': 19, 'dbo:Town': 20, 'dbo:ReligiousBuilding': 21, 'dbo:GridironFootballPlayer': 22, 'dbo:River': 23, 'dbo:FormulaOneTeam': 24, 'dbo:Village': 25, 'dbo:Film': 26, 'dbo:OfficeHolder': 27, 'dbo:University': 28, 'dbo:SportsClub': 29, 'dbo:PublicTransitSystem': 30, 'dbo:MusicalArtist': 31, 'dbo:Tournament': 32, 'dbo:NaturalPlace': 33, 'dbo:Museum': 34, 'dbo:Lake': 35, 'dbo:Band': 36, 'dbo:Road': 37, 'dbo:Organisation': 38, 'dbo:Biomolecule': 39, 'dbo:SocietalEvent': 40, 'dbo:AmericanFootballTeam': 41, 'dbo:City': 42, 'dbo:WorldHeritageSite': 43, 'dbo:Island': 44, 'dbo:Broadcaster': 45, 'dbo:AdministrativeRegion': 46, 'dbo:MountainRange': 47, 'dbo:WinterSportPlayer': 48, 'dbo:AnatomicalStructure': 49, 'dbo:SportsManager': 50, 'dbo:School': 51, 'dbo:Hotel': 52, 'dbo:Single': 53, 'dbo:TopicalConcept': 54, 'dbo:MotorcycleRider': 55, 'dbo:Group': 56, 'dbo:SoccerPlayer': 57, 'dbo:Cleric': 58, 'dbo:HistoricBuilding': 59, 'dbo:Album': 60, 'dbo:Congressman': 61, 'dbo:ComicsCreator': 62, 'dbo:MemberOfParliament': 63, 'dbo:Governor': 64, 'dbo:WrittenWork': 65, 'dbo:BaseballPlayer': 66, 'dbo:SkiArea': 67, 'dbo:Hospital': 68, 'dbo:Royalty': 69, 'dbo:Stadium': 70, 'dbo:Company': 71, 'dbo:Mammal': 72, 'dbo:RouteOfTransportation': 73, 'dbo:Coach': 74, 'dbo:SoccerClub': 75, 'dbo:MilitaryPerson': 76, 'dbo:MotorsportRacer': 77, 'dbo:CollegeCoach': 78, 'dbo:Station': 79, 'dbo:Magazine': 80, 'dbo:Scientist': 81, 'dbo:RaceHorse': 82, 'dbo:AdultActor': 83, 'dbo:ClassicalMusicArtist': 84, 'dbo:Castle': 85}])"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PorG_IvOK4_N",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "94fbe5bc-3bc0-414a-d117-dea7a0960424"
      },
      "source": [
        "resource_models = []\n",
        "\n",
        "for l in range(1, 6):\n",
        "  # model for resource classification\n",
        "  # get which rows are for resource only\n",
        "  # must only include rows that have the type at the level \n",
        "  # of classification \n",
        "  train_resource_rows = ((df_train[\"category\"] == \"resource\") & (df_train[\"type\"].map(lambda x: len(x)) >= l)).values\n",
        "  y_train_type = etl.type_to_int(df_train[train_resource_rows], type_no=l)\n",
        "\n",
        "  clf_type = MLPClassifier(\n",
        "      random_state=seed, max_iter=20, hidden_layer_sizes=(1000, 500, 300), verbose = 2).\\\n",
        "    fit(X_train_category[train_resource_rows], y_train_type)\n",
        "\n",
        "  resource_models.append(clf_type)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.14002143\n",
            "Iteration 2, loss = 1.03116504\n",
            "Iteration 3, loss = 0.62297222\n",
            "Iteration 4, loss = 0.37859177\n",
            "Iteration 5, loss = 0.25528809\n",
            "Iteration 6, loss = 0.19085766\n",
            "Iteration 7, loss = 0.15787654\n",
            "Iteration 8, loss = 0.13812974\n",
            "Iteration 9, loss = 0.12472322\n",
            "Iteration 10, loss = 0.11284379\n",
            "Iteration 11, loss = 0.10801872\n",
            "Iteration 12, loss = 0.10004498\n",
            "Iteration 13, loss = 0.09972821\n",
            "Iteration 14, loss = 0.09356413\n",
            "Iteration 15, loss = 0.09553271\n",
            "Iteration 16, loss = 0.08977378\n",
            "Iteration 17, loss = 0.08714985\n",
            "Iteration 18, loss = 0.09558418\n",
            "Iteration 19, loss = 0.10858665\n",
            "Iteration 20, loss = 0.10469689\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.08878833\n",
            "Iteration 2, loss = 0.99633147\n",
            "Iteration 3, loss = 0.60163255\n",
            "Iteration 4, loss = 0.38697955\n",
            "Iteration 5, loss = 0.27508625\n",
            "Iteration 6, loss = 0.21432184\n",
            "Iteration 7, loss = 0.18018812\n",
            "Iteration 8, loss = 0.15607034\n",
            "Iteration 9, loss = 0.14235186\n",
            "Iteration 10, loss = 0.13592778\n",
            "Iteration 11, loss = 0.12934041\n",
            "Iteration 12, loss = 0.12296272\n",
            "Iteration 13, loss = 0.12030714\n",
            "Iteration 14, loss = 0.11691538\n",
            "Iteration 15, loss = 0.11290476\n",
            "Iteration 16, loss = 0.11156831\n",
            "Iteration 17, loss = 0.10548721\n",
            "Iteration 18, loss = 0.10832688\n",
            "Iteration 19, loss = 0.10157562\n",
            "Iteration 20, loss = 0.10206518\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.13147989\n",
            "Iteration 2, loss = 1.16462734\n",
            "Iteration 3, loss = 0.77946541\n",
            "Iteration 4, loss = 0.54746403\n",
            "Iteration 5, loss = 0.41258115\n",
            "Iteration 6, loss = 0.32841607\n",
            "Iteration 7, loss = 0.27502682\n",
            "Iteration 8, loss = 0.24973910\n",
            "Iteration 9, loss = 0.22341338\n",
            "Iteration 10, loss = 0.21124127\n",
            "Iteration 11, loss = 0.19992840\n",
            "Iteration 12, loss = 0.19447786\n",
            "Iteration 13, loss = 0.18917758\n",
            "Iteration 14, loss = 0.18483735\n",
            "Iteration 15, loss = 0.17594290\n",
            "Iteration 16, loss = 0.16775517\n",
            "Iteration 17, loss = 0.16497381\n",
            "Iteration 18, loss = 0.16175275\n",
            "Iteration 19, loss = 0.16098529\n",
            "Iteration 20, loss = 0.15936385\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.37021883\n",
            "Iteration 2, loss = 1.34517080\n",
            "Iteration 3, loss = 0.91403112\n",
            "Iteration 4, loss = 0.66878606\n",
            "Iteration 5, loss = 0.50619872\n",
            "Iteration 6, loss = 0.40519070\n",
            "Iteration 7, loss = 0.33815534\n",
            "Iteration 8, loss = 0.30452070\n",
            "Iteration 9, loss = 0.29280518\n",
            "Iteration 10, loss = 0.25941069\n",
            "Iteration 11, loss = 0.24573571\n",
            "Iteration 12, loss = 0.23536696\n",
            "Iteration 13, loss = 0.22719698\n",
            "Iteration 14, loss = 0.21874023\n",
            "Iteration 15, loss = 0.21412281\n",
            "Iteration 16, loss = 0.20738422\n",
            "Iteration 17, loss = 0.20656873\n",
            "Iteration 18, loss = 0.20123626\n",
            "Iteration 19, loss = 0.19939758\n",
            "Iteration 20, loss = 0.20137851\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.05832300\n",
            "Iteration 2, loss = 1.21112987\n",
            "Iteration 3, loss = 0.86577162\n",
            "Iteration 4, loss = 0.62948037\n",
            "Iteration 5, loss = 0.46834557\n",
            "Iteration 6, loss = 0.36960250\n",
            "Iteration 7, loss = 0.32460028\n",
            "Iteration 8, loss = 0.27729944\n",
            "Iteration 9, loss = 0.25256042\n",
            "Iteration 10, loss = 0.23549953\n",
            "Iteration 11, loss = 0.21990894\n",
            "Iteration 12, loss = 0.21145509\n",
            "Iteration 13, loss = 0.19870009\n",
            "Iteration 14, loss = 0.19140986\n",
            "Iteration 15, loss = 0.18700273\n",
            "Iteration 16, loss = 0.18106746\n",
            "Iteration 17, loss = 0.17713714\n",
            "Iteration 18, loss = 0.17340469\n",
            "Iteration 19, loss = 0.16893132\n",
            "Iteration 20, loss = 0.16884262\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fxgGgeOMM9W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e51b8d9-53e3-4d24-b0fe-e0b6dfd3f941"
      },
      "source": [
        "clf_type.score(X_train_category[train_resource_rows], y_train_type) "
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9482627164796863"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwcZ38iu_YYH"
      },
      "source": [
        "## 4. Save models "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EJnfBXgkfyUr"
      },
      "source": [
        "with open(os.path.join(\"sklearn_objects\", \"category_model.pkl\"), \"wb\") as mdl_file:\n",
        "  pickle.dump(clf_category, mdl_file)"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_QaOf94RtQs"
      },
      "source": [
        "with open(os.path.join(\"sklearn_objects\", \"literal_model.pkl\"), \"wb\") as mdl_file:\n",
        "  pickle.dump(clf_literal, mdl_file)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASFCv0nkaphR"
      },
      "source": [
        "for l in range(1,6):\n",
        "  with open(os.path.join(\"sklearn_objects\", f\"resource_level_{l}_model.pkl\"), \"wb\") as mdl_file:\n",
        "    pickle.dump(resource_models[l-1], mdl_file)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pol2B42Cw2ck"
      },
      "source": [
        "## 5. Results & Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QV76HT3oUoxa"
      },
      "source": [
        "***Load Pre-trained models***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZmutptNCTqbc"
      },
      "source": [
        "with open(\"sklearn_objects/category_model.pkl\", \"rb\") as clf_cat_file:\n",
        "  clf_category = pickle.load(clf_cat_file)"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lxMJq2xsBOwL"
      },
      "source": [
        "with open(\"sklearn_objects/literal_model.pkl\", \"rb\") as clf_lit_file:\n",
        "  clf_literal = pickle.load(clf_lit_file)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Jx_Y4PaBQ0q"
      },
      "source": [
        "resource_models = []\n",
        "for l in range(1,6):\n",
        "  with open(f\"sklearn_objects/resource_level_{l}_model.pkl\", \"rb\") as res_mdl:\n",
        "    resource_models.append(pickle.load(res_mdl))"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZP9mCDz0BTeD"
      },
      "source": [
        "***Load processing class***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PYLTHE0gUll3"
      },
      "source": [
        "etl = ETL(path_to_type_maps=\"resource_types\", path_to_vectorizers=\"sklearn_objects\")"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orIFdQaDN9ba"
      },
      "source": [
        "class ModelEvaluation:\n",
        "  def __init__(self, etl_inst, cat_model, lit_model, res_models):\n",
        "    self.etl_inst = etl_inst\n",
        "    self.cat_model = cat_model\n",
        "    self.lit_model = lit_model\n",
        "    self.res_models = res_models\n",
        "  \n",
        "  # X is a df\n",
        "  def get_predictions(self, X, bow_type = \"tf\"):\n",
        "\n",
        "    X = X.copy()\n",
        "\n",
        "    X.reset_index(inplace = True, drop = True)\n",
        "    \n",
        "    X_norm = self.etl_inst.norm_data(X)\n",
        "    X_vec = self.etl_inst.bow_transform(X_norm, type = bow_type)\n",
        "\n",
        "    bool_int = self.etl_inst.category_map[\"boolean\"]\n",
        "    literal_int = self.etl_inst.category_map[\"literal\"]\n",
        "    resource_int = self.etl_inst.category_map[\"resource\"]\n",
        "\n",
        "    cat_pred = self.cat_model.predict(X_vec)\n",
        "\n",
        "    ind_bool = cat_pred == bool_int\n",
        "    ind_literal = cat_pred == literal_int\n",
        "    ind_resource = cat_pred == resource_int\n",
        "\n",
        "    if len(ind_bool) > 0:\n",
        "      X.loc[ind_bool, \"cat_prediction\"] = \"boolean\"\n",
        "      X.loc[ind_bool, \"type_prediction\"] = pd.Series(\n",
        "          cat_pred[ind_bool], name = \"type_prediction\")\\\n",
        "          .map(lambda x: [\"boolean\"]).values\n",
        "\n",
        "    if len(ind_literal) > 0:\n",
        "      X.loc[ind_literal, \"cat_prediction\"] = \"literal\"\n",
        "      literal_pred = self.lit_model.predict(X_vec[ind_literal])\n",
        "      X.loc[ind_literal, \"type_prediction\"] = pd.Series(\n",
        "          literal_pred, name = \"type_prediction\")\\\n",
        "          .map(lambda x: [self.etl_inst.inv_literal_map[x]]).values\n",
        "\n",
        "    if len(ind_resource) > 0:\n",
        "      resource_preds = []\n",
        "      for ind, type_model in enumerate(self.res_models):\n",
        "          resource_preds.append(\n",
        "            pd.Series(\n",
        "                type_model.predict(X_vec[ind_resource]), name = f\"type_{ind}\").\\\n",
        "                map(lambda x: self.etl_inst.invtype_maps[f\"type{ind+1}\"][x])\n",
        "                )\n",
        "      resource_preds = pd.Series(pd.concat(resource_preds, axis = 1).values.tolist(), name = \"type_prediction\")\n",
        "      X.loc[ind_resource, \"type_prediction\"] = resource_preds.values\n",
        "      X.loc[ind_resource, \"cat_prediction\"] = \"resource\"\n",
        "      \n",
        "      return X\n",
        "\n",
        "  def output_predictions(self):\n",
        "    return NotImplementedError"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D6BLutJ8jA-M"
      },
      "source": [
        "me = ModelEvaluation(etl, clf_category, clf_literal, resource_models)"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6b_rDe2EKOry"
      },
      "source": [
        "***Validate***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "00lDYJIE0pit"
      },
      "source": [
        "# out_val = me.get_predictions(df_val)\n",
        "\n",
        "# true_output = out_val.loc[:, [\"id\", \"question\", \"category\", \"type\"]]\n",
        "# true_output_dict = [pred for ind, pred in true_output.to_dict(orient = \"index\").items()]\n",
        "\n",
        "# system_output = out_val.loc[:, [\"id\", \"cat_prediction\", \"type_prediction\"]]\n",
        "# system_output.columns = [\"id\", \"category\", \"type\"]\n",
        "# system_output_dict = [pred for ind, pred in system_output.to_dict(orient = \"index\").items()]"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k9raG_KIyB1x"
      },
      "source": [
        "***Run Evaluation***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtiKCQsPxH_V"
      },
      "source": [
        "# os.makedirs(\"system_output/\", exist_ok = True)\n",
        "# with open(os.path.join(\"system_output\", \"ground_truth_json.json\"), \"w\") as gfile:\n",
        "#   json.dump(true_output_dict, gfile)\n",
        "\n",
        "# with open(os.path.join(\"system_output\", \"system_output_json.json\"), \"w\") as sfile:\n",
        "#   json.dump(system_output_dict, sfile)"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zPusetmQx_T7"
      },
      "source": [
        "# !python evaluate.py --type_hierarchy_tsv dbpedia_types.tsv  \\\n",
        "#  --ground_truth_json system_output/ground_truth_json.json \\\n",
        "#  --system_output_json system_output/system_output_json.json"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "plQTw8kzKQmC"
      },
      "source": [
        "***Save Test Output***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IvHRpUYDKHm6"
      },
      "source": [
        "out_test = me.get_predictions(test_data2021)\n",
        "\n",
        "#true_output = out_test.loc[:, [\"id\", \"question\"]]\n",
        "#true_output_dict = [pred for ind, pred in true_output.to_dict(orient = \"index\").items()]\n",
        "\n",
        "system_output = out_test.loc[:, [\"id\", \"cat_prediction\", \"type_prediction\"]]\n",
        "system_output.columns = [\"id\", \"category\", \"type\"]\n",
        "system_output_dict = [pred for ind, pred in system_output.to_dict(orient = \"index\").items()]"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mZ4WgtQq2k1g"
      },
      "source": [
        "os.makedirs(\"system_output/\", exist_ok = True)\n",
        "# with open(os.path.join(\"system_output\", \"ground_truth_json.json\"), \"w\") as gfile:\n",
        "#   json.dump(true_output_dict, gfile)\n",
        "\n",
        "with open(os.path.join(\"system_output\", \"system_output_json.json\"), \"w\") as sfile:\n",
        "  json.dump(system_output_dict, sfile)"
      ],
      "execution_count": 47,
      "outputs": []
    }
  ]
}