main.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GT.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "FrHVAW_sKbrO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import csv\n",
        "import pandas as pd\n",
        "from pandas import DataFrame\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "\n",
        "np.random.seed(113)\n",
        "path_ukara = \"drive/My Drive/Ukara/\"\n",
        "path_nlp = \"drive/My Drive/a/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ta4lq7xRK7rt",
        "colab_type": "code",
        "outputId": "de732a0e-bf18-4231-a9a0-54289b326a2a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OCPUst8JKv_0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_A = pd.read_csv(path_ukara + \"data_train_A.csv\").values\n",
        "data_B = pd.read_csv(path_ukara + \"data_train_B.csv\").values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E2Y5I6FRLohS",
        "colab_type": "code",
        "outputId": "5afeeaa7-4eee-4e47-a830-550ac7b372f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        }
      },
      "source": [
        "data_A[:3]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([['TRA1', 'intetraksi/beradaptasi terhadap lingkungan yang baru.',\n",
              "        1],\n",
              "       ['TRA2', 'seperti jatuhnya meteor tsunami gempa bumi', 0],\n",
              "       ['TRA3', 'hanya tuhan yang tahu tantangan nya itu apaan', 0]],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7QxrgpL-LsNy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xtrain_A, xtest_A, ytrain_A, ytest_A = train_test_split(data_A[:,:-1], data_A[:,-1],\n",
        "                                      test_size=0.2, \n",
        "                                      stratify=data_A[:,-1], \n",
        "                                      random_state=113)\n",
        "\n",
        "xtrain_B, xtest_B, ytrain_B, ytest_B = train_test_split(data_B[:,:-1], data_B[:,-1],\n",
        "                                      test_size=0.2, \n",
        "                                      stratify=data_B[:,-1], \n",
        "                                      random_state=113)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NlioKY4FzpZF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rwUnaw0WTCbk",
        "colab_type": "code",
        "outputId": "77f9b3dd-976f-4dfe-bc74-ccbeb7320c8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(xtrain_A[0], ytrain_A[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['TRA226' 'negara munggin menjari donasi hidup baru'] 0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jwIpnescaGyJ",
        "colab_type": "code",
        "outputId": "2b24a57d-b9e3-4f07-9f4e-938fc81fe054",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(np.unique(data_A[:,-1], return_counts=True))\n",
        "print(np.unique(data_B[:,-1], return_counts=True))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(array([0, 1], dtype=object), array([ 77, 191]))\n",
            "(array([0, 1], dtype=object), array([137, 168]))\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qRfGi0Sbenq_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def diff(first, second):\n",
        "    second = set(second)\n",
        "    return [item for item in first if item not in second]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WpU9mAfaMhOF",
        "colab_type": "text"
      },
      "source": [
        "# Fasttext Part"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U0dBD6LfMbOl",
        "colab_type": "code",
        "outputId": "a88f6e4d-02db-446b-d5a0-e3d8bd2644c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        }
      },
      "source": [
        "!pip install fasttext"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting fasttext\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/10/61/2e01f1397ec533756c1d893c22d9d5ed3fce3a6e4af1976e0d86bb13ea97/fasttext-0.9.1.tar.gz (57kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 2.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: pybind11>=2.2 in /usr/local/lib/python3.6/dist-packages (from fasttext) (2.4.2)\n",
            "Requirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from fasttext) (41.2.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from fasttext) (1.16.5)\n",
            "Building wheels for collected packages: fasttext\n",
            "  Building wheel for fasttext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fasttext: filename=fasttext-0.9.1-cp36-cp36m-linux_x86_64.whl size=2384810 sha256=8f8f730e115254945e09ef2636fed4dd2418c1e43ebefaf0de492c0d8c833f77\n",
            "  Stored in directory: /root/.cache/pip/wheels/9f/f0/04/caa82c912aee89ce76358ff954f3f0729b7577c8ff23a292e3\n",
            "Successfully built fasttext\n",
            "Installing collected packages: fasttext\n",
            "Successfully installed fasttext-0.9.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "guq49kx1M2VE",
        "colab_type": "code",
        "outputId": "92407cce-f372-4a38-bfa1-d2fd1f674f90",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 239
        }
      },
      "source": [
        "!wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.id.300.bin.gz"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-10-06 15:08:31--  https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.id.300.bin.gz\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.20.22.166, 104.20.6.166, 2606:4700:10::6814:16a6, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.20.22.166|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4507049071 (4.2G) [application/octet-stream]\n",
            "Saving to: ‘cc.id.300.bin.gz’\n",
            "\n",
            "cc.id.300.bin.gz    100%[===================>]   4.20G  29.6MB/s    in 2m 28s  \n",
            "\n",
            "2019-10-06 15:11:00 (29.0 MB/s) - ‘cc.id.300.bin.gz’ saved [4507049071/4507049071]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uUi9O3ytM4so",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!gunzip cc.id.300.bin.gz"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S0IQPsOCxDAx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !mv \"cc.id.300.bin\" \"drive/My Drive/a/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NYiMCGC7M7U5",
        "colab_type": "code",
        "outputId": "4e6488d6-4b9f-4020-e8dc-a3c67fe8c6b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import fasttext\n",
        "model = fasttext.load_model(path_nlp + \"trained_fasttext_ukara.pk\")\n",
        "# print(model['mesin'][:3])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zeOHuKsV_V3G",
        "colab_type": "code",
        "outputId": "ed07b605-bf77-426b-ba34-312b6d0bb8b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import fasttext\n",
        "model2 = fasttext.load_model(\"cc.id.300.bin\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ux7aPk7LkeY",
        "colab_type": "text"
      },
      "source": [
        "# PreProcess"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8Zwdi1pOjHa",
        "colab_type": "code",
        "outputId": "6cd4a4e1-90bd-42e8-da87-1341e06128ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        }
      },
      "source": [
        "!pip install PySastrawi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting PySastrawi\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/61/84/b0a5454a040f81e81e6a95a5d5635f20ad43cc0c288f8b4966b339084962/PySastrawi-1.2.0-py2.py3-none-any.whl (210kB)\n",
            "\u001b[K     |████████████████████████████████| 215kB 2.8MB/s \n",
            "\u001b[?25hInstalling collected packages: PySastrawi\n",
            "Successfully installed PySastrawi-1.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LJLCW5k-Oeef",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import string\n",
        "from nltk.util import skipgrams\n",
        "from nltk import ngrams\n",
        "from collections import Counter\n",
        "import math\n",
        "\n",
        "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
        "factory = StemmerFactory()\n",
        "stemmer = factory.create_stemmer()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5sk7yPrvTSc9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lena = []\n",
        "for d in data_A[:,1]:\n",
        "    lena.append(len(str(d).split()))\n",
        "datalen_mean_A = np.mean(lena)\n",
        "datalen_std_A = np.std(lena)\n",
        "\n",
        "lenb = []\n",
        "for d in data_B[:,1]:\n",
        "    lenb.append(len(str(d).split()))\n",
        "datalen_mean_B = np.mean(lenb)\n",
        "datalen_std_B = np.std(lenb)\n",
        "\n",
        "stop_b = ['yang', 'lebih', 'untuk', 'dan']\n",
        "stop_a = ['yang', 'akan', 'mereka', 'dan']\n",
        "\n",
        "table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
        "def rmpun(text):\n",
        "    return text.translate(table)\n",
        "    \n",
        "def preprocess(data, tipe=\"A\"):\n",
        "    if type(data) != str:\n",
        "        return \" \"\n",
        "    if len(data) <1:\n",
        "        return \" \"\n",
        "    data = data.lower()\n",
        "    data = data.replace('/',' atau ')\n",
        "    data = data.replace('yg ','yang ')\n",
        "    data = rmpun(data)\n",
        "    data = \" \".join(data.split())\n",
        "    if tipe==\"A\":\n",
        "        for w in stop_a:\n",
        "            data = data.replace(w, '')\n",
        "    else:\n",
        "        for w in stop_b:\n",
        "            data = data.replace(w, '')\n",
        "    return data\n",
        "\n",
        "def preprocess_2(data, tipe=\"A\"):\n",
        "    data = data.lower()\n",
        "    data = data.replace('/',' atau ')\n",
        "    data = data.replace('\"','')\n",
        "    data = data.replace('\\'','')\n",
        "    data = data.replace('yg ','yang ')\n",
        "    data = data.replace('_',' ')\n",
        "    data = data.replace(',',' , ')\n",
        "    return data\n",
        "\n",
        "def vectorize(x, pre=True, tipe=\"A\"):\n",
        "    ans = []\n",
        "    for data in x:\n",
        "        if pre:\n",
        "            data = preprocess(str(data), tipe)\n",
        "        vect = model.get_sentence_vector(data)\n",
        "        ans.append(vect)\n",
        "    return np.asarray(ans)\n",
        "\n",
        "def vectorize2(x, pre=True, tipe=\"A\"):\n",
        "    ans = []\n",
        "    for data in x:\n",
        "        if pre:\n",
        "            data = preprocess_2(str(data), tipe)\n",
        "        vect = model2.get_sentence_vector(data)\n",
        "        ans.append(vect)\n",
        "    return np.asarray(ans)\n",
        "\n",
        "def find_len(x, tipe='A'):\n",
        "    ans = []\n",
        "    for data in x:\n",
        "        ans.append(len(str(data).split()))\n",
        "    ans = np.asarray(ans, dtype=\"float32\")\n",
        "    \n",
        "    if tipe==\"A\":\n",
        "        datalen_mean = datalen_mean_A\n",
        "        datalen_std = datalen_std_A\n",
        "    else:\n",
        "        datalen_mean = datalen_mean_B\n",
        "        datalen_std = datalen_std_B\n",
        "    \n",
        "    ans -= datalen_mean\n",
        "    return ans / datalen_std\n",
        "\n",
        "# 1, 58"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U4X4rmk3Lm2K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
        "# def rmpun(text):\n",
        "#     return text.translate(table)\n",
        "    \n",
        "# def preprocess_istri(data):\n",
        "#     if type(data) != str:\n",
        "#         return \" \"\n",
        "#     if len(data) <1:\n",
        "#         return \" \"\n",
        "#     data = data.lower()\n",
        "#     data = data.replace('/',' atau ')\n",
        "#     data = data.replace('yg ','yang ')\n",
        "#     data = rmpun(data)\n",
        "#     data = \" \".join(data.split())\n",
        "#     # data = stem(data)\n",
        "#     return data\n",
        "\n",
        "# word_b = \"\"\n",
        "# for data in data_A:\n",
        "#     word_b += preprocess_istri(data[1])\n",
        "# counter = Counter(word_b.split())\n",
        "# counter = list(reversed(counter.most_common()))\n",
        "# counter[:50]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VBPRxMypOcTO",
        "colab_type": "text"
      },
      "source": [
        "# PPMI"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P1tTSHXGOx87",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
        "def rmpun(text):\n",
        "    return text.translate(table)\n",
        "\n",
        "def stem(text):\n",
        "    text = stemmer.stem(text)\n",
        "    return text\n",
        "\n",
        "def preprocess_ppmi(data):\n",
        "    if type(data) != str:\n",
        "        return \" \"\n",
        "    if len(data) <1:\n",
        "        return \" \"\n",
        "    data = data.lower()\n",
        "    data = data.replace('/',' atau ')\n",
        "    data = data.replace('yg ','yang ')\n",
        "    data = rmpun(data)\n",
        "    data = \" \".join(data.split())\n",
        "    # data = stem(data)\n",
        "    return data\n",
        "\n",
        "def pmi(w1, w2, unifreq, bifreq):\n",
        "    pw1 = unifreq[w1] / float(sum(unifreq.values()))\n",
        "    pw2 = unifreq[w2] / float(sum(unifreq.values()))\n",
        "    pw1w2 = bifreq[\" \".join([w1, w2])] / float(sum(bifreq.values()))\n",
        "    try:\n",
        "        res = math.log(pw1w2/float(pw1*pw2),2)\n",
        "        if res < 0:\n",
        "            res = 0\n",
        "    except:\n",
        "        res = 0\n",
        "    return res\n",
        "\n",
        "def mkbigram(text):\n",
        "    gram2 = list(ngrams(text.split(), 2))\n",
        "    if len(gram2) == 0:\n",
        "        return [text]\n",
        "    return gram2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pzYis9-eO2vq",
        "colab_type": "code",
        "outputId": "a5afbe90-f527-48c3-cea3-ed21452fb94a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "traina1ex = [\"Sebuah negara membeli pulau membeli tanah untuk memindahkan penduduknya di sana suatu hari.\",\n",
        "                \"Pemerintah berharap tidak menempatkan semua penduduk semua warga semua orang di sebidang tanah.\", \n",
        "                \"Tetapi jika itu diperlukan untuk menyelamatkan warga penduduk maka akan dilakukan.\",\n",
        "                \"Pulau tanah itu akan menjadi rumah baru bagi penduduk warga yang berpindah.\",\n",
        "                \"Relokasi warga penduduk karena perubahan iklim.\",\n",
        "                \"Sebuah negara sudah mulai memindahkan populasi penduduk dari daratan rendah ke sebuah pulau lain.\",\n",
        "                \"Tapi ini akan menjadi pertama kalinya seluruh negara harus pindah karena tanah yang dibangun tidak lagi ada.\",\n",
        "                \"Ini menimbulkan pertanyaan baru dan menakutkan: Jika suatu negara tidak lagi ada dalam bentuk fisik, dapatkah ia tetap ada sebagai entitas politik?\",\n",
        "                \"Bisakah suatu bangsa bangkit dan bergerak?\",\n",
        "                \"Jika negara itu berpindah kita tidak tahu apakah kebangsaan bangsa penduduknya bisa dilestarikan.\",\n",
        "                \"Negara harus tinggal, dilestarikan, jika memungkinkan.\",\n",
        "                \"Relokasi mungkin tidak dapat dihindari.\",\n",
        "                \"Meski dengan aktivis gerakan nol emisi tetapi negara itu masih tenggelam.\",\n",
        "                \"Secara historis, negara tidak dihancurkan secara fisik; mereka hanya menjadi negara lain,\",\n",
        "                \"tanah yang mereka tempati dikendalikan oleh orang lain.\",\n",
        "                \"Tetapi setidaknya, untuk eksis, suatu negara membutuhkan pemerintah, populasi, dan sebidang pemukiman di dalam wilayah yang baru.\",\n",
        "                \"untuk mempertahankan pengakuan internasional, negara kepulauan yang menghadapi kehancuran karena perubahan iklim harus memperkuat wilayah mereka\",\n",
        "                \"menjaga setidaknya beberapa struktur fisik di atas air dan menjaga sekelompok kecil penduduk, bahkan jika sebagian besar penduduk telah pindah.\",\n",
        "                \"Orang-orang dari negara kepulauan kecil ingin terus memiliki perwakilan politik dalam komunitas internasional, dan mereka memiliki kepentingan ekonomi untuk dilindungi - hak atas perikanan dan sumber daya alam di wilayah mereka\",\n",
        "                \"Sangat masuk akal bahwa dampak perubahan iklim akan membuat negara bangsa tertentu tidak dapat dihuni sebelum akhir abad ini.\",\n",
        "                \"Walaupun ini mungkin nasib sejumlah kecil negara bangsa yang paling rentan terhadap perubahan iklim\",\n",
        "                \"implikasinya terhadap evolusi kenegaraan dan hukum internasional.\",\n",
        "                \"Untuk menanggapi fenomena negara tak bertanah.\",\n",
        "                \"status yang memungkinkan keberlangsungan keberadaan negara berdaulat, memberikan semua hak dan manfaat kedaulatan di antara keluarga negara, untuk selamanya.\",\n",
        "                \"Dalam praktiknya hal ini membutuhkan pembentukan kerangka kerja pemerintah baru yang dapat menggunakan wewenang atas penduduk yang tersebar karena perpindahan\",\n",
        "                \"Prospek pendidikan dan pekerjaan di pulau terbatas, dan mayoritas anak muda yang keluarganya mampu berpindah ke negara lain.\",\n",
        "                \"Banyak orang berniat bermigrasi untuk merespons perubahan iklim.\",\n",
        "                \"sebagian besar generasi tua tidak ingin pindah karena mereka percaya akan kehilangan identitas, budaya, gaya hidup dan tradisi mereka.\",\n",
        "                \"Tapi pemerintah percaya bahwa generasi muda berniat untuk bermigrasi demi keselamatan generasi mendatang.\"]\n",
        "len(traina1ex)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "29"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "POQbpgw7QM6B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_ppmi(x_lengkap):\n",
        "    traina1 = x_lengkap[x_lengkap[:,2] == 1, 1]\n",
        "    # print(\"===================\")\n",
        "    # print(traina1)\n",
        "    # print(\"===================\")\n",
        "    # cleansing\n",
        "    trainAclear = [preprocess_ppmi(text) for text in list(traina1)]\n",
        "    trainAclear.extend([preprocess_ppmi(text) for text in list(traina1ex)])\n",
        "\n",
        "    # combination skipgram\n",
        "    unigram = []\n",
        "    bigram = []\n",
        "    for sent in trainAclear:        \n",
        "        unigram.extend(sent.split())\n",
        "        bigram.extend([\" \".join(item) for item in list(ngrams(sent.split(), 2))])\n",
        "        \n",
        "    count = Counter(unigram)\n",
        "    countbigram = Counter(bigram)\n",
        "\n",
        "    # print(\"===================\")\n",
        "    # print(unigram)\n",
        "    # print(\"===================\")\n",
        "    # print(bigram)\n",
        "    # print(\"===================\")\n",
        "\n",
        "    # traina['CL RESPONSE'] = traina.RESPONSE.apply(preprocess)\n",
        "    PPMI = []\n",
        "    idx = 0\n",
        "    for sent in x_lengkap[:,1]:\n",
        "        sent = preprocess_ppmi(sent)\n",
        "        try:\n",
        "            big = mkbigram(sent)\n",
        "            ppmi = sum([pmi(item[0],item[1],count,countbigram) for item in big])/len(big)\n",
        "            PPMI.append(ppmi)\n",
        "        except:\n",
        "            PPMI.append(0)\n",
        "        # print(idx)\n",
        "        idx+=1\n",
        "\n",
        "    return PPMI, count, countbigram\n",
        "\n",
        "def get_ppmi_test(x_lengkap, count, countbigram):\n",
        "    PPMI = []\n",
        "    idx = 0\n",
        "    for sent in x_lengkap[:,1]:\n",
        "        sent = preprocess_ppmi(sent)\n",
        "        try:\n",
        "            big = mkbigram(sent)\n",
        "            ppmi = sum([pmi(item[0],item[1],count,countbigram) for item in big])/len(big)\n",
        "            PPMI.append(ppmi)\n",
        "        except:\n",
        "            PPMI.append(0)\n",
        "        # print(idx)\n",
        "        idx+=1\n",
        "    return PPMI"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W_nhtWdQO7iN",
        "colab_type": "text"
      },
      "source": [
        "# Classifier Booster 1\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1cNtdS6lNBHP",
        "colab_type": "code",
        "outputId": "c2209513-b21c-4c39-a264-fc1d8d564e28",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 166
        }
      },
      "source": [
        "!pip install xgboost imblearn"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: xgboost in /usr/local/lib/python3.6/dist-packages (0.90)\n",
            "Requirement already satisfied: imblearn in /usr/local/lib/python3.6/dist-packages (0.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.16.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.3.1)\n",
            "Requirement already satisfied: imbalanced-learn in /usr/local/lib/python3.6/dist-packages (from imblearn) (0.4.3)\n",
            "Requirement already satisfied: scikit-learn>=0.20 in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (0.21.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.20->imbalanced-learn->imblearn) (0.13.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5rGCQuvRPKmn",
        "colab_type": "code",
        "outputId": "80ebba95-6590-49e1-cba8-c884364d2a90",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        }
      },
      "source": [
        "from xgboost import XGBClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "from sklearn.svm import SVC\n",
        "from imblearn.over_sampling import SMOTE\n",
        "\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/externals/six.py:31: DeprecationWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).\n",
            "  \"(https://pypi.org/project/six/).\", DeprecationWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iSMOn1Bzaafc",
        "colab_type": "text"
      },
      "source": [
        "### DATA A"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yc5IJskB0RYV",
        "colab_type": "code",
        "outputId": "c59cad9f-7502-4dfb-dec6-15f25ed4f6c8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "X = vectorize2(data_A[:,1].copy(), pre=True, tipe=\"A\")\n",
        "# XE = np.load(\"A_X.npy\")\n",
        "Y = np.load(\"A_Y.npy\")\n",
        "\n",
        "X = np.c_[X, find_len(data_A[:,1])]\n",
        "\n",
        "sampler = SMOTE(sampling_strategy=0.5, random_state=113)\n",
        "\n",
        "# clf = MLPClassifier(\n",
        "#         random_state=113,\n",
        "#         max_iter=250,\n",
        "#         hidden_layer_sizes=(200,), # 250,256-64\n",
        "#         beta_1=0.9,\n",
        "#         beta_2=0.999,\n",
        "#         learning_rate_init=0.0082,\n",
        "#         learning_rate = 'adaptive',\n",
        "#         solver='adam')\n",
        "\n",
        "hiddens = [(100,),(200,),(300,),(400)]\n",
        "hiddens = [100, 150, 200]\n",
        "lris = [0.001, 0.01, 0.005]\n",
        "solvers = ['adam']\n",
        "max_iters = [300, 400,500]\n",
        "max_iters = [0.8, 0.9]\n",
        "# lrs = ['adaptive', 'constant', 'invscaling']\n",
        "use_ppmi = False\n",
        "\n",
        "# hiddens = [(250,)]\n",
        "# lris = [0.0005]\n",
        "# solvers = ['adam']\n",
        "# max_iters = [300]\n",
        "lrs = ['constant']\n",
        "\n",
        "skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=False)\n",
        "\n",
        "for h in hiddens:\n",
        "    for lri in lris:\n",
        "        for solver in solvers:\n",
        "            for mi in max_iters:\n",
        "                for lr in lrs:\n",
        "                    print()\n",
        "                    print(h, lri, solver, mi, lr)\n",
        "                    \n",
        "                    avg_f1 = 0\n",
        "#                     clf = MLPClassifier(\n",
        "#                             random_state=113,\n",
        "#                             max_iter=mi,\n",
        "#                             hidden_layer_sizes=h, # 250,256-64\n",
        "#                             learning_rate_init=lri,\n",
        "#                             learning_rate = lr,\n",
        "#                             solver=solver)\n",
        "                    \n",
        "                    \n",
        "                    clf = XGBClassifier(random_state=113, n_estimators=h,\n",
        "                                        max_depth=3, learning_rate=lri, subsample=mi)\n",
        "                    \n",
        "                    done = True\n",
        "                    for i, (train_index, test_index) in enumerate(skf.split(X, Y)):\n",
        "                        done = False\n",
        "                        x_train, y_train = X[train_index], Y[train_index]\n",
        "                        \n",
        "                        if use_ppmi:\n",
        "                            ppmi_x_train, ppmic, ppmicb = get_ppmi(data_A[train_index])\n",
        "                            x_train = np.c_[x_train, ppmi_x_train]\n",
        "\n",
        "                        x_train, y_train = sampler.fit_resample(x_train, y_train)\n",
        "\n",
        "                        x_test, y_test = X[test_index], Y[test_index]\n",
        "                        \n",
        "                        if use_ppmi:\n",
        "                            ppmi_x_test = get_ppmi_test(data_A[test_index], ppmic, ppmicb)\n",
        "                            x_test = np.c_[x_test, ppmi_x_test]\n",
        "\n",
        "                        clf.fit(x_train, y_train)\n",
        "\n",
        "                        prediction = clf.predict(x_test).astype(int)\n",
        "\n",
        "                        # print(\"Accuracy pada train data:\", accuracy_score(clf.predict(x_train).astype(int), y_train))\n",
        "                        # print(\"Accuracy pada test data:\", accuracy_score(prediction, y_test))\n",
        "                        # print(\"Precision baseline pada test data:\", precision_score(prediction, y_test))\n",
        "                        # print(\"Recall baseline pada test data:\", recall_score(prediction, y_test))\n",
        "                        print(\"F1 baseline pada test data:\", f1_score(prediction, y_test))\n",
        "                        print()\n",
        "                        avg_f1 += f1_score(prediction, y_test)\n",
        "\n",
        "#                         if f1_score(prediction, y_test)<0.9 and i==0:\n",
        "#                             break\n",
        "#                         if f1_score(prediction, y_test)<0.87 and i==3:\n",
        "#                             break\n",
        "                        done = True\n",
        "                    \n",
        "                    if done:\n",
        "                        print(\"Average F1\", avg_f1/5)\n",
        "\n",
        "                    # clf = MLPClassifier(\n",
        "                    #         random_state=113,\n",
        "                    #         max_iter=mi,\n",
        "                    #         hidden_layer_sizes=h,\n",
        "                    #         learning_rate_init=lri,\n",
        "                    #         learning_rate = lr,\n",
        "                    #         solver=solver)\n",
        "                    \n",
        "                    # if use_ppmi:\n",
        "                    #     X = np.c_[X, ppmi_a]\n",
        "\n",
        "                    # X_resampled, Y_resampled = sampler.fit_resample(X, Y)\n",
        "                    # clf.fit(X_resampled, Y_resampled)\n",
        "   "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "100 0.001 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8604651162790696\n",
            "\n",
            "F1 baseline pada test data: 0.7894736842105263\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8275862068965517\n",
            "\n",
            "F1 baseline pada test data: 0.8860759493670887\n",
            "\n",
            "Average F1 0.8393868580173139\n",
            "\n",
            "100 0.001 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8604651162790696\n",
            "\n",
            "F1 baseline pada test data: 0.7894736842105263\n",
            "\n",
            "F1 baseline pada test data: 0.8192771084337349\n",
            "\n",
            "F1 baseline pada test data: 0.8470588235294116\n",
            "\n",
            "F1 baseline pada test data: 0.9090909090909091\n",
            "\n",
            "Average F1 0.8450731283087304\n",
            "\n",
            "100 0.01 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8705882352941177\n",
            "\n",
            "F1 baseline pada test data: 0.810126582278481\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8505747126436782\n",
            "\n",
            "F1 baseline pada test data: 0.9367088607594938\n",
            "\n",
            "Average F1 0.8602663448618209\n",
            "\n",
            "100 0.01 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8470588235294118\n",
            "\n",
            "F1 baseline pada test data: 0.7948717948717949\n",
            "\n",
            "F1 baseline pada test data: 0.8470588235294116\n",
            "\n",
            "F1 baseline pada test data: 0.8409090909090909\n",
            "\n",
            "F1 baseline pada test data: 0.8974358974358975\n",
            "\n",
            "Average F1 0.8454668860551214\n",
            "\n",
            "100 0.005 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8705882352941177\n",
            "\n",
            "F1 baseline pada test data: 0.8051948051948051\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8735632183908045\n",
            "\n",
            "F1 baseline pada test data: 0.9113924050632912\n",
            "\n",
            "Average F1 0.8588143994552704\n",
            "\n",
            "100 0.005 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8470588235294118\n",
            "\n",
            "F1 baseline pada test data: 0.7733333333333334\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8409090909090909\n",
            "\n",
            "F1 baseline pada test data: 0.935064935064935\n",
            "\n",
            "Average F1 0.845939903234021\n",
            "\n",
            "150 0.001 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8705882352941177\n",
            "\n",
            "F1 baseline pada test data: 0.7894736842105263\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8571428571428571\n",
            "\n",
            "F1 baseline pada test data: 0.8974358974358975\n",
            "\n",
            "Average F1 0.8495948014833464\n",
            "\n",
            "150 0.001 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8604651162790696\n",
            "\n",
            "F1 baseline pada test data: 0.7894736842105263\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8275862068965517\n",
            "\n",
            "F1 baseline pada test data: 0.9090909090909091\n",
            "\n",
            "Average F1 0.8439898499620779\n",
            "\n",
            "150 0.01 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8809523809523809\n",
            "\n",
            "F1 baseline pada test data: 0.8148148148148148\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8505747126436782\n",
            "\n",
            "F1 baseline pada test data: 0.9367088607594938\n",
            "\n",
            "Average F1 0.8632768205007402\n",
            "\n",
            "150 0.01 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8433734939759037\n",
            "\n",
            "F1 baseline pada test data: 0.7948717948717949\n",
            "\n",
            "F1 baseline pada test data: 0.8470588235294116\n",
            "\n",
            "F1 baseline pada test data: 0.8505747126436782\n",
            "\n",
            "F1 baseline pada test data: 0.8974358974358975\n",
            "\n",
            "Average F1 0.8466629444913373\n",
            "\n",
            "150 0.005 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8809523809523809\n",
            "\n",
            "F1 baseline pada test data: 0.7948717948717949\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8604651162790697\n",
            "\n",
            "F1 baseline pada test data: 0.9\n",
            "\n",
            "Average F1 0.8539245250873158\n",
            "\n",
            "150 0.005 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8470588235294118\n",
            "\n",
            "F1 baseline pada test data: 0.7894736842105263\n",
            "\n",
            "F1 baseline pada test data: 0.8470588235294116\n",
            "\n",
            "F1 baseline pada test data: 0.8409090909090909\n",
            "\n",
            "F1 baseline pada test data: 0.935064935064935\n",
            "\n",
            "Average F1 0.8519130714486751\n",
            "\n",
            "200 0.001 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8433734939759037\n",
            "\n",
            "F1 baseline pada test data: 0.7894736842105263\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8604651162790697\n",
            "\n",
            "F1 baseline pada test data: 0.8974358974358975\n",
            "\n",
            "Average F1 0.844816305046946\n",
            "\n",
            "200 0.001 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8470588235294118\n",
            "\n",
            "F1 baseline pada test data: 0.7733333333333334\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8705882352941177\n",
            "\n",
            "F1 baseline pada test data: 0.9090909090909091\n",
            "\n",
            "Average F1 0.846680926916221\n",
            "\n",
            "200 0.01 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8809523809523809\n",
            "\n",
            "F1 baseline pada test data: 0.8148148148148148\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8505747126436782\n",
            "\n",
            "F1 baseline pada test data: 0.9230769230769231\n",
            "\n",
            "Average F1 0.8605504329642262\n",
            "\n",
            "200 0.01 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8\n",
            "\n",
            "F1 baseline pada test data: 0.8470588235294116\n",
            "\n",
            "F1 baseline pada test data: 0.8505747126436782\n",
            "\n",
            "F1 baseline pada test data: 0.8974358974358975\n",
            "\n",
            "Average F1 0.8456805533884643\n",
            "\n",
            "200 0.005 adam 0.8 constant\n",
            "F1 baseline pada test data: 0.8674698795180724\n",
            "\n",
            "F1 baseline pada test data: 0.810126582278481\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8604651162790697\n",
            "\n",
            "F1 baseline pada test data: 0.9113924050632912\n",
            "\n",
            "Average F1 0.8565574632944495\n",
            "\n",
            "200 0.005 adam 0.9 constant\n",
            "F1 baseline pada test data: 0.8470588235294118\n",
            "\n",
            "F1 baseline pada test data: 0.7792207792207793\n",
            "\n",
            "F1 baseline pada test data: 0.8333333333333333\n",
            "\n",
            "F1 baseline pada test data: 0.8604651162790697\n",
            "\n",
            "F1 baseline pada test data: 0.9210526315789473\n",
            "\n",
            "Average F1 0.8482261367883084\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lDkZNxm00-m6",
        "colab_type": "code",
        "outputId": "9d5157ad-383f-4c13-e651-68e9333e6aec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 551
        }
      },
      "source": [
        "# STACKING LV 0 DATA A\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "avg_f1 = 0\n",
        "\n",
        "# ACTIVATE FOR EXPLORING\n",
        "for p1 in [(250,)]: #  hiddens\n",
        "    for p2 in [0.0001]: # learning_rate\n",
        "        for p3 in [800]: # max_iter\n",
        "# for p1 in [75]: # n_estimator, hiddens\n",
        "#     for p2 in [ 0.001]: # learning_rate\n",
        "#         for p3 in [0.75]: # subsample, max_iter\n",
        "            avg_exp_f1 = 0\n",
        "            print()\n",
        "            print(p1, p2, p3)\n",
        "\n",
        "            np.random.seed(113)\n",
        "            sampler = SMOTE(sampling_strategy=0.5, random_state=113)\n",
        "\n",
        "            X = data_A[:,1].copy()\n",
        "            Y = np.load(\"A_Y.npy\")\n",
        "            skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=False)\n",
        "\n",
        "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
        "                print(\"Fold %d/5\" % (kf+1))\n",
        "\n",
        "                X_train = vectorize(data_A[id_train,1].copy(), pre=True)\n",
        "                Y_train = np.load(\"A_Y.npy\")[id_train]\n",
        "\n",
        "                X_test = vectorize(data_A[id_test,1].copy(), pre=True)\n",
        "                Y_test = np.load(\"A_Y.npy\")[id_test]\n",
        "\n",
        "                X_test = np.c_[X_test, find_len(data_A[id_test,1], tipe='A')]\n",
        "                X_train = np.c_[X_train, find_len(data_A[id_train,1], tipe='A')]\n",
        "\n",
        "                X_train, Y_train = sampler.fit_resample(X_train, Y_train)\n",
        "\n",
        "                # XGB\n",
        "                # clf_b_1 = XGBClassifier(random_state=113, n_estimators=p1, learning_rate=p2, subsample=p3)\n",
        "                # clf_b_1.fit(X_train, Y_train)\n",
        "\n",
        "                # # MLP \n",
        "                clf_b_2 = MLPClassifier(random_state=113, max_iter=p3, hidden_layer_sizes=p1, learning_rate_init=p2, solver='adam')\n",
        "                clf_b_2.fit(X_train, Y_train)\n",
        "\n",
        "                # ev1 = clf_b_1.predict(X_test)\n",
        "                ev2 = clf_b_2.predict(X_test)\n",
        "\n",
        "                avg_exp_f1 += f1_score(ev2, Y_test) # <<<< change this\n",
        "\n",
        "                # print(\"M1 Accuracy pada test data:\", accuracy_score(ev1, Y_test))\n",
        "                # print(\"   F1 pada test data:\", f1_score(ev1, Y_test))\n",
        "                print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
        "                print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
        "\n",
        "            print(\"AVERAGE F1\", avg_exp_f1/5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "(250,) 0.0001 800\n",
            "Fold 1/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "M2 Accuracy pada test data: 0.8727272727272727\n",
            "   F1 pada test data: 0.9135802469135802\n",
            "Fold 2/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "M2 Accuracy pada test data: 0.7592592592592593\n",
            "   F1 pada test data: 0.8354430379746836\n",
            "Fold 3/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "M2 Accuracy pada test data: 0.7358490566037735\n",
            "   F1 pada test data: 0.8292682926829269\n",
            "Fold 4/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "M2 Accuracy pada test data: 0.8867924528301887\n",
            "   F1 pada test data: 0.9268292682926829\n",
            "Fold 5/5\n",
            "M2 Accuracy pada test data: 0.9245283018867925\n",
            "   F1 pada test data: 0.9444444444444444\n",
            "AVERAGE F1 0.8899130580616637\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGkhEaSn05Mp",
        "colab_type": "code",
        "outputId": "0348b86b-0a87-4258-b1a6-dbf2c0598916",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# STACKING LV 1 DATA A\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "for p1 in [(45,)]: # n_estimator, hiddens 48, 0.00005 800\n",
        "    for p2 in [0.00003]: # learning_rate\n",
        "        for p3 in [900]: # subsample, max_iter\n",
        "\n",
        "            avg_f1 = 0\n",
        "            avg_m1 = 0\n",
        "            avg_m2 = 0\n",
        "            np.random.seed(113)\n",
        "            sampler = SMOTE(sampling_strategy=0.5, random_state=113)\n",
        "\n",
        "            print()\n",
        "            print(p1, p2, p3)\n",
        "\n",
        "            X = data_A[:,1].copy()\n",
        "            Y = np.load(\"A_Y.npy\")\n",
        "            skf = StratifiedKFold(n_splits=5, random_state=113, shuffle=False)\n",
        "\n",
        "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
        "                print(\"Fold %d/5\" % (kf+1))\n",
        "\n",
        "                X_all = vectorize(data_A[:,1].copy(), pre=True)\n",
        "                Y = np.load(\"A_Y.npy\")\n",
        "\n",
        "                X_train = vectorize(data_A[id_train,1].copy(), pre=True)\n",
        "                Y_train = np.load(\"A_Y.npy\")[id_train]\n",
        "\n",
        "                X_test = vectorize(data_A[id_test,1].copy(), pre=True)\n",
        "                Y_test = np.load(\"A_Y.npy\")[id_test]\n",
        "\n",
        "                X_test = np.c_[X_test, find_len(data_A[id_test,1], tipe='A')]\n",
        "                X_train = np.c_[X_train, find_len(data_A[id_train,1], tipe='A')]\n",
        "                X_all = np.c_[X_all, find_len(data_A[:,1], tipe='A')]\n",
        "\n",
        "                X_train, Y_train = sampler.fit_resample(X_train, Y_train)\n",
        "\n",
        "                # # BASELINE\n",
        "                clf_a_1 = XGBClassifier(random_state=113, n_estimators=75, learning_rate=0.001, subsample=0.75)\n",
        "                clf_a_1.fit(X_train, Y_train)\n",
        "\n",
        "                # MLP / 14\n",
        "                clf_a_2 = MLPClassifier(random_state=113, max_iter=800, hidden_layer_sizes=(250,), learning_rate_init=0.0001, solver='adam')\n",
        "                clf_a_2.fit(X_train, Y_train)\n",
        "\n",
        "                A1 = np.log(clf_a_1.predict_proba(X_train))\n",
        "                A2 = np.log(clf_a_2.predict_proba(X_train))\n",
        "                out_A = np.c_[A1, A2]\n",
        "\n",
        "                # r1 = 0.05\n",
        "                # r2 = 0.35\n",
        "                # r3 = 0.25\n",
        "                # r4 = 0.35\n",
        "                # tot_B = B1*r1 + B2*r2 + B3*r3 + B4*r4\n",
        "\n",
        "                A1t = np.log(clf_a_1.predict_proba(X_test))\n",
        "                A2t = np.log(clf_a_2.predict_proba(X_test))\n",
        "                out_At = np.c_[A1t, A2t]\n",
        "                # tot_Bt = B1t*r1 + B2t*r2 + B3t*r3 + B4t*r4\n",
        "\n",
        "                ev1 = clf_a_1.predict(X_test)\n",
        "                ev2 = clf_a_2.predict(X_test)\n",
        "\n",
        "                print(\"==========================\")\n",
        "                print(\"M1 Accuracy pada test data:\", accuracy_score(ev1, Y_test))\n",
        "                print(\"   F1 pada test data:\", f1_score(ev1, Y_test))\n",
        "                print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
        "                print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
        "                print(\"==========================\")\n",
        "\n",
        "                meta_clf = MLPClassifier(\n",
        "                        random_state=113,\n",
        "                        max_iter=p3,\n",
        "                        hidden_layer_sizes=p1,\n",
        "                        learning_rate_init=p2,\n",
        "                        learning_rate = 'adaptive',\n",
        "                        solver='adam')\n",
        "\n",
        "                meta_clf.fit(out_A, Y_train)\n",
        "                prediction = meta_clf.predict(out_At).astype(int)\n",
        "                # prediction = np.argmax(tot_Bt, 1)\n",
        "\n",
        "                print(\"Accuracy pada test data:\", accuracy_score(prediction, Y_test))\n",
        "                print(\"F1 pada test data:\", f1_score(prediction, Y_test))\n",
        "                \n",
        "                avg_f1 += f1_score(prediction, Y_test)\n",
        "                avg_m1 += f1_score(ev1, Y_test)\n",
        "                avg_m2 += f1_score(ev2, Y_test)\n",
        "                            \n",
        "            print(\"\\nAverage F1\", avg_f1/5)\n",
        "            print(\"Average Lv0 F1-1\", avg_m1/5)\n",
        "            print(\"Average Lv0 F1-2\", avg_m2/5)\n",
        "\n",
        "            clf_a_1 = XGBClassifier(random_state=113, n_estimators=75, learning_rate=0.001, subsample=0.75)\n",
        "            clf_a_2 = MLPClassifier(random_state=113, max_iter=800, hidden_layer_sizes=(250,), learning_rate_init=0.0001, solver='adam')\n",
        "            clf_a_1.fit(X_all, Y)\n",
        "            clf_a_2.fit(X_all, Y)\n",
        "            A1 = np.log(clf_a_1.predict_proba(X_all))\n",
        "            A2 = np.log(clf_a_2.predict_proba(X_all))\n",
        "            out_A_all = np.c_[A1, A2]\n",
        "            meta_clf = MLPClassifier(\n",
        "                        random_state=113,\n",
        "                        max_iter=p3,\n",
        "                        hidden_layer_sizes=p1,\n",
        "                        learning_rate_init=p2,\n",
        "                        solver='adam')\n",
        "            meta_clf.fit(out_A_all, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "(45,) 3e-05 900\n",
            "Fold 1/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M1 Accuracy pada test data: 0.8363636363636363\n",
            "   F1 pada test data: 0.888888888888889\n",
            "M2 Accuracy pada test data: 0.8727272727272727\n",
            "   F1 pada test data: 0.9135802469135802\n",
            "==========================\n",
            "Accuracy pada test data: 0.8727272727272727\n",
            "F1 pada test data: 0.9135802469135802\n",
            "Fold 2/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M1 Accuracy pada test data: 0.7592592592592593\n",
            "   F1 pada test data: 0.8470588235294116\n",
            "M2 Accuracy pada test data: 0.7592592592592593\n",
            "   F1 pada test data: 0.8354430379746836\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (900) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.7592592592592593\n",
            "F1 pada test data: 0.8354430379746836\n",
            "Fold 3/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M1 Accuracy pada test data: 0.7547169811320755\n",
            "   F1 pada test data: 0.8470588235294116\n",
            "M2 Accuracy pada test data: 0.7358490566037735\n",
            "   F1 pada test data: 0.8292682926829269\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (900) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.7547169811320755\n",
            "F1 pada test data: 0.8433734939759038\n",
            "Fold 4/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M1 Accuracy pada test data: 0.8301886792452831\n",
            "   F1 pada test data: 0.891566265060241\n",
            "M2 Accuracy pada test data: 0.8867924528301887\n",
            "   F1 pada test data: 0.9268292682926829\n",
            "==========================\n",
            "Accuracy pada test data: 0.8679245283018868\n",
            "F1 pada test data: 0.9156626506024096\n",
            "Fold 5/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M1 Accuracy pada test data: 0.8301886792452831\n",
            "   F1 pada test data: 0.8831168831168831\n",
            "M2 Accuracy pada test data: 0.9245283018867925\n",
            "   F1 pada test data: 0.9444444444444444\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (900) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.9433962264150944\n",
            "F1 pada test data: 0.958904109589041\n",
            "\n",
            "Average F1 0.8933927078111237\n",
            "Average Lv0 F1-1 0.8715379368249673\n",
            "Average Lv0 F1-2 0.8899130580616637\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (800) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r-UDmpdOa7iC",
        "colab_type": "text"
      },
      "source": [
        "### DATA B"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vf7eViPm5Ik6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# avg_f1 = 0\n",
        "# X = vectorize2(data_B[:,1].copy(), pre=True)\n",
        "# # XE = np.load(\"B_X.npy\")\n",
        "# # X = vectorize(data_B[:,1].copy())\n",
        "# Y = np.load(\"B_Y.npy\")[:]\n",
        "\n",
        "# # X = np.c_[X, XE]\n",
        "# X = np.c_[X, find_len(data_B[:,1], tipe='B')]\n",
        "\n",
        "# skf = StratifiedKFold(n_splits=5, random_state=11, shuffle=True)\n",
        "\n",
        "# # hiddens = [(100,), (200,), (300,), (400,), (80,80),(120,120),(120,80)]\n",
        "# hiddens = [(200,100), (128,64), (300, 100)]\n",
        "# lris = [0.001, 0.0001, 0.005, 0.0005]\n",
        "# # solvers = ['adam', 'sgd']\n",
        "# max_iters = [400, 500, 600]\n",
        "# # lrs = ['adaptive', 'constant', 'invscaling']\n",
        "\n",
        "# hiddens = [(128,64)]\n",
        "# lris = [0.0001]\n",
        "# solvers = ['adam']\n",
        "# max_iters = [500]\n",
        "# lrs = ['adaptive']\n",
        "\n",
        "# # ssmpls = [0.7,  0.8]\n",
        "# # nestrs = [50, 75, 100, 125, 150, 175]\n",
        "# # lris = [0.001, 0.01, 0.002, 0.008, 0.005]\n",
        "# # mds = [3]\n",
        "# # cbt = [0.8, 0.9, 1]\n",
        "\n",
        "# # ssmpls = [0.8]\n",
        "# # nestrs = [200]\n",
        "# # mds = [3]\n",
        "# # lris = [0.01]\n",
        "# # cbt = [1]\n",
        "\n",
        "# # for cb in cbt:\n",
        "# #     for lri in lris:\n",
        "# #         for ss in ssmpls:\n",
        "# #             for ne in nestrs:\n",
        "# #                 for md in mds:\n",
        "# for h in hiddens:\n",
        "#     for lri in lris:\n",
        "#         for solver in solvers:\n",
        "#             for mi in max_iters:\n",
        "#                 for lr in lrs:\n",
        "# # for h in range(1):\n",
        "# #     for lri in range(1):\n",
        "# #         for c in [0.001, 0.01, 0.1, 0.3, 0.5, 0.7, 1.0]:\n",
        "# #             for gm in [0.01, 0.05, 0.1, 0.5, 1.0]:\n",
        "# #                 for lr in range(1):\n",
        "#                     print()\n",
        "#                     print(h, lri, solver, mi, lr)\n",
        "# #                     print(lri, ss, ne, md, cb)\n",
        "# #                     print(c, gm)\n",
        "                    \n",
        "#                     avg_f1 = 0\n",
        "            \n",
        "#                     # BASELINE\n",
        "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=200, max_depth=3, learning_rate=0.01, subsample=0.8)\n",
        "\n",
        "#                     # ATTEMPT 3\n",
        "# #                     clf2 = AdaBoostClassifier(random_state=113, n_estimators=50, learning_rate=0.05)\n",
        "    \n",
        "#                     # ATTEMPT 5\n",
        "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=150, gamma=5, max_depth=3, learning_rate=0.01, subsample=0.8)\n",
        "                    \n",
        "    \n",
        "#                     # ATTEMPT 12\n",
        "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=150, max_depth=3, learning_rate=0.0017, subsample=0.7, colsample_bytree=0.8)\n",
        "            \n",
        "#                     # Final attempt\n",
        "#                     clf2 = MLPClassifier(\n",
        "#                             random_state=113,\n",
        "#                             max_iter=mi,\n",
        "#                             hidden_layer_sizes=h,\n",
        "#                             learning_rate_init=lri,\n",
        "#                             learning_rate = lr,\n",
        "#                             solver=solver)\n",
        "    \n",
        "# #                     clf2 = SVC(C=c, gamma=gm, probability=True, class_weight=None, random_state=113)\n",
        "                    \n",
        "#                     done = True\n",
        "#                     for i, (train_index, test_index) in enumerate(skf.split(X, Y)):\n",
        "#                         done = False\n",
        "#                         x_train, y_train = X[train_index], Y[train_index]\n",
        "# #                         x_train, y_train = sampler.fit_resample(x_train, y_train)\n",
        "\n",
        "#                         x_test, y_test = X[test_index], Y[test_index]\n",
        "\n",
        "\n",
        "#                         clf2.fit(x_train, y_train)\n",
        "\n",
        "#                         prediction = clf2.predict(x_test).astype(int)\n",
        "\n",
        "#                         from sklearn.metrics import accuracy_score\n",
        "#                         from sklearn.metrics import precision_score\n",
        "#                         from sklearn.metrics import recall_score\n",
        "#                         from sklearn.metrics import f1_score\n",
        "\n",
        "# #                         print(\"Accuracy pada train data:\", accuracy_score(clf2.predict(x_train).astype(int), y_train))\n",
        "# #                         print(\"Accuracy pada test data:\", accuracy_score(prediction, y_test))\n",
        "# #                         print(\"Precision baseline pada test data:\", precision_score(prediction, y_test))\n",
        "# #                         print(\"Recall baseline pada test data:\", recall_score(prediction, y_test))\n",
        "#                         print(\"F1 baseline pada test data:\", f1_score(prediction, y_test))\n",
        "# #                         print()\n",
        "#                         avg_f1 += f1_score(prediction, y_test)\n",
        "\n",
        "# #                         if f1_score(prediction, y_test)<0.64 and i==2:\n",
        "# #                             break\n",
        "# #                         if f1_score(prediction, y_test)<0.84 and i==3:\n",
        "# #                             break\n",
        "#                         done = True\n",
        "                    \n",
        "#                     if done:\n",
        "#                         print(\"Average F1\", avg_f1/5)\n",
        "\n",
        "#                     # FINAL\n",
        "#                     clf2 = MLPClassifier(\n",
        "#                             random_state=113,\n",
        "#                             max_iter=mi,\n",
        "#                             hidden_layer_sizes=h,\n",
        "#                             learning_rate_init=lri,\n",
        "#                             learning_rate = lr,\n",
        "#                             solver=solver)\n",
        "\n",
        "#                     # BASELINE\n",
        "# #                     clf2 = XGBClassifier(random_state=113, n_estimators=200, max_depth=3, learning_rate=0.01, subsample=0.8)\n",
        "                    \n",
        "#                     # 12\n",
        "#                     # clf2 = XGBClassifier(random_state=113, n_estimators=150, max_depth=3, learning_rate=0.0017, subsample=0.7, colsample_bytree=0.8)\n",
        "                    \n",
        "#                     clf2.fit(X, Y)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "93etZfyTcdLP",
        "colab_type": "code",
        "outputId": "6abf915b-64b1-4ab7-989a-709a8c124065",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 348
        }
      },
      "source": [
        "# STACKING LV 0\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "avg_f1 = 0\n",
        "\n",
        "# ACTIVATE FOR EXPLORING\n",
        "for p1 in [70]: # n_estimator, hiddens\n",
        "    for p2 in [0.02]: # learning_rate\n",
        "        for p3 in [0.9]: # subsample, max_iter\n",
        "# for p1 in [(128, 64), (500,), (600,)]: # n_estimator, hiddens\n",
        "#     for p2 in [0.0001, 0.001]: # learning_rate\n",
        "#         for p3 in [500, 300, 800]: # subsample, max_iter\n",
        "            avg_exp_f1 = 0\n",
        "            print()\n",
        "            print(p1, p2, p3)\n",
        "\n",
        "            np.random.seed(113)\n",
        "\n",
        "            X = data_B[:,1].copy()\n",
        "            XE = np.load(\"B_X.npy\")\n",
        "            Y = np.load(\"B_Y.npy\")\n",
        "#             X = np.c_[X, XE]\n",
        "            skf = StratifiedKFold(n_splits=5, random_state=11, shuffle=True)\n",
        "\n",
        "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
        "                print(\"Fold %d/5\" % (kf+1))\n",
        "\n",
        "                X_train = vectorize2(data_B[id_train,1].copy(), pre=True, tipe=\"B\")\n",
        "#                 X_train = XE[id_train]\n",
        "                Y_train = np.load(\"B_Y.npy\")[id_train]\n",
        "\n",
        "                X_test = vectorize2(data_B[id_test,1].copy(), pre=True, tipe=\"B\")\n",
        "#                 X_test = XE[id_test]\n",
        "                Y_test = np.load(\"B_Y.npy\")[id_test]\n",
        "\n",
        "                X_test = np.c_[X_test, XE[id_test], find_len(data_B[id_test,1], tipe='B')]\n",
        "                X_train = np.c_[X_train, XE[id_train],find_len(data_B[id_train,1], tipe='B')]\n",
        "\n",
        "                # # XGB\n",
        "                clf_b_1 = XGBClassifier(random_state=113, n_estimators=p1,\n",
        "                                        learning_rate=p2, subsample=p3,\n",
        "                                        gamma=0, colsample_bytree=0.5)\n",
        "                clf_b_1.fit(X_train, Y_train)\n",
        "\n",
        "#                 # # MLP\n",
        "#                 clf_b_2 = MLPClassifier(random_state=113, max_iter=p3, hidden_layer_sizes=p1, learning_rate_init=p2, solver='adam')\n",
        "#                 clf_b_2.fit(X_train, Y_train)\n",
        "\n",
        "                ev1 = clf_b_1.predict(X_test)\n",
        "#                 ev2 = clf_b_2.predict(X_test)\n",
        "\n",
        "                avg_exp_f1 += f1_score(ev1, Y_test) # <<<< change this\n",
        "\n",
        "                print(\"M1 Accuracy pada test data:\", accuracy_score(ev1, Y_test))\n",
        "                print(\"   F1 pada test data:\", f1_score(ev1, Y_test))\n",
        "#                 print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
        "#                 print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
        "\n",
        "            print(\"AVERAGE F1\", avg_exp_f1/5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "70 0.02 0.9\n",
            "Fold 1/5\n",
            "M1 Accuracy pada test data: 0.6290322580645161\n",
            "   F1 pada test data: 0.7012987012987012\n",
            "Fold 2/5\n",
            "M1 Accuracy pada test data: 0.5806451612903226\n",
            "   F1 pada test data: 0.675\n",
            "Fold 3/5\n",
            "M1 Accuracy pada test data: 0.7213114754098361\n",
            "   F1 pada test data: 0.767123287671233\n",
            "Fold 4/5\n",
            "M1 Accuracy pada test data: 0.7\n",
            "   F1 pada test data: 0.7428571428571429\n",
            "Fold 5/5\n",
            "M1 Accuracy pada test data: 0.7833333333333333\n",
            "   F1 pada test data: 0.7936507936507938\n",
            "AVERAGE F1 0.7359859850955741\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mGif9g-q3O0Q",
        "colab_type": "code",
        "outputId": "4586367e-6059-404f-816e-b44cfc2fe046",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# STACKING LV 1 FILM B\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "for p1 in [(32,)]: # n_estimator, hiddens\n",
        "    for p2 in [0.00003]: # learning_rate\n",
        "        for p3 in [1000]: # subsample, max_iter\n",
        "\n",
        "            avg_f1 = 0\n",
        "            avg_m1 = 0\n",
        "            avg_m2 = 0\n",
        "            avg_m3 = 0\n",
        "            avg_m4 = 0\n",
        "            np.random.seed(113)\n",
        "            print()\n",
        "            print(p1, p2, p3)\n",
        "\n",
        "            X = data_B[:,1].copy()\n",
        "            Y = np.load(\"B_Y.npy\")\n",
        "            skf = StratifiedKFold(n_splits=5, random_state=11, shuffle=True)\n",
        "\n",
        "            for kf, (id_train, id_test) in enumerate(skf.split(X, Y)):\n",
        "                print(\"Fold %d/5\" % (kf+1))\n",
        "\n",
        "                # id_test = np.random.choice(len(data_B), 39)\n",
        "                # id_train = diff(range(len(data_B)), id_test)\n",
        "\n",
        "                X_all = vectorize2(data_B[:,1].copy(), pre=True)\n",
        "                Y = np.load(\"B_Y.npy\")\n",
        "\n",
        "                X_train = vectorize2(data_B[id_train,1].copy(), pre=True)\n",
        "                Y_train = np.load(\"B_Y.npy\")[id_train]\n",
        "\n",
        "                X_test = vectorize2(data_B[id_test,1].copy(), pre=True)\n",
        "                Y_test = np.load(\"B_Y.npy\")[id_test]\n",
        "\n",
        "                X_test = np.c_[X_test, find_len(data_B[id_test,1], tipe='B')]\n",
        "                X_train = np.c_[X_train, find_len(data_B[id_train,1], tipe='B')]\n",
        "                X_all = np.c_[X_all, find_len(data_B[:,1], tipe='B')]\n",
        "\n",
        "                # # BASELINE\n",
        "                # 12\n",
        "                clf_b_2 = XGBClassifier(random_state=113, n_estimators=100, learning_rate=0.01, subsample=0.8, colsample_bytree=0.8)\n",
        "                clf_b_2.fit(X_train, Y_train)\n",
        "\n",
        "                # MLP / 14\n",
        "                clf_b_3 = MLPClassifier(random_state=113, max_iter=500, hidden_layer_sizes=(128,64), learning_rate_init=0.0001, solver='adam')\n",
        "                clf_b_3.fit(X_train, Y_train)\n",
        "\n",
        "                B2 = np.log(clf_b_2.predict_proba(X_train))\n",
        "                B3 = np.log(clf_b_3.predict_proba(X_train))\n",
        "                out_B = np.c_[B2, B3]\n",
        "\n",
        "                B2t = np.log(clf_b_2.predict_proba(X_test))\n",
        "                B3t = np.log(clf_b_3.predict_proba(X_test))\n",
        "                out_Bt = np.c_[B2t, B3t]\n",
        "\n",
        "                ev2 = clf_b_2.predict(X_test)\n",
        "                ev3 = clf_b_3.predict(X_test)\n",
        "\n",
        "                print(\"==========================\")\n",
        "                print(\"M2 Accuracy pada test data:\", accuracy_score(ev2, Y_test))\n",
        "                print(\"   F1 pada test data:\", f1_score(ev2, Y_test))\n",
        "                print(\"M3 Accuracy pada test data:\", accuracy_score(ev3, Y_test))\n",
        "                print(\"   F1 pada test data:\", f1_score(ev3, Y_test))\n",
        "                print(\"==========================\")\n",
        "\n",
        "                meta_clf2 = MLPClassifier(\n",
        "                        random_state=113,\n",
        "                        max_iter=p3,\n",
        "                        hidden_layer_sizes=p1,\n",
        "                        learning_rate_init=p2,\n",
        "                        learning_rate = 'adaptive',\n",
        "                        solver='adam')\n",
        "\n",
        "                meta_clf2.fit(out_B, Y_train)\n",
        "\n",
        "                prediction = meta_clf2.predict(out_Bt).astype(int)\n",
        "\n",
        "                print(\"Accuracy pada test data:\", accuracy_score(prediction, Y_test))\n",
        "                print(\"F1 pada test data:\", f1_score(prediction, Y_test))\n",
        "                \n",
        "                avg_f1 += f1_score(prediction, Y_test)\n",
        "                avg_m2 += f1_score(ev2, Y_test)\n",
        "                avg_m3 += f1_score(ev3, Y_test)\n",
        "\n",
        "                            \n",
        "            print(\"\\nAverage F1\", avg_f1/5)\n",
        "            print(\"Average Lv0 F1-2\", avg_m2/5)\n",
        "            print(\"Average Lv0 F1-3\", avg_m3/5)\n",
        "\n",
        "            clf_b_2 = XGBClassifier(random_state=113, n_estimators=100, learning_rate=0.01, subsample=0.8, colsample_bytree=0.8)\n",
        "            clf_b_3 = MLPClassifier(random_state=113, max_iter=500, hidden_layer_sizes=(128,64), learning_rate_init=0.0001, solver='adam')\n",
        "            clf_b_2.fit(X_all, Y)\n",
        "            clf_b_3.fit(X_all, Y)\n",
        "            B2 = np.log(clf_b_2.predict_proba(X_all))\n",
        "            B3 = np.log(clf_b_3.predict_proba(X_all))\n",
        "            out_B_all = np.c_[B2, B3]\n",
        "            meta_clf2 = MLPClassifier(\n",
        "                        random_state=113,\n",
        "                        max_iter=p3,\n",
        "                        hidden_layer_sizes=p1,\n",
        "                        learning_rate_init=p2,\n",
        "                        solver='adam')\n",
        "            meta_clf2.fit(out_B_all, Y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "(32,) 3e-05 1000\n",
            "Fold 1/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M2 Accuracy pada test data: 0.6290322580645161\n",
            "   F1 pada test data: 0.6666666666666666\n",
            "M3 Accuracy pada test data: 0.6612903225806451\n",
            "   F1 pada test data: 0.6956521739130436\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.6612903225806451\n",
            "F1 pada test data: 0.7042253521126761\n",
            "Fold 2/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M2 Accuracy pada test data: 0.5967741935483871\n",
            "   F1 pada test data: 0.6987951807228915\n",
            "M3 Accuracy pada test data: 0.6290322580645161\n",
            "   F1 pada test data: 0.6933333333333332\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.6451612903225806\n",
            "F1 pada test data: 0.725\n",
            "Fold 3/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M2 Accuracy pada test data: 0.7868852459016393\n",
            "   F1 pada test data: 0.8219178082191781\n",
            "M3 Accuracy pada test data: 0.7868852459016393\n",
            "   F1 pada test data: 0.8059701492537314\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.7540983606557377\n",
            "F1 pada test data: 0.782608695652174\n",
            "Fold 4/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M2 Accuracy pada test data: 0.7833333333333333\n",
            "   F1 pada test data: 0.8169014084507042\n",
            "M3 Accuracy pada test data: 0.7666666666666667\n",
            "   F1 pada test data: 0.8108108108108109\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.7666666666666667\n",
            "F1 pada test data: 0.8157894736842105\n",
            "Fold 5/5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "==========================\n",
            "M2 Accuracy pada test data: 0.6833333333333333\n",
            "   F1 pada test data: 0.7164179104477613\n",
            "M3 Accuracy pada test data: 0.7666666666666667\n",
            "   F1 pada test data: 0.7741935483870968\n",
            "==========================\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.75\n",
            "F1 pada test data: 0.7761194029850745\n",
            "\n",
            "Average F1 0.7607485848868271\n",
            "Average Lv0 F1-2 0.7441397949014403\n",
            "Average Lv0 F1-3 0.7559920031396032\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (1000) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mzNIrkhuPWun",
        "colab_type": "text"
      },
      "source": [
        "# Classifier LSTM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "351bImnsPVo1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "\n",
        "def seed():\n",
        "    torch.manual_seed(113)\n",
        "    torch.cuda.manual_seed(113)\n",
        "    torch.cuda.manual_seed_all(113)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "\n",
        "    np.random.seed(113)\n",
        "\n",
        "    import random\n",
        "    random.seed(113)\n",
        "\n",
        "def _init_fn(worker_id):\n",
        "    np.random.seed(int(seed))\n",
        "    \n",
        "seed()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JECKBpP5SRuB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len_seq = 14\n",
        "def batcher(sent):\n",
        "    ret = []\n",
        "    counter = 0\n",
        "    null = np.zeros(150)\n",
        "    for word in sent.split():\n",
        "        if word in [\"./,;!?()\"]:\n",
        "            continue\n",
        "        word = word.lower()\n",
        "        ret.append(model[word])\n",
        "    while(len(ret) < len_seq):\n",
        "        ret.insert(0, null)\n",
        "    while(len(ret) > len_seq):\n",
        "        ret.pop(-1)  \n",
        "    return np.asarray(ret)\n",
        "\n",
        "def batcher2(sent):\n",
        "    ret = []\n",
        "    counter = 0\n",
        "    null = np.zeros(300)\n",
        "    for word in sent.split():\n",
        "        if word in [\"./,;!?()\"]:\n",
        "            continue\n",
        "        ret.append(model2[word])\n",
        "    while(len(ret) < len_seq):\n",
        "        ret.insert(0, null)\n",
        "    while(len(ret) > len_seq):\n",
        "        ret.pop(-1)  \n",
        "    return np.asarray(ret)\n",
        "\n",
        "def data_batcher(x):\n",
        "    res = []\n",
        "    for i in x:\n",
        "        res.append(batcher(i[1]))\n",
        "    return np.asarray(res) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "15DIxkgUTqCz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# xwer_A = data_batcher(xtrain_A)\n",
        "# xwer_B = data_batcher(xtrain_B)\n",
        "# xwet_A = data_batcher(xtest_A)\n",
        "# xwet_B = data_batcher(xtest_B)\n",
        "\n",
        "# yr_A = np.asarray(ytrain_A).astype(int)\n",
        "# yt_A = np.asarray(ytest_A).astype(int)\n",
        "# yr_B = np.asarray(ytrain_B).astype(int)\n",
        "# yt_B = np.asarray(ytest_A).astype(int)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "57FC6Sba88mI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class LSTMModel(nn.Module):   \n",
        "\n",
        "    def __init__(self, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):\n",
        "        super(LSTMModel, self).__init__()\n",
        "\n",
        "        self.output_size = output_size\n",
        "        self.n_layers = n_layers\n",
        "        self.hidden_dim = hidden_dim\n",
        "        \n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, \n",
        "                            dropout=drop_prob, batch_first=True)\n",
        "        \n",
        "        self.dropout = nn.Dropout(0.5)\n",
        "        \n",
        "        self.fc = nn.Linear(hidden_dim, output_size)\n",
        "\n",
        "    def forward(self, x, hidden):\n",
        "        \n",
        "        batch_size = x.size(0)\n",
        "        seq_len = x.size(1)\n",
        "\n",
        "        lstm_out, hidden = self.lstm(x, hidden)\n",
        "#         print(lstm_out.shape)\n",
        "        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)\n",
        "#         print(lstm_out.shape)\n",
        "        out = self.dropout(lstm_out)\n",
        "        out = self.fc(out)\n",
        "        out = F.relu(out)\n",
        "        \n",
        "        out = out.view(batch_size, seq_len, -1)\n",
        "#         print(out.shape)\n",
        "        out = out[:, -1]\n",
        "#         print(out.shape)\n",
        "        return F.log_softmax(out), hidden\n",
        "    \n",
        "    \n",
        "    def init_hidden(self, batch_size):\n",
        "        weight = next(self.parameters()).data\n",
        "        \n",
        "        hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),\n",
        "                  weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())\n",
        "        \n",
        "        return hidden"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W8Hudww27Trd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# def reset_data():\n",
        "# dataloaders\n",
        "batch_size = 42\n",
        "\n",
        "# make sure the SHUFFLE your training data\n",
        "train_loader = []\n",
        "valid_loader = []\n",
        "test_size = [0.2, 0.15, 0.1, 0.2, 0.15]\n",
        "rand_i = [113, 1117, 11, 3, 317]\n",
        "\n",
        "for i in range(5):\n",
        "    xtrain_A, xtest_A, ytrain_A, ytest_A = train_test_split(data_A[:,:-1], data_A[:,-1],\n",
        "                                      test_size=test_size[i], \n",
        "                                      stratify=data_A[:,-1], \n",
        "                                      random_state=rand_i[i])\n",
        "\n",
        "    xtrain_B, xtest_B, ytrain_B, ytest_B = train_test_split(data_B[:,:-1], data_B[:,-1],\n",
        "                                          test_size=test_size[i], \n",
        "                                          stratify=data_B[:,-1], \n",
        "                                          random_state=rand_i[i])\n",
        "\n",
        "    xwer_A = data_batcher(xtrain_A)\n",
        "    xwer_B = data_batcher(xtrain_B)\n",
        "    xwet_A = data_batcher(xtest_A)\n",
        "    xwet_B = data_batcher(xtest_B)\n",
        "\n",
        "    yr_A = np.asarray(ytrain_A).astype(int)\n",
        "    yt_A = np.asarray(ytest_A).astype(int)\n",
        "    yr_B = np.asarray(ytrain_B).astype(int)\n",
        "    yt_B = np.asarray(ytest_A).astype(int)\n",
        "\n",
        "    train_data = TensorDataset(torch.from_numpy(xwer_A), torch.from_numpy(yr_A))\n",
        "    valid_data = TensorDataset(torch.from_numpy(xwet_A), torch.from_numpy(yt_A))\n",
        "\n",
        "    train_loader.append(DataLoader(train_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))\n",
        "    valid_loader.append(DataLoader(valid_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))\n",
        "        \n",
        "# reset_data()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTJtWuhf-dCS",
        "colab_type": "code",
        "outputId": "8bce2efe-a7c2-42e3-ac96-ff1394ba6e8a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 570
        }
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score\n",
        "\n",
        "seed()\n",
        "\n",
        "output_size = 2\n",
        "embedding_dim = 150\n",
        "hidden_dim = 256\n",
        "n_layers = 1\n",
        "lrs= 0.00007\n",
        "epochs = 45\n",
        "counter = 0\n",
        "print_every = 500\n",
        "clip=5 # gradient clipping\n",
        "\n",
        "# for epoch in epochs:\n",
        "#     for lr in lrs:\n",
        "#         for hidden_dim in hidden_dims:\n",
        "#             for n_layer in n_layers:\n",
        "                \n",
        "#                 print()\n",
        "#                 print(epoch, lr, hidden_dim, n_layer)\n",
        "\n",
        "avg_f1 = 0\n",
        "\n",
        "for i in range(5):\n",
        "\n",
        "    clf = LSTMModel(output_size, embedding_dim, hidden_dim, n_layers)\n",
        "    criterion = nn.NLLLoss()\n",
        "    optimizer = torch.optim.Adam(clf.parameters(), lr=lrs)\n",
        "\n",
        "    clf.cuda()\n",
        "    clf.train()\n",
        "\n",
        "    for e in range(epochs):\n",
        "\n",
        "        for _, (inputs, labels) in enumerate(train_loader[i]):\n",
        "\n",
        "            counter += 1\n",
        "\n",
        "            inputs, labels = inputs.cuda(), labels.cuda()\n",
        "            inputs = inputs.float()\n",
        "\n",
        "            h = clf.init_hidden(inputs.size(0))\n",
        "            h = tuple([each.data for each in h])\n",
        "\n",
        "            clf.zero_grad()\n",
        "            output, h = clf(inputs, h)\n",
        "\n",
        "            loss = criterion(output.squeeze(), labels)\n",
        "            loss.backward()\n",
        "\n",
        "            # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
        "#             nn.utils.clip_grad_norm_(clf.parameters(), clip)\n",
        "            optimizer.step()\n",
        "\n",
        "        clf.eval()\n",
        "        val_losses = []\n",
        "        acc = torch.Tensor([]).cuda()\n",
        "        for inputs, labels in valid_loader[i]:\n",
        "\n",
        "            val_h = clf.init_hidden(inputs.size(0))\n",
        "            val_h = tuple([each.data for each in val_h])\n",
        "\n",
        "            inputs, labels = inputs.cuda(), labels.cuda()\n",
        "            inputs = inputs.float()\n",
        "\n",
        "            output, val_h = clf(inputs, val_h)\n",
        "    #         print(labels)\n",
        "    #         print(torch.argmax(output.squeeze(), dim=1))\n",
        "            pred = torch.argmax(output.squeeze(), dim=1)==labels\n",
        "            acc = torch.cat((acc, pred.float()),0)\n",
        "            val_loss = criterion(output.squeeze(), labels)\n",
        "\n",
        "            val_losses.append(val_loss.item())\n",
        "\n",
        "        clf.train()\n",
        "#                         if e%print_every == 0:\n",
        "#                             print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
        "#                                   \"Loss: {:.6f}...\".format(loss.item()),\n",
        "#                                   \"Val Loss: {:.6f}\".format(np.mean(val_losses)),\n",
        "#                                   \"Val Acc: {:.6f}\".format(torch.mean(acc)))\n",
        "\n",
        "\n",
        "    clf.eval()\n",
        "    preds = torch.Tensor([]).cuda()\n",
        "    labels_all = torch.Tensor([]).cuda()\n",
        "    for inputs, labels in valid_loader[i]:\n",
        "\n",
        "        val_h = clf.init_hidden(inputs.size(0))\n",
        "        val_h = tuple([each.data for each in val_h])\n",
        "\n",
        "        inputs, labels = inputs.cuda(), labels.cuda()\n",
        "        inputs = inputs.float()\n",
        "\n",
        "        output, val_h = clf(inputs, val_h)\n",
        "    #     print(output.shape)\n",
        "        pred = torch.argmax(output.squeeze(), dim=1)\n",
        "        preds = torch.cat((preds, pred.float()),0)\n",
        "        labels_all = torch.cat((labels_all, labels.float()),0)\n",
        "\n",
        "    preds = preds.detach().cpu().numpy()\n",
        "    labels_all = labels_all.cpu().numpy()\n",
        "#                     print(preds)\n",
        "\n",
        "    print(\"Accuracy pada test data:\", accuracy_score(preds, labels_all))\n",
        "    print(\"Accuracy pada test data:\", accuracy_score(preds, labels_all))\n",
        "    print(\"Precision baseline pada test data:\", precision_score(preds, labels_all))\n",
        "    print(\"Recall baseline pada test data:\", recall_score(preds, labels_all))\n",
        "    print(\"F1 baseline pada test data:\", f1_score(preds, labels_all))\n",
        "    print()\n",
        "    avg_f1 += f1_score(preds, labels_all)\n",
        "\n",
        "\n",
        "print(avg_f1/5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:54: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
            "  \"num_layers={}\".format(dropout, num_layers))\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:34: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy pada test data: 0.7777777777777778\n",
            "Precision baseline pada test data: 0.868421052631579\n",
            "Recall baseline pada test data: 0.825\n",
            "F1 baseline pada test data: 0.8461538461538461\n",
            "\n",
            "Accuracy pada test data: 0.8048780487804879\n",
            "Precision baseline pada test data: 0.9655172413793104\n",
            "Recall baseline pada test data: 0.8\n",
            "F1 baseline pada test data: 0.8750000000000001\n",
            "\n",
            "Accuracy pada test data: 0.7777777777777778\n",
            "Precision baseline pada test data: 0.8421052631578947\n",
            "Recall baseline pada test data: 0.8421052631578947\n",
            "F1 baseline pada test data: 0.8421052631578947\n",
            "\n",
            "Accuracy pada test data: 0.7592592592592593\n",
            "Precision baseline pada test data: 0.9473684210526315\n",
            "Recall baseline pada test data: 0.7659574468085106\n",
            "F1 baseline pada test data: 0.8470588235294116\n",
            "\n",
            "Accuracy pada test data: 0.8292682926829268\n",
            "Precision baseline pada test data: 0.9655172413793104\n",
            "Recall baseline pada test data: 0.8235294117647058\n",
            "F1 baseline pada test data: 0.888888888888889\n",
            "\n",
            "0.8598413643460084\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_iCdiAaHgLc4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# dataloaders\n",
        "batch_size = 42\n",
        "\n",
        "# make sure the SHUFFLE your training data\n",
        "train_loader = []\n",
        "valid_loader = []\n",
        "test_size = [0.2, 0.15, 0.1, 0.2, 0.15]\n",
        "rand_i = [113, 1117, 11, 3, 317]\n",
        "\n",
        "for i in range(5):\n",
        "    xtrain_B, xtest_B, ytrain_B, ytest_B = train_test_split(data_B[:,:-1], data_B[:,-1],\n",
        "                                          test_size=test_size[i], \n",
        "                                          stratify=data_B[:,-1], \n",
        "                                          random_state=rand_i[i])\n",
        "    \n",
        "    xwer_B = data_batcher(xtrain_B)\n",
        "    xwet_B = data_batcher(xtest_B)\n",
        "\n",
        "    yr_B = np.asarray(ytrain_B).astype(int)\n",
        "    yt_B = np.asarray(ytest_A).astype(int)\n",
        "    \n",
        "    train_data = TensorDataset(torch.from_numpy(xwer_B), torch.from_numpy(yr_B))\n",
        "    valid_data = TensorDataset(torch.from_numpy(xwet_B), torch.from_numpy(yt_B))\n",
        "    \n",
        "    train_loader.append(DataLoader(train_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))\n",
        "    valid_loader.append(DataLoader(valid_data, shuffle=True, batch_size=batch_size, worker_init_fn=_init_fn))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pgniphDhgFHe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed()\n",
        "\n",
        "output_size = 2\n",
        "embedding_dim = 300\n",
        "hidden_dim = 500\n",
        "n_layers = 1\n",
        "lr=0.00007\n",
        "epochs = 75 # 3-4 is approx where I noticed the validation loss stop decreasing\n",
        "counter = 0\n",
        "print_every = 30\n",
        "clip=5 # gradient clipping\n",
        "\n",
        "avg_f1 = 0\n",
        "\n",
        "for i in range(5):\n",
        "    \n",
        "    clf = LSTMModel(output_size, embedding_dim, hidden_dim, n_layers)\n",
        "    criterion = nn.NLLLoss()\n",
        "    optimizer = torch.optim.Adam(clf.parameters(), lr=lr)\n",
        "    \n",
        "    clf.cuda()\n",
        "    clf.train()\n",
        "    \n",
        "    for e in range(epochs):\n",
        "\n",
        "        for inputs, labels in train_loader[i]:\n",
        "\n",
        "            counter += 1\n",
        "\n",
        "            inputs, labels = inputs.cuda(), labels.cuda()\n",
        "            inputs = inputs.float()\n",
        "\n",
        "            h = clf.init_hidden(inputs.size(0))\n",
        "            h = tuple([each.data for each in h])\n",
        "\n",
        "            clf.zero_grad()\n",
        "            output, h = clf(inputs, h)\n",
        "\n",
        "            loss = criterion(output.squeeze(), labels)\n",
        "            loss.backward()\n",
        "\n",
        "            # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
        "            nn.utils.clip_grad_norm_(clf.parameters(), clip)\n",
        "            optimizer.step()\n",
        "\n",
        "        clf.eval()\n",
        "        val_losses = []\n",
        "        acc = torch.Tensor([]).cuda()\n",
        "        for inputs, labels in valid_loader[i]:\n",
        "\n",
        "            val_h = clf.init_hidden(inputs.size(0))\n",
        "            val_h = tuple([each.data for each in val_h])\n",
        "\n",
        "            inputs, labels = inputs.cuda(), labels.cuda()\n",
        "            inputs = inputs.float()\n",
        "\n",
        "            output, val_h = clf(inputs, val_h)\n",
        "    #         print(labels)\n",
        "    #         print(torch.argmax(output.squeeze(), dim=1))\n",
        "            pred = torch.argmax(output.squeeze(), dim=1)==labels\n",
        "            acc = torch.cat((acc, pred.float()),0)\n",
        "            val_loss = criterion(output.squeeze(), labels)\n",
        "\n",
        "            val_losses.append(val_loss.item())\n",
        "\n",
        "        clf.train()\n",
        "        if e%print_every == 0:\n",
        "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
        "                  \"Loss: {:.6f}...\".format(loss.item()),\n",
        "                  \"Val Loss: {:.6f}\".format(np.mean(val_losses)),\n",
        "                  \"Val Acc: {:.6f}\".format(torch.mean(acc)))\n",
        "\n",
        "\n",
        "    clf.eval()\n",
        "    preds = torch.Tensor([]).cuda()\n",
        "    labels_all = torch.Tensor([]).cuda()\n",
        "    for inputs, labels in valid_loader[i]:\n",
        "\n",
        "        val_h = clf.init_hidden(inputs.size(0))\n",
        "        val_h = tuple([each.data for each in val_h])\n",
        "\n",
        "        inputs, labels = inputs.cuda(), labels.cuda()\n",
        "        inputs = inputs.float()\n",
        "\n",
        "        output, val_h = clf(inputs, val_h)\n",
        "    #     print(output.shape)\n",
        "        pred = torch.argmax(output.squeeze(), dim=1)\n",
        "        preds = torch.cat((preds, pred.float()),0)\n",
        "        labels_all = torch.cat((labels_all, labels.float()),0)\n",
        "\n",
        "    preds = preds.detach().cpu().numpy()\n",
        "    labels_all = labels_all.cpu().numpy()\n",
        "    print(preds)\n",
        "    from sklearn.metrics import accuracy_score\n",
        "    from sklearn.metrics import precision_score\n",
        "    from sklearn.metrics import recall_score\n",
        "    from sklearn.metrics import f1_score\n",
        "\n",
        "    print(\"Accuracy pada test data:\", accuracy_score(preds, labels_all))\n",
        "    print(\"Precision baseline pada test data:\", precision_score(preds, labels_all))\n",
        "    print(\"Recall baseline pada test data:\", recall_score(preds, labels_all))\n",
        "    print(\"F1 baseline pada test data:\", f1_score(preds, labels_all))\n",
        "    print()\n",
        "    avg_f1 += f1_score(preds, labels_all)\n",
        "    \n",
        "    \n",
        "print(avg_f1/5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TmeYjiPQevx9",
        "colab_type": "text"
      },
      "source": [
        "# Submission"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oro3tPO8ew7B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_dev_A_df = pd.read_csv(\"drive/My Drive/Ukara/data_dev_A.csv\")\n",
        "data_dev_B_df = pd.read_csv(\"drive/My Drive/Ukara/data_dev_B.csv\")\n",
        "data_dev_A = data_dev_A_df.values\n",
        "data_dev_B = data_dev_B_df.values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LK_Le69DBehz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_tes_A_df = pd.read_csv(\"drive/My Drive/Ukara/data_test_A.csv\")\n",
        "data_tes_B_df = pd.read_csv(\"drive/My Drive/Ukara/data_test_B.csv\")\n",
        "data_tes_A = data_tes_A_df.values\n",
        "data_tes_B = data_tes_B_df.values\n",
        "\n",
        "# ppmi_test_a = pd.read_csv(path_ukara + 'data_test_A_ppmi.csv')['PPMI'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nkELjk6P8Iq5",
        "colab_type": "code",
        "outputId": "78354be0-4ae6-4286-d445-d8b585b99c9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "source": [
        "data_tes_A[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['TSA1',\n",
              "       'beradaptasi dengan lingkunagn baru, lahan dan harta mereka yang dulu akan hilang, tertinggalnya teknologi karena bencana lingkunagn tersebut.'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3lLRLPUMfA93",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# DEV\n",
        "# xfd_A = vectorize(data_dev_A[:,1], pre=True)\n",
        "# xfd_B = vectorize2(data_dev_B[:,1], pre=True)\n",
        "\n",
        "# xfd_A = np.c_[xfd_A, find_len(data_dev_A[:,1], tipe='A'), ppmi_test_a]\n",
        "# xfd_B = np.c_[xfd_B, find_len(data_dev_B[:,1], tipe='B')]\n",
        "\n",
        "# predict_A = clf.predict(xfd_A)\n",
        "# predict_B = clf2.predict(xfd_B)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d2e_BxiOiyIG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xft_A = vectorize(data_tes_A[:,1], pre=True)\n",
        "xft_B = vectorize2(data_tes_B[:,1], pre=True)\n",
        "\n",
        "# STACKING DATA\n",
        "xft_A = np.c_[xft_A, find_len(data_tes_A[:,1], tipe='A')]\n",
        "xft_B = np.c_[xft_B, find_len(data_tes_B[:,1], tipe='B')]\n",
        "\n",
        "A1 = np.log(clf_a_1.predict_proba(xft_A))\n",
        "A2 = np.log(clf_a_2.predict_proba(xft_A))\n",
        "B1 = np.log(clf_b_2.predict_proba(xft_B))\n",
        "B2 = np.log(clf_b_3.predict_proba(xft_B))\n",
        "# r1 = 0.05\n",
        "# r2 = 0.35\n",
        "# r3 = 0.25\n",
        "# r4 = 0.35\n",
        "# tot_B = B1*r1 + B2*r2 + B3*r3 + B4*r4\n",
        "\n",
        "out_A = np.c_[A1, A2]\n",
        "out_B = np.c_[B1, B2]\n",
        "\n",
        "predict_tes_A = meta_clf.predict(out_A)\n",
        "predict_tes_B = meta_clf2.predict(out_B)\n",
        "# predict_tes_B = np.argmax(tot_B, 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9D2OC3QWi_fL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # PREDICT NON_STACKING\n",
        "xft_A = vectorize(data_tes_A[:,1].copy(), pre=True)\n",
        "xft_B = vectorize2(data_tes_B[:,1].copy(), pre=True)\n",
        "xft_A = np.c_[xft_A, find_len(data_tes_A[:,1], tipe='A'), ppmi_test_a]\n",
        "xft_B = np.c_[xft_B, find_len(data_tes_B[:,1], tipe='B')]\n",
        "\n",
        "predict_tes_A = clf.predict(xft_A)\n",
        "predict_tes_B = clf2.predict(xft_B)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8WGdYk3VOxJa",
        "colab_type": "code",
        "outputId": "06b3bb1a-ff96-4f4f-9f18-18bdb4dfcc4c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        }
      },
      "source": [
        "for i in range(len(data_tes_A)):\n",
        "    if len(str(data_tes_A[i,1]))<=5:\n",
        "        print(data_tes_A[i,0], predict_tes_A[i])\n",
        "        predict_tes_A[i]=0\n",
        "\n",
        "for i in range(len(data_tes_B)):\n",
        "    if len(str(data_tes_B[i,1]))<=5:\n",
        "        print(data_tes_B[i,0], predict_tes_B[i])\n",
        "        predict_tes_B[i]=0"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TSA369 0\n",
            "TSA373 0\n",
            "TSA390 0\n",
            "TSA465 0\n",
            "TSA536 1\n",
            "TSA715 0\n",
            "TSB592 0\n",
            "TSB751 0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9aONw_r2fJuy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from pandas import DataFrame\n",
        "\n",
        "# prediksi_data_A = {'RES_ID': data_dev_A_df['RES_ID'],\n",
        "#                 'LABEL': predict_A\n",
        "#                 }\n",
        "\n",
        "# prediksi_data_B = {'RES_ID': data_dev_B_df['RES_ID'],\n",
        "#                 'LABEL': predict_B\n",
        "#                 }"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Y5fRteSgMMJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# df_data_A = DataFrame(prediksi_data_A, columns= ['RES_ID', 'LABEL'])\n",
        "# df_data_B = DataFrame(prediksi_data_B, columns= ['RES_ID', 'LABEL'])\n",
        "# frame = [df_data_A, df_data_B]\n",
        "# answer = pd.concat(frame)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "71XOS2W-gdR0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# answer.to_json('ANS_BEST_DEV.json', orient='records')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNtwR3nbENdt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pandas import DataFrame\n",
        "\n",
        "prediksi_data_A = {'RES_ID': data_tes_A_df['RES_ID'],\n",
        "                'LABEL': predict_tes_A\n",
        "                }\n",
        "\n",
        "prediksi_data_B = {'RES_ID': data_tes_B_df['RES_ID'],\n",
        "                'LABEL': predict_tes_B\n",
        "                }\n",
        "\n",
        "df_data_A = DataFrame(prediksi_data_A, columns= ['RES_ID', 'LABEL'])\n",
        "df_data_B = DataFrame(prediksi_data_B, columns= ['RES_ID', 'LABEL'])\n",
        "frame = [df_data_A, df_data_B]\n",
        "answer = pd.concat(frame)\n",
        "\n",
        "answer.to_json('ANS_STACKING_ANEWPRE_BFINAL.json', orient='records')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PSW33Ebmyn90",
        "colab_type": "text"
      },
      "source": [
        "# TRIAL SECTION"
      ]
    }
  ]
}