doc2vec_experiment.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "doc2vec_experiment.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jackyluo-aiit/colab_files/blob/master/doc2vec_experiment.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5e66q-xH_8ID",
        "colab_type": "code",
        "outputId": "cd8b1762-cdf7-4e05-eb9a-04768a9df989",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379
        }
      },
      "source": [
        "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
        "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "!apt-get -y install -qq google-drive-ocamlfuse fuse\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "from oauth2client.client import GoogleCredentials\n",
        "creds = GoogleCredentials.get_application_default()\n",
        "import getpass\n",
        "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
        "vcode = getpass.getpass()\n",
        "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "E: Package 'python-software-properties' has no installation candidate\n",
            "Selecting previously unselected package google-drive-ocamlfuse.\n",
            "(Reading database ... 132681 files and directories currently installed.)\n",
            "Preparing to unpack .../google-drive-ocamlfuse_0.7.13-0ubuntu1~ubuntu18.04.1_amd64.deb ...\n",
            "Unpacking google-drive-ocamlfuse (0.7.13-0ubuntu1~ubuntu18.04.1) ...\n",
            "Setting up google-drive-ocamlfuse (0.7.13-0ubuntu1~ubuntu18.04.1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "··········\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "Please enter the verification code: Access token retrieved correctly.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W1jKxpAkAVcE",
        "colab_type": "code",
        "outputId": "a52e9d92-65d6-42f4-d07c-1d4339330b70",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233
        }
      },
      "source": [
        "!mkdir -p drive\n",
        "!google-drive-ocamlfuse drive\n",
        "!ls\n",
        "import os\n",
        "os.chdir('drive/Dataset')\n",
        "!ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "adc.json  drive  sample_data\n",
            "doc2vec_model\n",
            "drive\n",
            "epoch10_mincount500_window5_dm_alpha025_worker7_minalpha001_model_v2\n",
            "epoch10_mincount500_window5_dm_alpha025_worker7_minalpha001_model_v2.docvecs.vectors_docs.npy\n",
            "LSH_data.txt\n",
            "preprocessed_data.csv\n",
            "sentiment_cleaned.csv\n",
            "stream_data.txt\n",
            "testSet2.txt\n",
            "testSet.txt\n",
            "training.1600000.processed.noemoticon.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j42SQXlz-Bz3",
        "colab_type": "code",
        "outputId": "4ade967c-7127-44f0-d317-b6a237126af5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 829
        }
      },
      "source": [
        "import pickle\n",
        "import pandas as pd\n",
        "import numpy\n",
        "import re\n",
        "import os\n",
        "import numpy as np\n",
        "import gensim\n",
        "import sklearn as sk\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.decomposition import PCA\n",
        "from gensim.models import Doc2Vec\n",
        "import string\n",
        "import re\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.metrics import pairwise_distances_argmin_min\n",
        "from scipy import spatial\n",
        "from numpy.linalg import norm\n",
        "\n",
        "\n",
        "def load_dataset(filename='sentiment_cleaned.csv'):\n",
        "    translator = str.maketrans('', '', string.punctuation)\n",
        "\n",
        "    train_dataset = pd.read_csv('sentiment_cleaned.csv')\n",
        "    train_dataset['content'] = train_dataset['content'].apply(lambda x: str(x).lower())\n",
        "    train_dataset['content'] = train_dataset['content'].apply(lambda x: x.translate(translator))\n",
        "    train_dataset['content'] = train_dataset['content'].apply(lambda x: re.sub('\\s+', ' ', x).strip())\n",
        "    print(\"dataset: \\n\", train_dataset.head)\n",
        "    return train_dataset\n",
        "\n",
        "\n",
        "def split_words(train_dataset):\n",
        "    LabeledSentence1 = gensim.models.doc2vec.TaggedDocument\n",
        "    all_content_train = []\n",
        "    j = 0\n",
        "    for sentence in train_dataset['content'].values:\n",
        "        all_content_train.append(LabeledSentence1(sentence, [j]))\n",
        "        j += 1\n",
        "    print('Num of text processed: ', j)\n",
        "    return all_content_train\n",
        "\n",
        "\n",
        "def doc2vec_model(train_content):\n",
        "    print('...start to build doc2vec model...')\n",
        "    model = Doc2Vec(epochs=10, min_count=500, window=5, dm=1, alpha=0.025, workers=7, min_alpha=0.001)\n",
        "    model.build_vocab(train_content)\n",
        "    print('...vocab built completed...')\n",
        "    model.train(train_content, total_examples=model.corpus_count, epochs=model.epochs)\n",
        "    model.save('epoch10_mincount500_window5_dm_alpha025_worker7_minalpha001_model_v2')\n",
        "    print('...saved model...')\n",
        "    return model\n",
        "\n",
        "\n",
        "def distCosin(vecA, vecB):\n",
        "    # # Vec = np.vstack([vecA,vecB])\n",
        "    # # dist = 1 - pdist(Vec,'cosine'\n",
        "    vecA = np.mat(vecA)\n",
        "    vecB = np.mat(vecB)\n",
        "    num = float(vecA * vecB.T)\n",
        "    denom = np.linalg.norm(vecA) * np.linalg.norm(vecB)\n",
        "    cos = num / denom\n",
        "    dist = 1 - cos\n",
        "    # dist = np.dot(vecA, vecB) / (np.linalg.norm(vecA)*np.linalg.norm(vecB))\n",
        "    # dist = 1 - spatial.distance.cosine(vecA, vecB)\n",
        "    return dist\n",
        "\n",
        "\n",
        "def get_cent(points, k):\n",
        "    m, n = np.shape(points)\n",
        "    cluster_centers = np.mat(np.zeros((k, n)))\n",
        "    index = np.random.randint(0, m)\n",
        "    cluster_centers[0, :] = np.copy(points[index, :])\n",
        "    d = [0.0 for _ in range(m)]\n",
        "    centroids = []\n",
        "    centroids.append(index)\n",
        "    for i in range(0, k - 1):\n",
        "        farest = furest(cluster_centers[i, :], points, centroids)\n",
        "        cluster_centers[i + 1] = np.copy(points[farest, :])\n",
        "        # sum_all = 0\n",
        "        # for j in range(m):\n",
        "        #     d[j] = nearest(points[j, ], cluster_centers[0:i, ])\n",
        "        #     sum_all += d[j]\n",
        "        # sum_all *= random.random()\n",
        "        # for j, di in enumerate(d):\n",
        "        #     sum_all -= di\n",
        "        #     if sum_all > 0:\n",
        "        #         continue\n",
        "        #     cluster_centers[i] = np.copy(points[j, ])\n",
        "        #     break\n",
        "    return cluster_centers\n",
        "\n",
        "\n",
        "def get_custom_cent(points, negative_index, positive_index):\n",
        "    m, n = np.shape(points)\n",
        "    cluster_centers = np.mat(np.zeros((2, n)))\n",
        "    ne_index = np.random.choice(negative_index)\n",
        "    po_index = np.random.choice(positive_index)\n",
        "    cluster_centers[0, :] = np.copy(points[ne_index, :])\n",
        "    cluster_centers[1, :] = np.copy(points[po_index, :])\n",
        "    print(\"Chosen centers:\\n\", (ne_index, po_index))\n",
        "    return cluster_centers\n",
        "\n",
        "\n",
        "def furest(centroid, points, centroids):\n",
        "    m = np.shape(points)[0]\n",
        "    max_dist = 0\n",
        "    max_index = -1\n",
        "    for i in range(m):\n",
        "        if i in centroids:\n",
        "            continue\n",
        "        dist = distCosin(points[i], centroid)\n",
        "        # dist = 1 - spatial.distance.cosine(points[i], centroid)\n",
        "        if dist > max_dist:\n",
        "            max_dist = dist\n",
        "            max_index = i\n",
        "    return max_index\n",
        "\n",
        "\n",
        "def kMeans(dataSet, k, ne_index, po_index, distMeans=distCosin, createCent=get_custom_cent):\n",
        "    m = np.shape(dataSet)[0]\n",
        "    clusterRecord = np.mat(np.zeros((m, 2)))\n",
        "    # centroids = createCent(dataSet, k)\n",
        "    custom_centroids = createCent(dataSet, ne_index, po_index)\n",
        "    clusterUpdate = True\n",
        "    while clusterUpdate:\n",
        "        clusterUpdate = False\n",
        "        for i in range(m):  # for each point in dataset (d1,d2,dn...)==(x,y,z,...)\n",
        "            # initiate distance and cluter\n",
        "            minDist = np.inf\n",
        "            minIndexofCluster = -1\n",
        "            for clusterCent in range(k):\n",
        "                dist = distMeans(dataSet[i, :], custom_centroids[clusterCent, :])\n",
        "                if dist < minDist:\n",
        "                    minDist = dist\n",
        "                    minIndexofCluster = clusterCent\n",
        "            if clusterRecord[i, 0] != minIndexofCluster:\n",
        "                clusterUpdate = True\n",
        "                clusterRecord[i, :] = minIndexofCluster, minDist\n",
        "        # print(centroids)\n",
        "        for cent in range(k):\n",
        "            pointsInClust = dataSet[np.nonzero(clusterRecord[:, 0].A == cent)[0],\n",
        "                            :]  # select the nonzero rows whose index is equal to the current centroid index, and then select the corresponding rows in dataset.\n",
        "            custom_centroids[cent, :] = np.mean(pointsInClust, axis=0)\n",
        "\n",
        "    return custom_centroids, clusterRecord\n",
        "\n",
        "\n",
        "if __name__ == '__main__':\n",
        "    dataset = load_dataset()\n",
        "    all_text = list(dataset['content'])\n",
        "    test_dataset1 = dataset.loc[0:1, ['label', 'content']]\n",
        "    test_dataset2 = dataset.loc[1599997:1599998, ['label', 'content']]\n",
        "    test_dataset = test_dataset1.append(test_dataset2, ignore_index=True)\n",
        "    train_dataset = dataset.loc[2:1599996, ['label', 'content']]\n",
        "    train_dataset.reset_index(drop=True, inplace=True)\n",
        "    negative_set = train_dataset.loc[(train_dataset['label'] == 0)]\n",
        "    positive_set = train_dataset.loc[(train_dataset['label'] == 4)]\n",
        "    negative_setIndex = negative_set.index.values\n",
        "    positive_setIndex = positive_set.index.values\n",
        "    print(\"negative_index:\",negative_setIndex)\n",
        "    print(\"positive_index:\",positive_setIndex)\n",
        "    print(\"train_dataset: \\n\", train_dataset.head)\n",
        "    print(\"test_dataset: \\n\", test_dataset.head)\n",
        "    train_content = split_words(train_dataset)\n",
        "    # model = doc2vec_model(train_content)\n",
        "    model = Doc2Vec.load('epoch10_mincount500_window5_dm_alpha025_worker7_minalpha001_model_v2')\n",
        "    #                      'Data/project/the-disagreeable-frogs/doc2vec_model'\n",
        "    #                      '/epoch10_mincount500_window5_dm_alpha025_worker7_minalpha001_model')\n",
        "\n",
        "    # print('testing model:')\n",
        "    # for test_sentence in test_dataset['content'].values:\n",
        "    #     test_text = test_sentence.split(' ')\n",
        "    #     inferred_vector = model.infer_vector(doc_words=test_text, alpha=0.025, steps=500)\n",
        "    #     sim_sentence = model.docvecs.most_similar([inferred_vector], topn=5)\n",
        "    #     print(\"testing text:\", test_text)\n",
        "    #     for index, similarity in sim_sentence:\n",
        "    #         # print(index, similarity)\n",
        "    #         sentence = all_text[index]\n",
        "    #         print(\"similar text in train_dataset: \", index, sentence, similarity)\n",
        "\n",
        "    print('using kmeans++:')\n",
        "    X = model.docvecs.vectors_docs\n",
        "\n",
        "    # negative = X[0]\n",
        "    # negative1 = X[1]\n",
        "    # positive = X[1599998]\n",
        "    # # result = spatial.distance.cosine(negative, positive)\n",
        "    # os_sim = distCosin(negative, positive)\n",
        "    # result = distCosin(negative, negative1)\n",
        "    # print(\"distance between %s and %s:\"%(train_dataset.loc[0], train_dataset.loc[1]))\n",
        "    # print(result)\n",
        "    # print(\"distance between %s and %s:\"%(train_dataset.loc[0], train_dataset.loc[1599998]))\n",
        "    # print(os_sim)\n",
        "\n",
        "    # kmeans_model = KMeans(n_clusters=2, init='k-means++', max_iter=100)\n",
        "    # kmeans_model.fit(X)\n",
        "    # centorids = kmeans_model.cluster_centers_\n",
        "    # # closest, _ = pairwise_distances_argmin_min(kmeans_model.cluster_centers_, X)\n",
        "    # # for index in closest:\n",
        "    # #     print(\"The closest index of cluster: \", index)\n",
        "    # #     print(train_dataset.loc[index])\n",
        "    #\n",
        "    # count = 0\n",
        "    # cluster_fp_set = {}\n",
        "    # label_dict = {i: np.where(kmeans_model.labels_ == i)[0] for i in range(kmeans_model.n_clusters)}\n",
        "    # print(\"label_dict:\", label_dict)\n",
        "    # for key in label_dict.keys():\n",
        "    #     count = 0\n",
        "    #     cluster_set = label_dict[key]\n",
        "    #     print(\"cluster_set:\", cluster_set)\n",
        "    #     for item in cluster_set:\n",
        "    #         if train_dataset.loc[item, ['label']].values != key:\n",
        "    #             count += 1\n",
        "    #     cluster_fp = count / len(cluster_set)\n",
        "    #     cluster_fp_set[key] = cluster_fp\n",
        "    #\n",
        "    # print(\"fpp: \\n\", cluster_fp_set)\n",
        "\n",
        "\n",
        "    # label = kmeans_model.labels_\n",
        "    #\n",
        "    # pca = PCA(n_components=2).fit(X)\n",
        "    # datapoint = pca.transform(X)\n",
        "    #\n",
        "    # plt.figure\n",
        "    # label1 = [\"r\", \"b\"]\n",
        "    # color = [label1[i] for i in label]\n",
        "    # plt.scatter(datapoint[:, 0], datapoint[:, 1], c=color)\n",
        "    #\n",
        "    # centroidpoint = pca.transform(centorids)\n",
        "    # plt.scatter(centroidpoint[:, 0], centroidpoint[:, 1], marker='^', s=150, c='k')\n",
        "    # plt.show()\n",
        "\n",
        "    # custom kmeans++\n",
        "    k = 2\n",
        "    custom_centroid, custom_cluster = kMeans(X, k, negative_setIndex, positive_setIndex)\n",
        "    df = dfcluster = pd.DataFrame(custom_cluster, columns=['label', 'distance'])\n",
        "    label_dict = {i:dfcluster[(dfcluster['label'] == i)].index for i in range(0, k)}\n",
        "    print(label_dict)\n",
        "    cluster_fp_set = {}\n",
        "    for key in label_dict.keys():\n",
        "      count = 0\n",
        "      cluster_set = label_dict[key]\n",
        "      # print(\"cluster_set:\", cluster_set)\n",
        "      for item in cluster_set:\n",
        "        if key != 0:\n",
        "          key = 4 \n",
        "        if train_dataset.loc[item, ['label']].values != key:\n",
        "          count += 1\n",
        "      cluster_fp = count / len(cluster_set)\n",
        "      cluster_fp_set[key] = cluster_fp\n",
        "    print(\"fpp: \\n\", cluster_fp_set)\n",
        "    with open('result_doc2vec.csv','wb') as f:\n",
        "      w = csv.writer(f)\n",
        "      w.writerows(cluster_fp_set.items())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dataset: \n",
            " <bound method NDFrame.head of          Unnamed: 0  label                                            content\n",
            "0                 0      0  is upset that he cant update his facebook by t...\n",
            "1                 1      0  i dived many times for the ball managed to sav...\n",
            "2                 2      0     my whole body feels itchy and like its on fire\n",
            "3                 3      0  no its not behaving at all im mad why am i her...\n",
            "4                 4      0                                 not the whole crew\n",
            "...             ...    ...                                                ...\n",
            "1599994     1599994      4  just woke up having no school is the best feel...\n",
            "1599995     1599995      4  thewdbcom very cool to hear old walt interview...\n",
            "1599996     1599996      4  are you ready for your mojo makeover ask me fo...\n",
            "1599997     1599997      4  happy th birthday to my boo of alll time tupac...\n",
            "1599998     1599998      4                                              happy\n",
            "\n",
            "[1599999 rows x 3 columns]>\n",
            "negative_index: [     0      1      2 ... 799994 799995 799996]\n",
            "positive_index: [ 799997  799998  799999 ... 1599992 1599993 1599994]\n",
            "train_dataset: \n",
            " <bound method NDFrame.head of          label                                            content\n",
            "0            0     my whole body feels itchy and like its on fire\n",
            "1            0  no its not behaving at all im mad why am i her...\n",
            "2            0                                 not the whole crew\n",
            "3            0                                         need a hug\n",
            "4            0  hey long time no see yes rains a bit only a bi...\n",
            "...        ...                                                ...\n",
            "1599990      4                                                nan\n",
            "1599991      4  yeah that does work better than just waiting f...\n",
            "1599992      4  just woke up having no school is the best feel...\n",
            "1599993      4  thewdbcom very cool to hear old walt interview...\n",
            "1599994      4  are you ready for your mojo makeover ask me fo...\n",
            "\n",
            "[1599995 rows x 2 columns]>\n",
            "test_dataset: \n",
            " <bound method NDFrame.head of    label                                            content\n",
            "0      0  is upset that he cant update his facebook by t...\n",
            "1      0  i dived many times for the ball managed to sav...\n",
            "2      4  happy th birthday to my boo of alll time tupac...\n",
            "3      4                                              happy>\n",
            "Num of text processed:  1599995\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "using kmeans++:\n",
            "Chosen centers:\n",
            " (78167, 1187254)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}