LDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 591
    },
    "colab_type": "code",
    "id": "pXQRiPWAEifM",
    "outputId": "af42a2ea-d597-44ce-a0ae-0cf34771b50b"
   },
   "outputs": [],
   "source": [
    "#!pip install pyLDAvis\n",
    "import gensim\n",
    "import pyLDAvis\n",
    "from pyLDAvis import gensim as gensimvis\n",
    "from pyLDAvis import sklearn\n",
    "import spacy\n",
    "\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer \n",
    "from sklearn.preprocessing import Normalizer\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pprint as pp\n",
    "import pickle\n",
    "import tqdm\n",
    "from collections import Counter\n",
    "from nltk import ngrams, bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kbTl1ZbLFSn5"
   },
   "outputs": [],
   "source": [
    "with open('./datasets/borges_full_texts.pkl', 'rb') as f:\n",
    "   data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 400
    },
    "colab_type": "code",
    "id": "iXYo-ZxxFm-r",
    "outputId": "89580ceb-92ac-4210-c4e5-2c09c48e5210"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>text_metadata</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://ciudadseva.com/texto/abel-y-cain-borges/</td>\n",
       "      <td>{'title': 'Abel y Caín', 'metadata': '[Minicue...</td>\n",
       "      <td>Abel y Caín se encontraron después de la muert...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://ciudadseva.com/texto/adrogue/</td>\n",
       "      <td>{'title': 'Adrogué', 'metadata': '[Minicuento ...</td>\n",
       "      <td>Era muy lindo, un pueblo laberíntico. A veces,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://ciudadseva.com/texto/alguien-sonara/</td>\n",
       "      <td>{'title': 'Alguien soñará', 'metadata': '[Mini...</td>\n",
       "      <td>¿Qué soñará el indescifrable futuro? Soñará qu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://ciudadseva.com/texto/andres-armoa/</td>\n",
       "      <td>{'title': 'Andrés Armoa', 'metadata': '[Minicu...</td>\n",
       "      <td>Los años le han dejado unas palabras en guaran...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://ciudadseva.com/texto/argumentum-ornith...</td>\n",
       "      <td>{'title': 'Argumentum ornithologicum', 'metada...</td>\n",
       "      <td>Cierro los ojos y veo una bandada de pájaros. ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                link  \\\n",
       "0   https://ciudadseva.com/texto/abel-y-cain-borges/   \n",
       "1              https://ciudadseva.com/texto/adrogue/   \n",
       "2       https://ciudadseva.com/texto/alguien-sonara/   \n",
       "3         https://ciudadseva.com/texto/andres-armoa/   \n",
       "4  https://ciudadseva.com/texto/argumentum-ornith...   \n",
       "\n",
       "                                       text_metadata  \\\n",
       "0  {'title': 'Abel y Caín', 'metadata': '[Minicue...   \n",
       "1  {'title': 'Adrogué', 'metadata': '[Minicuento ...   \n",
       "2  {'title': 'Alguien soñará', 'metadata': '[Mini...   \n",
       "3  {'title': 'Andrés Armoa', 'metadata': '[Minicu...   \n",
       "4  {'title': 'Argumentum ornithologicum', 'metada...   \n",
       "\n",
       "                                                text  \n",
       "0  Abel y Caín se encontraron después de la muert...  \n",
       "1  Era muy lindo, un pueblo laberíntico. A veces,...  \n",
       "2  ¿Qué soñará el indescifrable futuro? Soñará qu...  \n",
       "3  Los años le han dejado unas palabras en guaran...  \n",
       "4  Cierro los ojos y veo una bandada de pájaros. ...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qOZXACE6Frjd"
   },
   "outputs": [],
   "source": [
    "corpus_vectorizado = CountVectorizer( max_features = 40000, ngram_range=(1,3), min_df=3, max_df=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "                dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "                lowercase=True, max_df=0.4, max_features=40000, min_df=3,\n",
       "                ngram_range=(1, 3), preprocessor=None, stop_words=None,\n",
       "                strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "                tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_vectorizado"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vt3HSEcRFvQ8"
   },
   "outputs": [],
   "source": [
    "lda = LatentDirichletAllocation(n_components = 30,learning_method='batch',max_iter=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4MgSV9TWHFrw"
   },
   "outputs": [],
   "source": [
    "corpus_para_lda = corpus_vectorizado.fit_transform(data.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Qk2SAKeOMIIR"
   },
   "outputs": [],
   "source": [
    "def imprimir_palabras_mas_relevantes(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        message = \"Topic #%d: \" % topic_idx\n",
    "        message += \" \".join([feature_names[i]+\" || \"\n",
    "                             for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
    "        print(message)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 151
    },
    "colab_type": "code",
    "id": "FcJ0qhfrUm_P",
    "outputId": "8e84c40b-41e9-4337-9184-54851e14eaa1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "                          evaluate_every=-1, learning_decay=0.7,\n",
       "                          learning_method='batch', learning_offset=10.0,\n",
       "                          max_doc_update_iter=100, max_iter=100,\n",
       "                          mean_change_tol=0.001, n_components=30, n_jobs=None,\n",
       "                          perp_tol=0.1, random_state=None,\n",
       "                          topic_word_prior=None, total_samples=1000000.0,\n",
       "                          verbose=0)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda.fit(corpus_para_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "colab_type": "code",
    "id": "zj10HVG3U8nZ",
    "outputId": "b332a51a-7624-4fdc-a886-3c3ad84ffc22"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topics in LDA model:\n",
      "Topic #0: juan ||  mujer ||  iba || \n",
      "Topic #1: sueño ||  desierto ||  luego || \n",
      "Topic #2: negro ||  que yo ||  cara || \n",
      "Topic #3: pedro ||  coronel ||  tan || \n",
      "Topic #4: que las ||  para el ||  el mundo || \n",
      "Topic #5: mapa ||  imperio ||  del imperio || \n",
      "Topic #6: serpiente ||  será ||  arca || \n",
      "Topic #7: vi ||  aleph ||  carlos || \n",
      "Topic #8: biblioteca ||  libros ||  la biblioteca || \n",
      "Topic #9: habían ||  padre ||  muchacha || \n",
      "Topic #10: que me ||  cuatro ||  palabra || \n",
      "Topic #11: infamia ||  nosotros ||  crucifixión de || \n",
      "Topic #12: interlocutor ||  hacia el alba ||  de la frontera || \n",
      "Topic #13: único ||  entrar ||  entonces la || \n",
      "Topic #14: jardín ||  iba ||  usted || \n",
      "Topic #15: único ||  entrar ||  entonces la || \n",
      "Topic #16: obra ||  han ||  segundo || \n",
      "Topic #17: máquina ||  sueño ||  cuerpo || \n",
      "Topic #18: infinito ||  azar ||  el libro || \n",
      "Topic #19: que me ||  te ||  gente || \n",
      "Topic #20: luego ||  recordó ||  pensó || \n",
      "Topic #21: ciudad ||  la ciudad ||  último || \n",
      "Topic #22: ya no ||  soy ||  niño || \n",
      "Topic #23: no hay ||  dije ||  te || \n",
      "Topic #24: único ||  entrar ||  entonces la || \n",
      "Topic #25: cruz ||  vi ||  dios || \n",
      "Topic #26: juan ||  judas ||  dios || \n",
      "Topic #27: que es ||  oro ||  tortuga || \n",
      "Topic #28: tiene ||  sabe ||  cuenta || \n",
      "Topic #29: un hombre que ||  hombre que ||  circular || \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nTopics in LDA model:\")\n",
    "tf_feature_names = corpus_vectorizado.get_feature_names()\n",
    "n_top_words = 3\n",
    "imprimir_palabras_mas_relevantes(lda, tf_feature_names, n_top_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lematizando"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 62/62 [00:10<00:00,  5.68it/s]\n"
     ]
    }
   ],
   "source": [
    "nlp = spacy.load(\"es_core_news_sm\")\n",
    "\n",
    "data_base = data.copy()\n",
    "\n",
    "def lemmafy(doc):\n",
    "    lemmas = []\n",
    "    for token in doc:\n",
    "        if not token.is_stop and token.is_alpha:\n",
    "             lemmas.append(token.lemma_)\n",
    "    return ' '.join(lemmas)\n",
    "\n",
    "texts = []\n",
    "for text in tqdm.tqdm(data_base[\"text\"]):\n",
    "    doc = nlp(text)\n",
    "    lemmed = lemmafy(doc)\n",
    "    texts.append(lemmed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     Abel y Caín encontrar muerte Abel Caminaban de...\n",
       "1     lindar poblar laberíntico A noche verano salir...\n",
       "2     soñar indescifrable futuro Soñará Alonso Quija...\n",
       "3     año dejar palabra guaraní ocasión requerir tra...\n",
       "4     Cierro ojo y ver bandada pájaro visión duro o ...\n",
       "                            ...                        \n",
       "57    I Debo a conjunción espejar y enciclopedia des...\n",
       "58    Asia Menor o Alejandría siglo fe Basílides pub...\n",
       "59    Imaginemos Toledo descubrir papel texto arábig...\n",
       "60    desierto Irán alto torrar piedra puerta ventan...\n",
       "61    cerro igualar tierra llanura y ir caminar llan...\n",
       "Name: texts, Length: 62, dtype: object"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_texts = pd.DataFrame({\"texts\":texts})\n",
    "df_texts[\"texts\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_lemmed = LatentDirichletAllocation(n_components = 30,learning_method='batch',max_iter=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus_lemmed = corpus_vectorizado.fit_transform(df_texts.texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LatentDirichletAllocation(batch_size=128, doc_topic_prior=None,\n",
       "                          evaluate_every=-1, learning_decay=0.7,\n",
       "                          learning_method='batch', learning_offset=10.0,\n",
       "                          max_doc_update_iter=100, max_iter=100,\n",
       "                          mean_change_tol=0.001, n_components=30, n_jobs=None,\n",
       "                          perp_tol=0.1, random_state=None,\n",
       "                          topic_word_prior=None, total_samples=1000000.0,\n",
       "                          verbose=0)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda_lemmed.fit(corpus_lemmed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Topics in LDA model AFTER LEMMING:\n",
      "Topic #0: universo ||  tigre ||  caminar || \n",
      "Topic #1: azar ||  número ||  luna || \n",
      "Topic #2: laberinto ||  jardín ||  bifurcar || \n",
      "Topic #3: único ||  enumerar ||  entró || \n",
      "Topic #4: biblioteca ||  libro ||  letra || \n",
      "Topic #5: quijote ||  novelar ||  capítulo || \n",
      "Topic #6: carlos ||  aleph ||  argentino || \n",
      "Topic #7: juan ||  tío ||  madre || \n",
      "Topic #8: cuerpo ||  máquina ||  ulises || \n",
      "Topic #9: cruz ||  juan ||  negro || \n",
      "Topic #10: judas ||  publicar ||  jesús || \n",
      "Topic #11: tomo ||  objeto ||  realidad || \n",
      "Topic #12: recuerdo ||  treinta ||  latín || \n",
      "Topic #13: único ||  enumerar ||  entró || \n",
      "Topic #14: monedar ||  pedro ||  enciclopedia || \n",
      "Topic #15: único ||  enumerar ||  entró || \n",
      "Topic #16: obrar ||  acto ||  autor || \n",
      "Topic #17: padre ||  fábrica ||  temor || \n",
      "Topic #18: padre ||  hijo ||  indio || \n",
      "Topic #19: único ||  enumerar ||  entró || \n",
      "Topic #20: abrir ||  señor ||  sacar || \n",
      "Topic #21: nombre ||  crimen ||  letra || \n",
      "Topic #22: abuelo ||  inglés ||  destinar || \n",
      "Topic #23: cuerpo ||  cabeza ||  ser || \n",
      "Topic #24: sur ||  viejo ||  tren || \n",
      "Topic #25: ciudad ||  reír ||  aguar || \n",
      "Topic #26: pájaro ||  número ||  diez || \n",
      "Topic #27: inglés ||  galería ||  cicatriz || \n",
      "Topic #28: rey ||  piedra ||  puerta || \n",
      "Topic #29: fuego ||  hijo ||  templar || \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nTopics in LDA model AFTER LEMMING:\")\n",
    "tf_feature_names = corpus_vectorizado.get_feature_names()\n",
    "n_top_words = 3\n",
    "imprimir_palabras_mas_relevantes(lda_lemmed, tf_feature_names, n_top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unigrams: [[(('Abel',), 5)], [(('lindar',), 1)], [(('Soñará',), 9)], [(('y',), 5)], [(('ver',), 5)], [(('a',), 28)], [(('a',), 28)], [(('y',), 6)], [(('y',), 3)], [(('y',), 51)], [(('reprobar',), 2)], [(('y',), 98)], [(('y',), 12)], [(('y',), 49)], [(('y',), 27)], [(('y',), 80)], [(('y',), 129)], [(('y',), 101)], [(('y',), 34)], [(('y',), 51)], [(('y',), 63)], [(('y',), 4)], [(('y',), 4)], [(('y',), 6)], [(('y',), 94)], [(('y',), 68)], [(('y',), 9)], [(('y',), 35)], [(('y',), 68)], [(('a',), 55)], [(('y',), 55)], [(('y',), 119)], [(('y',), 3)], [(('y',), 42)], [(('y',), 74)], [(('y',), 31)], [(('y',), 24)], [(('y',), 53)], [(('y',), 34)], [(('y',), 39)], [(('o',), 7)], [(('y',), 37)], [(('y',), 89)], [(('y',), 68)], [(('a',), 6)], [(('y',), 24)], [(('y',), 8)], [(('y',), 60)], [(('tanto',), 1)], [(('y',), 19)], [(('y',), 17)], [(('y',), 71)], [(('y',), 11)], [(('y',), 7)], [(('y',), 7)], [(('y',), 70)], [(('y',), 28)], [(('y',), 159)], [(('y',), 44)], [(('y',), 15)], [(('y',), 3)], [(('y',), 53)]]\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "counts = []\n",
    "for text in df_texts[\"texts\"]:\n",
    "    uni_tokens = ngrams(text.split(\" \"),1)\n",
    "    #bi_tokens = bigrams(df_texts)\n",
    "    #bi_counts = Counter(bi_tokens)\n",
    "    uni_counts = Counter(uni_tokens)\n",
    "    counts.append(uni_counts.most_common(1))\n",
    "#print([(item, counts.count(item)) for item in sorted(set(bi_tokens))])\n",
    "print(\"Unigrams:\", counts)\n",
    "#print(\"\\n\")\n",
    "#print(\"Bigrams:\", bi_counts.most_common(5))\n",
    "print(\"\\n\")\n",
    "#df_texts[\"texts\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove stopwords!\n",
    "https://stackoverflow.com/questions/5486337/how-to-remove-stop-words-using-nltk-or-python"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "Ejemplo LDA twitter.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}