CS_W2V.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8UOtzlzU4frl"
      },
      "outputs": [],
      "source": [
        "from konlpy.tag import Twitter\n",
        "from gensim.models import Word2Vec\n",
        "import csv"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\"\n",
        "@author: lumyjuwon\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "bHsZpJbr4gpt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "twitter = Twitter()\n",
        "\n",
        "file = open(\"Article_shuffled.csv\", 'r', encoding='euc-kr')\n",
        "line = csv.reader(file)\n",
        "token = []\n",
        "embeddingmodel = []"
      ],
      "metadata": {
        "id": "Tt0KJ9k04gsL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in line:\n",
        "    content = i[3]  # csv에서 뉴스 제목 또는 뉴스 본문 column으로 변경\n",
        "    sentence = twitter.pos(i[0], norm=True, stem=True)\n",
        "    temp = []\n",
        "    temp_embedding = []\n",
        "    all_temp = []\n",
        "    for k in range(len(sentence)):\n",
        "        temp_embedding.append(sentence[k][0])\n",
        "        temp.append(sentence[k][0] + '/' + sentence[k][1])\n",
        "    all_temp.append(temp)\n",
        "    embeddingmodel.append(temp_embedding)\n",
        "    category = i[1]  # csv에서 category column으로 변경\n",
        "    category_number_dic = {'IT과학': 0, '경제': 1, '정치': 2, 'e스포츠': 3, '골프': 4, '농구': 5, '배구': 6, '야구': 7, '일반 스포츠': 8, '축구': 9, '사회': 10, '생활문화': 11}\n",
        "    all_temp.append(category_number_dic.get(category))\n",
        "    token.append(all_temp)\n",
        "print(\"토큰 처리 완료\")"
      ],
      "metadata": {
        "id": "XnIjCz-v4gus"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embeddingmodel = []\n",
        "for i in range(len(token)):\n",
        "    temp_embeddingmodel = []\n",
        "    for k in range(len(token[i][0])):\n",
        "        temp_embeddingmodel.append(token[i][0][k])\n",
        "    embeddingmodel.append(temp_embeddingmodel)\n",
        "embedding = Word2Vec(embeddingmodel, size=300, window=5, min_count=10, iter=5, sg=1, max_vocab_size=360000000)\n",
        "embedding.save('post.embedding')"
      ],
      "metadata": {
        "id": "8MJchVMU4gww"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}