week07_homework.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = pyspark.sql.SparkSession.builder.master('local').getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Korean document version "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "linesDF = spark.read \\\n",
    "        .format('csv') \\\n",
    "        .option('encoding', 'EUC-kr') \\\n",
    "        .option('header', 'False') \\\n",
    "        .option('sep', '\\n') \\\n",
    "        .load('MisCurriculum.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(_c0='현대경영과기업가정신[(구)경영학원론]'),\n",
       " Row(_c0='(Contemporary Business and Entrepreurship)'),\n",
       " Row(_c0='자유자본주의 체제하에서의 기업경영과 관련된 전반적 개념을 연구한다. 구체적으로 경영환경을 포함하여 경영활동의 계획, 집행 및 통제기법과 그 밖에 개인 및 집단행동, 조직구조 등과 관련된 기초 원리를 연구한다. ')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linesDF.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformation lines to RDD\n",
    "linesRDD = linesDF.rdd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "wcRDD = linesRDD.map(lambda line: line[0]) \\\n",
    "        .flatMap(lambda line : re.split('\\\\W+', line)) \\\n",
    "        .filter(lambda word : len(word)>0) \\\n",
    "        .map(lambda word : (word,1)) \\\n",
    "        .reduceByKey(lambda v, w : v+w) \\\n",
    "        .sortBy(lambda vw : vw[1], False) \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformation RDD to DF\n",
    "wcDF = spark.createDataFrame(wcRDD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _1: string (nullable = true)\n",
      " |-- _2: long (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDF.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+---+\n",
      "|      _1| _2|\n",
      "+--------+---+\n",
      "|      및| 41|\n",
      "|학습한다| 15|\n",
      "|Business| 14|\n",
      "+--------+---+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDF.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark.sql.functions as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "wcDFnamed = wcDF.withColumnRenamed('_1', 'words')\\\n",
    "                 .withColumnRenamed('_2', 'freq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- words: string (nullable = true)\n",
      " |-- freq: long (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDFnamed.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----+\n",
      "|   words|freq|\n",
      "+--------+----+\n",
      "|      및|  41|\n",
      "|학습한다|  15|\n",
      "|Business|  14|\n",
      "+--------+----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDFnamed.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "wcDFlength = wcDFnamed.withColumn('wordLen', F.length('words'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+----+-------+\n",
      "|   words|freq|wordLen|\n",
      "+--------+----+-------+\n",
      "|      및|  41|      1|\n",
      "|학습한다|  15|      4|\n",
      "|Business|  14|      8|\n",
      "+--------+----+-------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDFlength.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----+-------+\n",
      "|     words|freq|wordLen|\n",
      "+----------+----+-------+\n",
      "|        및|  41|      1|\n",
      "|  학습한다|  15|      4|\n",
      "|  Business|  14|      8|\n",
      "|      등을|  13|      2|\n",
      "|      대한|  12|      2|\n",
      "|      분석|  11|      2|\n",
      "|        수|  11|      1|\n",
      "|      이를|  11|      2|\n",
      "|Management|  11|     10|\n",
      "|  공부한다|  10|      4|\n",
      "|    다양한|   9|      3|\n",
      "|      위해|   9|      2|\n",
      "|      한다|   9|      2|\n",
      "|        IT|   9|      2|\n",
      "|       and|   8|      3|\n",
      "|        본|   8|      1|\n",
      "|  목적으로|   8|      4|\n",
      "|    이론을|   8|      3|\n",
      "|        등|   8|      1|\n",
      "|    디지털|   8|      3|\n",
      "+----------+----+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDFlength.filter(wcDFlength.wordLen >= 1).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# agg function and show average using 'F.avg'\n",
    "wordLengthFrequency = wcDFlength\\\n",
    "                     .groupby(F.col('wordLen'))\\\n",
    "                     .agg(F.count(F.col('freq')), F.avg(F.col('freq')))\\\n",
    "                     .orderBy(F.col('wordLen'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------+------------------+\n",
      "|wordLen|count(freq)|         avg(freq)|\n",
      "+-------+-----------+------------------+\n",
      "|      1|         23| 4.130434782608695|\n",
      "|      2|        197|2.1319796954314723|\n",
      "|      3|        308|1.4545454545454546|\n",
      "|      4|        216|1.4953703703703705|\n",
      "|      5|        128|         1.2109375|\n",
      "|      6|         53| 1.320754716981132|\n",
      "|      7|         31|1.1935483870967742|\n",
      "|      8|         22|1.9090909090909092|\n",
      "|      9|         13|1.2307692307692308|\n",
      "|     10|         15|1.7333333333333334|\n",
      "|     11|          8|             1.625|\n",
      "|     12|          6|1.1666666666666667|\n",
      "|     13|          4|               1.0|\n",
      "|     14|          2|               1.0|\n",
      "+-------+-----------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wordLengthFrequency.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plus argument 'F.sum'\n",
    "wordLengthFrequency2 = wcDFlength\\\n",
    "                     .groupby(F.col('wordLen'))\\\n",
    "                     .agg(F.count(F.col('freq')), F.avg(F.col('freq')), F.sum(F.col('freq')))\\\n",
    "                     .orderBy(F.col('wordLen'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------+------------------+---------+\n",
      "|wordLen|count(freq)|         avg(freq)|sum(freq)|\n",
      "+-------+-----------+------------------+---------+\n",
      "|      1|         23| 4.130434782608695|       95|\n",
      "|      2|        197|2.1319796954314723|      420|\n",
      "|      3|        308|1.4545454545454546|      448|\n",
      "|      4|        216|1.4953703703703705|      323|\n",
      "|      5|        128|         1.2109375|      155|\n",
      "|      6|         53| 1.320754716981132|       70|\n",
      "|      7|         31|1.1935483870967742|       37|\n",
      "|      8|         22|1.9090909090909092|       42|\n",
      "|      9|         13|1.2307692307692308|       16|\n",
      "|     10|         15|1.7333333333333334|       26|\n",
      "|     11|          8|             1.625|       13|\n",
      "|     12|          6|1.1666666666666667|        7|\n",
      "|     13|          4|               1.0|        4|\n",
      "|     14|          2|               1.0|        2|\n",
      "+-------+-----------+------------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wordLengthFrequency2.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = wordLengthFrequency2.select(F.col('wordLen')).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "xA = np.array(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform 1 dimension using 'ravel' fun\n",
    "xA1 = np.ravel(xA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = wordLengthFrequency2.select(F.col('count(freq)')).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "xB = np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "xB1 = np.ravel(xB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEGCAYAAACevtWaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3wU9dX48c8hCUmAGAQhXIKCgMhFCIabYIWIIl6RQhWNgJeKTxGVWmx92v681kfb0opWpWK9QEVRUKsFFBSIqAWFSAABEYqgEcL9koABkpzfHzMJgdw2yWwm2T3v12tfuzs7e+YkgbOz3/nOGVFVjDHGhJZ6fidgjDHGe1bcjTEmBFlxN8aYEGTF3RhjQpAVd2OMCUGRficAcMYZZ2jbtm2DFv/w4cM0bNjQ4taxuMGMbXGDG9fUjPT09D2q2qzUF1XV91tycrIG05IlSyxuHYwbzNgWN7hxTc0AVmoZddWGZYwxJgRZcTfGmBBkxd0YY0JQrTigaowp2/Hjx2nUqBEbNmzwPHZ8fHxQ4hpvxcTEkJiYSFRUVMDvseJuTC2XmZlJQkICiYmJiIinsbOzs4mLi/M0pvGWqrJ3714yMzNp165dwO+zYRljarnc3Fzi4+M9L+ymbhARmjZtSm5ubqXeZ8W9ttmxg6R77oGsLL8zMbWIFfbwVpW/vxX32ubRR4lfuxYeecTvTIwxdZgV99oiNhZEYOpURBWmTnWex8b6nZkxdcrTTz9N586dSU1N9TsVX9kBVR/IwyW/YrW4EyYvgOvXQaTCsXrwZleYNCSXnaWsrw/aRVZMOXbsgFGj4I03oEULv7MpIT8/n4iIiKDEfu6553j//fdLHHzMy8sjMjJ8Sp7tudcSWXGQUx8i3JodVQCHomGnTWQwVfHoo/Dpp54M71177bUkJyfTtWtXpk2bBsDUqVP59a9/XbTOK6+8wl133QXAq6++Sp8+fUhKSuKOO+4gPz8fgEaNGvHAAw/Qt29fli1bxiOPPELv3r3p1q0b48aNQ92rwq1YsYLu3btzwQUXcN9999GtWzfA+UC477776N27N927d+f5558vkev//M//sGXLFq655hqefPJJHnroIcaNG8eQIUMYM2ZMmTFUlQkTJtClSxeuvPJKrrjiCubMmQNA27Zt2bNnDwArV65k0KBBgNOX59Zbb6V379707NmTd999t+h38dOf/pShQ4fSsWPHk35PH3zwAeeffz49evRg8ODBFBQU0LFjR3bv3g1AQUEBHTp0KNpedVT4MSYiMcBSINpdf46qPigi7YBZQBPgS2C0qh4TkWhgBpAM7AWuV9Wt1c40DHTeAwLsbNuKhK3bab/P74xMrTNxImRklP36J59AQcGJ51OnOrd69eAnPymxemx+PiQnw5QpZYZ86aWXaNKkCT/++CO9e/dmxIgRjBw5kgsuuIA//elPALzxxhv87ne/Y8OGDbzxxht89tlnREVFMX78eGbOnMmYMWM4fPgw3bp14xH3A6dLly488MADAIwePZq5c+dy9dVXc8sttzBt2jT69+/P/fffX5THiy++SHx8PCtWrODo0aMMGDCAIUOGnLSH/ve//50PPviAJUuWcMYZZ/DQQw+Rnp7Op59+SmxsLNOmTSs1xqpVq9i4cSNr165l586ddOnShVtvvbXcP8Vjjz3GxRdfzEsvvcSBAwfo06cPl1xyCQAZGRmsWrWK6OhoOnXqxF133UVMTAy33347S5cupV27duzbt4969epx0003MXPmTCZOnMhHH31Ejx49OOOMM8rddiAC2XM/Clysqj2AJGCoiPQD/gg8qaodgf3Abe76twH7VbUD8KS7ngnAitaQGwHz7hpNAfCfM/3OyNQ5ffpA8+ZOMQfnvnlz6Nu3yiGffvppevToQb9+/fj+++/ZtGkTzZo14+yzz2b58uXs3buXjRs3MmDAABYtWkR6ejq9e/cmKSmJRYsWsWXLFgAiIiIYMWJEUdwlS5bQt29fzjvvPBYvXsy6des4cOAA2dnZ9O/fH4Abb7yxaP2FCxcyY8YMkpKS6Nu3L3v37mXTpk0V5n/NNdcQ6x67KivG0qVLueGGG4iIiKBVq1ZcfPHFFcZduHAhTzzxBElJSQwaNIjc3Fy+++47AAYPHkx8fDwxMTF06dKFbdu2sXz5ci666KKiD6MmTZoAcOuttzJjxgzA+SC95ZZbKtx2ICrcc3c7j+W4T6PcmwIXA4W/+enAQ8BUYJj7GGAO8IyIiBZ+5zJluuobSGsL+1s2I60t3LQGHhmIsztvDJS7h13kF7+AadMgJgaOHYMRI+C550pd9ccKTmJKS0vjo48+YtmyZTRo0KCoiAFcf/31vPnmm5x77rkMHz4cEUFVGTt2LI8//niJWDExMUXj7Lm5uYwfP56VK1fSpk0bHnroIXJzcymvTKgqf/vb37jssssq/h0UU7ylcVkx5s+fX+Z0w8jISArcb0PF55qrKm+99RadOnU6af3PP/+c6OjooucRERHk5eWhqqVuo02bNiQkJLB48WI+//xzZs6cWamfrywBHV0QkQggHegAPAv8FzigqnnuKplAa/dxa+B7AFXNE5GDQFNgzykxxwHjABISEkhLS6vWD1KenJycoMSvatzJ50wusazxjt102vtHMq+6lsToRI5f+jM6vjCbVyPuJqtDyV34qmy3tv0e/Ixdl+LGx8eTn59PdnZ2QOvH/PADeuutHL/lFqJefhnJzCS3jPdWFDcrK4u4uDjy8/NJT09n+fLlHDlyhOzsbC699FL+8Ic/0KZNGx5++GGys7Pp168fo0aN4vbbb6dZs2bs27ePnJwczjzT+TdcuK0DBw6gqkRHR7Njxw7efPNNhg0bRmRkJA0bNmTRokX06dOHGTNmUFBQQHZ2NgMHDuRvf/sbvXv3Jioqik2bNtGqVasS/ehVlZycHKKjozl69ChRUVFF2y0rRu/evXnppZcYPnw4u3fvZsmSJQwfPpzs7GzatGnDJ598wpAhQ3j99deLfmcpKSn85S9/YfLkyYgIq1evpkePHuTm5nLs2LGibebl5XHkyBHOO+88xo8fz9q1a2nbti379u0r2ntPTU0lNTWVUaNGceTIkVL/Frm5uZX6txVQcVfVfCBJRBoD7wCdS1vNvS/t46/Ex7GqTgOmAfTq1UsLD1IEQ1paGsGIX9W4KQ+nlFh2zzLn/ueN/8WEoxdyR5PZZEXAnnlPM+nykjH0hsp/Eaptvwc/Y9eluBs2bCAiIiLwNgHvvQdAfQB3eKOsjiQVtR8YPnw406dPZ8CAAXTq1Il+/frRoEED4uLiiIuLo2vXrqxfv56UFOffdO/evfm///s/fvrTn1JQUEBUVBTPPvts0TaK348bN47+/fvTtm1b+vbtS3R0NHFxcbz88svcfvvtNGzYkEGDBnH66acTFxfHhAkTyMrKYuDAgagqzZo141//+leJ/EWERo0aERcXR3R0dFFcoMwYN954I8uWLaN///6cc845DBw4kNjYWOLi4njkkUe47bbbmDJlCn379i36Wzz66KNMnDiRAQMGoKq0bduWuXPnEhMTQ/369Yu2GRkZSYMGDWjXrh0vvPACY8aMoaCggObNm/Phhx8Czreg8ePHc8cdd5T594iJiaFnz57l/+2LK6vRe1k34EHgPpw98Uh32QXAAvfxAuAC93Gku56UFzPcLtbBQ5S4fdgO/aqZ83jya5OVh9DZndGshmjE/yu5fk3m61fcYMauS3HXr1+vhw4d8jyuqgYtbnVkZ2cXPX788cf17rvvrvEcxo4dq7Nnz66x7a1YsUIvvPDCctdZv359iWVU52IdItLM3WNHRGKBS4ANwBJgpLvaWOBd9/F77nPc1xe7SZgyxOXCwG0w95yTl8/sDgmH4ZIt/uRljB/mzZtHUlIS3bp145NPPuH3v/+93ykF1RNPPMGIESNKPU5RHYEMy7QEprvj7vWAN1V1roisB2aJyB+AVcCL7vovAv8Ukc3APmCUpxmHoEu3OPPaTy3u8zvC/hhIXQsLOvqTmzE17frrr+f666/3NYdXXnmlxrZ1//33nzTl0yuBzJZZA5QY6FHVLUCfUpbnAj/zJLswcdU3sC8GliWevPxYJMzuAjeuhQbH4Eh9f/IzxtQ9doaqz6QArtgEH3SA/FLOxp7ZHRodh2Ff13xuxpi6y4q7z3ptd8bVTx2SKfTJmfDdac7QjDHGBMqKu8+u+gbyxdlzL43Wg9fOg8s2wxmHazY3Y0zdFT4t0mqpKzfBf9rA/gZlrzOzO9z/GVy3Dp4rcZTDhJvSuopWh98dRqdMmcK4ceNo0KCc/wSm0mzP3UetDkHyDphXwUyYrxJgTXOnHYExoWbKlCllnpVpqs6Ku4+ucHselTXeXtyr3eGCTDjbOkUaH8yYMYPu3bvTo0cPRo8ezbZt2xg8eDDdu3dn8ODBRQ2zbr755qJWueC0+YUTZ+6OHDmSc889l9TUVFSVp59+mu3bt5OSklJ0lqvxhhV3H135DWyNh3XNK1739fOgAGdapDE1ad26dTz22GMsXryY1atX89RTTzFhwgTGjBnDmjVrSE1N5e67764wzqpVq5gyZQrr169ny5YtfPbZZ9x99920atWKJUuWsGTJkhr4acKHFXefRB93Tl6adw4BdX3MjIelZ7lDM3a+r6lBixcvZuTIkUU9xps0acKyZcuK2vGOHj2aTz/9tMI4ffr0ITExkXr16pGUlMTWrVuDmXbYs+Luk0FboeHxwIZkCs3sDp32QvL2oKVlTAlaRqva4gpfL94eV1U5duxY0TqltcE1wWPF3SdXboLDUbCkbeDvmdMFjkbYnHdTswYPHsybb77J3r17Adi3bx/9+/dn1qxZAMycOZMLL7wQcC5Jl56eDsC7777L8ePHK4wfFxcXcDtjEzibCukHdea3L2oHR8vqxVqKA7HOzJpRXwF5eRBGF/s1J3g5dTGQotq1a1d+97vfMXDgQCIiIujZsydPP/00t956K3/+859p1qwZL7/8MgC33347w4YNo0+fPgwePLhEr/XSjBs3jssvv5yWLVvauLuHrDr4oMtuaHcAHr+w8u+d2R1++jWweDEMGeJ5bsaUZuzYsYwdO/akZYsXLy6xXkJCAsuXLy96XtjpcNCgQSf1uX/mmWeKHt91111FF9c23rFhGR9c+Y1zP68S4+2F5nWEA9GAR5fiMsaEJivuPrjqG1jVArafVvn3Ho1yxt55+22wEz+MMWWw4l7T9u1jwPeVmyVzqpndgZwc+Pe/PUvL1G52vZvwVpW/vxX3mrZgARFaveL+8VlA69bw6quepWVqr5iYGA4ePGgFPkypKnv37iUmJqZS77MDqjVt7lx2NYAVraoeQusBN94ITz4Je/aAe3KJCU2JiYmsXr2anJwcz2Pn5uZWumiYmhcTE0NiYmLFKxZjxb0m5eXB++8zv6NboKsjNRX+/GeYPRt+8QtP0jO1U1RUFDk5OfTq1cvz2GlpafTsWeJCayYE2LBMTVq+HPbvr9aQTJHu3aFrV5s1Y4wplRX3mjR3LkRG8mF7D2KJwE03wWefwbffehDQGBNKrLjXpHnz4KKLOOTVEOcNNzj3r73mUUBjTKiw4l5Ttm6Fr76CK6/0LuZZZ8FPfuLMmrGZFMaYYqy415R585z7q67yNm5qKnz9Naxa5W1cY0ydZsW9psybBx07wjleHE0t5mc/g6goO7BqjDmJFfeacPiw0+jLyyGZQk2awBVXwOuvQ36+9/GNMXVShcVdRNqIyBIR2SAi60TkHnf5QyLyg4hkuLcrir3nf0Vks4hsFJHLgvkD1AmLFsHRo94PyRRKTYUdOyAtLTjxjTF1TiAnMeUBv1LVL0UkDkgXkQ/d155U1cnFVxaRLsAooCvQCvhIRM5R1fDdrZw3D+LinIOfwXDVVU78V1+FwYODsw1jTJ1S4Z67qu5Q1S/dx9nABqB1OW8ZBsxS1aOq+i2wGejjRbJ1kqozv33IEKhfPzjbiI2FkSPhrbfgxx+Dsw1jTJ0ilWlGJCJtgaVAN+Be4GbgELASZ+9+v4g8AyxX1Vfd97wIvK+qc06JNQ4YB5CQkJBceMmuYMjJyaFRo0a+xG20aRO9xo3j69/8hqyhQwFI35Fe7nsSoxPJPJpZ7jrJLZNPet44PZ2kSZNY9+CD7C52UYTK5lsVwYobzNgWN7hxTc1ISUlJV9VS+1IEXNxFpBHwMfCYqr4tIgnAHkCBR4GWqnqriDwLLDuluM9X1bfKit2rVy9duXJlpX6oykhLSzvpKjA1GvcPf4AHHnDGxBMSAJCHy7/Y8ORzJjPpm0nlrlPiUmv5+dCmDfTpA//6V9XzrYJgxQ1mbIsb3LimZohImcU9oNkyIhIFvAXMVNW3AVR1p6rmq2oB8AInhl4ygTbF3p4IbK9q8nXe3LnQu3dRYQ+aiAinU+T8+bBvX3C3ZYyp9QKZLSPAi8AGVf1rseUti602HPjKffweMEpEokWkHdAR+MK7lOuQXbvgiy+CN0vmVKmpcPy40ynSGBPWApktMwAYDawVkQx32W+BG0QkCWdYZitwB4CqrhORN4H1ODNt7gzbmTLvv+8cUK2p4p6UBJ07Oyc03XFHzWzTGFMrVVjcVfVToLRB4vnlvOcx4LFq5BUa5s6FVq2colsTRJy999//HrZtc3rPGGPCkp2hGizHjsGCBc5ZqVL+AVRP3Xijc2+dIo0Ja1bcg+XTTyE7OzgtB8rTrh0MGOAMzVinSGPClhX3YJk7F6Kj/TljNDUV1q2DNWtqftvGmFrBinuwzJ0LKSngxwki110HkZFOOwJjTFiy4h4M33wDmzbV/JBMoaZN4fLLrVOkMWHMinswFF6Yw6/iDs7QzA8/wNKl/uVgjPGNFfdgmDcPunZ1Dm765eqrnSEhu4iHMWHJirvXDh2Cjz/2d68doEEDGDHCOVs1N9ffXIwxNc6Ku9cWLoS8vJo7K7U8qanOh03hMJExJmxYcffavHlw+ulwwQV+ZwIXXwwtWtjQjDFhyIq7lwoKnOI+dKgzFdFvERFwww1OTvv3+52NMaYGWXH30ooVsHt37RiSKZSa6rRCmDOn4nWNMSHDiruX5s2DevWcPffa4vzzoVMnG5oxJsxYcffS3LnQvz80aeJ3JicUdor8+GOid+3yOxtjTA2x4u6VH36AVatq15BModRUAJLuuQeysnxOxhhTE6y4e2W+296+Nhb3s8+GhARisrLgkUf8zsYYUwOsuHtl7lzn4hhduvidycliY52hmZ07nSuuTJ3qPI+N9TszY0wQ1YL5eiEgNxc++ghuuaVmL8xxCnm45LZb3AmTF8Dwr6FBHhyvB290hUlDctlZyvr6oPWANyYU2J67F9LS4MgR/1sOlCIrDg5FQ3Q+FIgQWQA59WFnnN+ZGWOCyYq7F+bOdXq5pKT4nUmpmh+GvydD2k3XIED3nX5nZIwJNivu1aXqFPdLLoGYGL+zKdXIUTDhKlh9aX92NYAfTvM7I2NMsFlxr67162Hbtlo5JHOqgsgIZnaHazbC6Uf8zsYYE0xW3Ktr7lznvg4Ud4DpPZzx91Ff+Z2JMSaYrLhX19y50LMntG7tdyYBWd0SMhLg5gy/MzHGBFOFxV1E2ojIEhHZICLrROQed3kTEflQRDa596e7y0VEnhaRzSKyRkTOD/YP4ZfIQ4fgP/+pM3vthaYnQZ/t0Nm6ERgTsgLZc88DfqWqnYF+wJ0i0gW4H1ikqh2BRe5zgMuBju5tHDDV86xriSZffOG0+a2NZ6WWY+Z5znz3sav9zsQYEywVFndV3aGqX7qPs4ENQGtgGDDdXW06cK37eBgwQx3LgcYi0tLzzP22YwcdnnkGmjaF3r39zqZSdjeC9zvATWugXoHf2RhjgkFUAz8jUUTaAkuBbsB3qtq42Gv7VfV0EZkLPKGqn7rLFwG/UdWVp8Qah7NnT0JCQvKsWbOq+aOULScnh0aNGnkas+Nf/0qrf/+bI23asGLGjEq9N31HermvJ0Ynknk0s9x1klsmVytuhy/WMmzKdOb85na29ehUbtyKBOP3G+zYFje4cU3NSElJSVfVXqW9FnD7ARFpBLwFTFTVQ1L2afalvVDiE0RVpwHTAHr16qWDBg0KNJVKS0tLw7P4sbEnXXC64fffMyglxZnj/uOPAYVIebj8k50mnzOZSd9MKncdvaHkh3Jl4tZvBNtj4fi8F5hUrM1MaXEr4unvt4ZiW9zgxjX+C6i4i0gUTmGfqapvu4t3ikhLVd3hDrsUHp7LBNoUe3sisN2rhGtKaX1a4ESvlp+th/oFcCQS3u5c93q1HIuE17vBbasg/kc4aH3EjAkpgcyWEeBFYIOq/rXYS+8BY93HY4F3iy0f486a6QccVNUdHubsq8JeLVEFzteR6HzneV3s1fJKEsTmwXXr/M7EGOO1QGbLDABGAxeLSIZ7uwJ4ArhURDYBl7rPAeYDW4DNwAvAeO/T9lfzw5BdH7Z278TfkyEhx++Mqia9FaxrZrNmjAlFFQ7LuAdGyxpgH1zK+grcWc28arXbr4F9f4KMLu2ZcO5Gv9OpOnH23v/8IXTcA5vO8DshY4xX7AzVKujhdlXcdVbdOCu1PK92h3yBMbb3bkxIseJeBUnuZUh3t23lbyIeyIqDhe2d4i42592YkGHFvQqSsmB7IzgSXwePopbilSQ48xCkbPU7E2OMV6y4V0HPHZDRwu8svPNuJzgQDWOtmZgxIcOKeyVFH4fOe0KruB+NglndYMQGIDvb73SMMR6w4l5JXXY7c9xXhVi3nOlJ0PA4MGeO36kYYzxgxb2SCg+mhtKeO8DyRNjYFHjlFb9TMcZ4wIp7JfXMgpwo+O/pfmfiMXGu0sTSpfDtt35nY4ypJivulZSUBatbgIbgb+6fPQARqGSXS2NM7ROCJSp4pAB6ZMGqEBuSKZQZDwweDNOnOxchMcbUWVbcK6HdATjtWOiNt59k7FhnWObTT/3OxBhTDVbcK6Gn29sypIv78OEQF2cHVo2p46y4V0JSFuQJfNXc70yCqGFD+NnPYPZsOHzY72yMMVVkxb0SkrJgQzPnpJ+QNnYs5OTAO+/4nYkxpoqsuFdCz6wQH5IpdOGFcPbZNjRjTB1mxT1AzXKgdXaYFPd69WDMGFi8GL77zu9sjDFVYMU9QIU93EN1GmQJY8aAKvzzn35nYoypAivuASpsO7A6XIp7u3YwcKAz511r50W+jTFls+IeoJ474LvTYF8DvzOpQTffDJs2wbJlfmdijKkkK+4BSgqXg6nFjRgBDRo4e+/GmDrFinsAYo9Bp72h1+a3QnFxToF/4w348Ue/szHGVIIV9wB02wURGoZ77uAMzRw8CO++63cmxphKsOIegJ4h2sM9IIMGwZln2tCMMXWMFfcAJGU51xjd2tjvTHxQOOd94ULYvt3vbIwxAbLiHoCig6nidyY+GTPGaQH86qt+Z2KMCVCFxV1EXhKRXSLyVbFlD4nIDyKS4d6uKPba/4rIZhHZKCKXBSvxmlKvALrvDNMhmUIdO8KAAU47ApvzbkydEMie+yvA0FKWP6mqSe5tPoCIdAFGAV3d9zwnIhFeJeuHjnudC0eHdXEHp5nYhg2wcqXfmRhjAlBhcVfVpcC+AOMNA2ap6lFV/RbYDPSpRn6+C9ULYlfadddBTIwdWDWmjhAN4Gu2iLQF5qpqN/f5Q8DNwCFgJfArVd0vIs8Ay1X1VXe9F4H3VXVOKTHHAeMAEhISkmfNmuXBj1O6nJwcGjVqVKn3pO9IB+Anr88jef5Snn75MQoiI09aJzE6kcyjmeXGSW6ZXGbsstS2uIU6P/ooTVas4D9z5qD16xctr8rvN1DBim1xgxvX1IyUlJR0Ve1V2muRpS0MwFTgUUDd+78At1L6IcdSPz1UdRowDaBXr146aNCgKqZSsbS0NCobP+XhFADeXw9rmsG9W+4vsc7kcyYz6ZtJ5cbRG0r++IWxy1Lb4hY5ehSGDmVgdrZzcpOrKr/fQAUrtsUNblzjvyrNllHVnaqar6oFwAucGHrJBNoUWzURqLvz5zSMergH4pJLoFUr6/NuTB1QpeIuIsVPxB8OFM6keQ8YJSLRItIO6Ah8Ub0U/dMiBxIOW3EvEhEBo0fD++/Dzp1+Z2OMKUcgUyFfB5YBnUQkU0RuA/4kImtFZA2QAvwSQFXXAW8C64EPgDtVNT9o2QdZ4cHUsOnhHoixYyE/H2bO9DsTY0w5KhxzV9UbSln8YjnrPwY8Vp2kaoueO5z7sOnhHojOnaFPH2fWzL33+p2NMaYMdoZqOZKyYEtjOBTjdya1zNixsGYNZGT4nYkxpgxW3MuRlBWGbX4DMWoU1K9vB1aNqcWsuJeh0VHosM8OppaqSRO45hpn3P3YMb+zMcaUwop7GbrvdH45VtzLcPPNsGePM3PGGFPrWHEvg7UdqMBll0FCgrUjMKaWsuJehqQs2BMLmaf5nUktFRkJqakwdy5RBw/6nY0x5hRW3MsQ9j3cAzF2LBw/Ts/x4yEry+9sjDHFWHEvzfHjnBfuPdwD0b07NG1K7Pbt8MgjfmdjjCnGintpNm4kJt+Ke7liY0EE9u51vtxMneo8j431OzNjDFXvChna3JNzbI67Qx4uOTbV4k6YvACGfw0N8iBPYFY3mDQkl52lrK8P2hWcjKlJtudemowMciNgY1O/E6m9suLgUDRE50NBPSFC4VgE7IzzOzNjDFhxL92qVaxNgPw6fYHA4Gt+GP6eDO/dMwYB+pZ/HRBjTA2yYZlTqUJGBhlt/U6k9hs5yrmffM55vN/BmWEUlQfH7V+VMb6zPfdTZWbCvn3W5reSpvSDljlw3Tq/MzHGgBX3klatAmymTGUtbA/rz4BfLqeMCysaY2qSFfdTZWSACGsT/E6kjhF4qh8k74AB3/mdjDHGivupMjKgQwdyov1OpO75Z3fYFwMTl/udiTHGivupMjKgZ0+/s6iTfqwPz/dy5r6ftd/vbIwJb1bciztwAL79FpKS/M6kznq2tzPkPqHOXhbdmNBgxb241audeyvuVfZDPMzuCj//0rngiTHGH1bciyu8JqgNy1TLlH7Q+CiMtUusGl/oiEMAABYYSURBVOMbK+7FZWQ4F6BoYfMgq+OLRFiWCPd8DlLgdzbGhCcr7sWtWmVDMh6Z0g867oMrNvmdiTHhyYp7oWPHYP16K+4eebszfH+aTYs0xi8VFncReUlEdonIV8WWNRGRD0Vkk3t/urtcRORpEdksImtE5PxgJu+p9evh+HEbb/dIXgQ80wcu+Ra67fQ7G2PCTyB77q8AQ09Zdj+wSFU7Aovc5wCXAx3d2zhgqjdp1oDCg6m25+6ZF86HI5Fwj+29G1PjKizuqroU2HfK4mFA4WXvpwPXFls+Qx3LgcYiUjcuebFqFTRoAB06+J1JyNjfAKYnwU1rgN27/U7HmLBS1TH3BFXdAeDeN3eXtwa+L7Zeprus9svIcK4JGmFN3L30dF+IyQeef97vVIwJK6JacQs/EWkLzFXVbu7zA6rauNjr+1X1dBGZBzyuqp+6yxcBv1bV9FJijsMZuiEhISF51qxZHvw4pcvJyaFRo0Zlr6DKhVdfzc7Bg9n0y18CkL6jRMolJEYnknm0/CtUJLdMLrGsotihFvenf3yBVt/vZvmsWWhUVLmxTlXh366KLG5w45qakZKSkq6qvUp7raqXVdgpIi1VdYc77LLLXZ4JtCm2XiKwvbQAqjoNmAbQq1cvHTRoUBVTqVhaWhrlxv/2Wzh8mNZXXklrd72Uh1MqjDv5nMlM+mZSuevoDSU/PCuKHWpxF54HC1bDwJ074aabyo11qgr/dlVkcYMb1/ivqsMy7wFj3cdjgXeLLR/jzprpBxwsHL6p1dwe7nYwNTgWtgfOPReefNK50pUxJugCmQr5OrAM6CQimSJyG/AEcKmIbAIudZ8DzAe2AJuBF4DxQcnaaxkZUK8edOvmdyahSYB77oEvv4TPPvM7G2PCQoXDMqp6QxkvDS5lXQXurG5SNS4jw9mzbNDA70xC1+jR8NvfwpQpcOGFfmdjTMizM1TB2g7UhIYNYdw4eOcd2LrV72yMCXlW3PfscS6KbcU9+O68E0TgmWf8zsSYkGfF3Xq415w2bWDkSPjHPyA72+9sjAlpVtyt7UDNmjgRDh6E6dMrXtcYU2VW3FetgtatoVkzvzMJD/36Qd++8NRTUGDN3o0JFivuGRm2117TJk6EzZth/ny/MzEmZIV3cf/xR/j6a2vzW9NGjHC+LU2Z4ncmxoSs8C7u69ZBfr7tude0qCiYMAEWLYK1a/3OxpiQFN7F3doO+Of22yE21hl7N8Z4LryLe0YGxMVBu3Z+ZxJ+mjaFMWPg1Vet17sxQWDFPSnJ6Stjat7dd8PRozBtmt+ZGBNywreqFRQ4JzDZkIx/unSByy6DZ591LlBujPFM+Bb3zZvh8GEr7n6bOBF27IDZs/3OxJiQEr7F3c5MrR2GDHE6ck6ZYr3ejfFQeBf3yEjo2tXvTMJbvXpOr/eVK+E///E7G2NCRvgW91WrnDHf6Gi/MzGjR8Ppp9tJTcZ4KHyLu7UdqD0Ke72//TZs2+Z3NsaEhPAs7llZzs2Ke+1hvd6N8VR4FvfCHu7WU6b2KOz1/sILkJPjdzbG1HnhWdwL2w706OFvHuZk1uvdGM+EZ3HPyICzznIO4pnaw3q9G+OZ8C3uNt5eO02cCJs2wfvv+52JMXVa+BX3w4fhm29svL22sl7vxngi/Ir7mjXOmZC25147FfZ6/+gjWLKEpHvucWY2GWMqJfyKu7UdqP0Ke72PH0/82rXwyCN+Z2RMnVOt4i4iW0VkrYhkiMhKd1kTEflQRDa597XrqGVGhnMg9cwz/c7ElCUxsegSiKIKU6c6c+BjY/3OzJg6I9KDGCmquqfY8/uBRar6hIjc7z7/jQfb8UbhwVQRvzMJe/Jw6X+DFnfC8/+Gq78BAY7Wg9ldYdKQXHae8h590JqNGVOaYAzLDAMKJypPB64NwjaqJi/PGXO3IZlaLSsOfoiDAqBAhOgCGLgVovP9zsyYukO0Gm1WReRbYD+gwPOqOk1EDqhq42Lr7FfVEkMzIjIOGAeQkJCQPGvWrCrnUZGcnBwaNWpEg61b6XPLLWy4/352XnZZue9J35FeYdzE6EQyj2aWu05yy+RKx7a4cM2Tr3A4/jR2Dr2Gzv+YRutvtpJfP4q01KtZe3Hfom9epcUNROG/Ca9ZXFOTUlJS0lW1V2mvVbe4t1LV7SLSHPgQuAt4L5DiXlyvXr105cqVVc6jImlpaQwaNAheew1SU5299/POK/c9ZQ0ZFDf5nMlM+mZSueuUNmxQUWyLWzL2Wfvhxfdg8Lew8Gy4bRhkxld9WKbo34THLK6pSSJSZnGv1rCMqm5373cB7wB9gJ0i0tLdcEtgV3W24amMDKhf37k4hKlTtp0Ol46GX1wJ/b+Hr56DW7/ELvBhTBmqXNxFpKGIxBU+BoYAXwHvAWPd1cYC71Y3Sc+sWgXdujlzqU2do/Xg772h+y9gVQtnT54rroDM8oeEjAlH1dlzTwA+FZHVwBfAPFX9AHgCuFRENgGXus/9p2ptB0LEt03g4rEw4XJg6VLnA3v6dNuLN6aYKhd3Vd2iqj3cW1dVfcxdvldVB6tqR/d+n3fpVsP27bBnj7UdCBFaD57ti3P8pHt3uPlmuPpq5+9sjAmjM1QL2/zanntoad8e0tKcXjSLFzvXxP3nP20v3oS98CnuhW0Hunf3Nw/jvcKLbK9e7RT3MWPg2mutJ40Ja+FV3Nu3h9NO8zsTEywdO8LHH8Nf/gILFzqF/vXXbS/ehKXwKu423h76IiLg3nudv/c558CNNzpthHfu9DszY2pUWBT3iJwc+O9/bbw9nHTqBJ9+Cn/6E8yf7+zFv/mm89qOHdZK2IS8sCjujbZscR5YcQ8vERFw333OwfT27eH66+G66+C3v7VWwibkhUdx37zZeWDFPTx17gyffQaRkTB7NrzyirUSNiHPi5a/tV6jzZuhWTNo1crvVEwNKLOV8D3wwnswdDNEusdY84HlZ+Qy9xJhQXvIaOHMobdWwqauC5/ibj3cw15WHHx/mtMjPi8qknrH81jdAhoeh8cXObfdDeDDs4GzXoEhQ2yHwNRZoV/ct21zivsll/idiakFmh+GvydD3Ii7yX7rr7TIgZGjICEbLtkCQ/7r3LjlFucN3bo5RX7IELjoIhvCMXVG6Bf3++5zxlc3bfI7E1MLjBzl3E8+qxWTrjqxfGcczOzh3FDQ4RnOXPmFC+HZZ+Gvf4XoaKfADxkCl13mFP7i3wYLZ+EsWAAtWtToz2XMqUL3gGpsrPMfb/Zs5/lHH9nBMxMYAXr0cGbafPgh7NsH778P48c7vWvuu88507lVKxg7FmbOhF274NFHbRaOqTXq/J57edfhnLwArlsPUQVwOBLe6Vz6dTjBDqCZcjRoAEOHOjeAH35wiv6CBTBvHsyYUbSqgDMLZ+pUiIlxLvRtjA/qfHEvS1YcHIqGegp5kZHE5OdxKNr5+m1MVZTYkTgX5By4dAs88SF03wUR7j7CcYGPW+by78uFhe3h6zMAsZ0IU3NCtrhD6QfPjPGS1oOFHeDaDU5xL5yFs6EZtDkET33grPfdabCwPdD5TRg8GJo29TVvE/pCuriXdfDMGK+VNQvnrP3Onv2Q/8KIDThnyYpA794nZuH06+fP1cHsAHBIC90DqsbUoJGjYMJVsPusVky46sSOxbbT4R/JTteDZvcBy5bBQw85xfzxx53ZN02bwrBh8NxzsHlzyS6WweqF8/DDdgA4hIX0nrsxtUl+BM5eer9+8MADcOAALFniTLdcsADee89ZsV27E9MtL7745Fk4zz3nrKMKhw87MQ4cgIMHA3+8cWNRTicdAK5f37laWZwdmAoFVtyN8UvjxjB8uHMDp3PpggVOsX/tNXj++aJVTyrC4DRFy88vN3xuBByIcW4HY048PnYeJO+ADvugfgEU4H6FP3aMgtNOY11z+KI1fN7auc945rjTl8fUKfYXM8ZnJWbh9ITI7nDFN/CHxdBljzMLJ0/gu3hIawvb4/JPKtgHo2HhXZ87Hxjx8RAfT+wfyz6n47l/Q6e9zgFgycvjxR4wpyv0+QH6ZsKwr+E298qU/DMekpOhTx/o29e5tWlTdjuPHTtg1Ch44w0by/eRFXdjaqG8CHivs9PkrMueE0X4gw5wZ1mTA/r0CTh+aQeAP+jo3ABQOHu/U+hfS/w5fPEFPPOMc5UrgISEE4W+Tx/nAHF8vPPao486vfSLDyOZGmfF3ZhaLFjTeSucSSawpYlze+3Bp5xlx44516n94gv4/HPnVnicAJw9+eIHgwuHkaKjnbH8hg2r17zPvhFUihV3Y2qxWjWdt3596N0bmd8H2gPtofGP0PsH6PsD/GSbctE2iDn1UMDRoxAXR56cPPaffG7KiWGk4vdlLXv44eB8IwjRDw0r7saYKjsQCx92cG7gjOWPS3eGlaLynfbJc7pC41znFp974jHHjzsN/Qpn8mRnB7bRwm8EInD55SU/EMr7oIiJKfntIVjDSD5/aFhxN8Z4pvlh+HsvmNYLxq2EFjnOPP/S6IOfnPQ84kHhtKMlPwjOOgCj10D3nc7snjyB7XGwLV5psGr+SetGVtTdoX79E4X+1HMKCj80IiPhpZdK/3CIi4N6AZ4e5POxh6AVdxEZCjwFRAD/UNUngrUtY0ztUDiMBM5JXZVRUM/5JnCglEk+5+6BnlnwYyTUz4e555RyYFmhwXE4PD7zpHn9N7x0ZbEPjGM0zt1F/NFdJJwF5+2Epj86U0EVd8ppXh6MGVN6kiJFs5HK/Ibw+ONOjEI+NZILSnEXkQjgWeBSIBNYISLvqer6YGzPGBPaCg8sF/9GUILAkfpA69bOzTVrRdlxn/s3jPsSfoxwPjReOB/+38UnvgnEF/sm8eLAv5z0ofHu5zOIPwCNN55YLz635Gn/fnWkDdaeex9gs6puARCRWcAwwIq7MabSqvONoDylfWjsaejcTvXivfee9Pzah2eUWEcKoNExeHYepK6FYxHOAWY/OtKKntrHwougIiOBoar6c/f5aKCvqk4ots44YJz7tBOwsUQg75wB7LG4dS5uMGNb3ODGDWsdof1xOL4bdjeDZlEQtQn+G4RNnaWqzUp7IVh77qVNZj3pU0RVpwHTgrT9k5MRWamqvSxu3YobzNgWN7hxjf+C1RUyE2hT7HkisD1I2zLGGHOKYBX3FUBHEWknIvWBUcB7FbzHGGOMR4IyLKOqeSIyAViAMxXyJVVdF4xtBShYwz8WN7hxgxnb4gY3rvFZUA6oGmOM8ZddickYY0KQFXdjjAlBIV3cReQlEdklIl95HLeNiCwRkQ0isk5E7vEoboyIfCEiq924D3sRt1j8CBFZJSJzPYy5VUTWikiGiKz0MG5jEZkjIl+7v+cLPIjZyc2z8HZIRCZ6lO8v3b/ZVyLyuojEeBT3HjfmuurmWtr/BxFpIiIfisgm9/706mdtaoOQLu7AK8DQIMTNA36lqp2BfsCdItLFg7hHgYtVtQeQBAwVkX4exC10D7DBw3iFUlQ1yeP50k8BH6jquUAPPMhbVTe6eSYBycAR4J3qxhWR1sDdQC9V7YYziWBU+e8KKG434HacM757AFeJSMfy31WuVyj5/+F+YJGqdgQWuc9NCAjp4q6qS4F9QYi7Q1W/dB9n4xSe1uW/K6C4qqqFXTOi3JsnR7xFJBG4EviHF/GCSUROAy4CXgRQ1WOqesDjzQwG/quq2zyKFwnEikgk0ABvzuvoDCxX1SOqmgd8DAyvarAy/j8MA6a7j6cD11Y1vqldQrq41wQRaQv0BD73KF6EiGQAu4APVdWTuMAU4Nc410P2kgILRSTdbSnhhbOB3cDL7jDSP0SklG4f1TIKeN2LQKr6AzAZ+A7YARxU1YUehP4KuEhEmopIA+AKTj450AsJqroDnJ0WoLnH8Y1PrLhXg4g0At4CJqrqIS9iqmq+O2yQCPRxv5pXi4hcBexS1fRqJ1jSAFU9H7gcZ3jqIg9iRgLnA1NVtSdwGA+HC9wT664BZnsU73ScPeB2QCugoYjcVN24qroB+CPwIfABsBpnSNCYCllxryIRicIp7DNV9W2v47vDEGl4c8xgAHCNiGwFZgEXi8irHsRFVbe797twxq8Dv0pz2TKBzGLfWubgFHuvXA58qao7PYp3CfCtqu5W1ePA20B/LwKr6ouqer6qXoQzpLLJi7jF7BSRlgDu/S6P4xufWHGvAhERnPHgDar6Vw/jNhORxu7jWJyi8XV146rq/6pqoqq2xRmOWKyq1d6zFJGGIhJX+BgYgjOUUC2qmgV8LyKd3EWD8bZd9A14NCTj+g7oJyIN3H8bg/HowLWINHfvzwR+ird5g9MWZKz7eCzwrsfxjU9C+jJ7IvI6MAg4Q0QygQdV9UUPQg8ARgNr3fFxgN+q6vxqxm0JTHcvdlIPeFNVPZu2GAQJwDtOPSMSeE1VP/Ao9l3ATHcIZQtwixdB3bHrS4E7vIgHoKqfi8gc4EucYZNVeHda/1si0hQ4DtypqvurGqi0/w/AE8CbInIbzofUz6qfsqkNrP2AMcaEIBuWMcaYEGTF3RhjQpAVd2OMCUFW3I0xJgRZcTfGmBBkxd3UeiLyZPGOiCKyQET+Uez5X0Tk3mrEf0hEJgW63Csi0lZEbiz2/GYReSZY2zPhxYq7qQv+g3vGp4jUA84AuhZ7vT/wWSCB3HMIaou2wI0VrWRMVVhxN3XBZ5w4nb8rzlmw2SJyuohE43RPXCWOP7v9z9eKyPUAIjLI7b//GrDWXfY7EdkoIh8BnUpusmwicpM4ffczROT5wg8MEckRkcfE6ce/XEQS3OXt3ecrROQRESns/PkE8BM3zi/dZa1E5AO3v/qfqvwbM2HPirup9dz+NXnuKfj9gWU4XTgvAHoBa1T1GM7p+Uk4vc8vAf5c2DcFp+fN71S1i4gk47Rh6Om+p3eguYhIZ+B6nIZpSUA+kOq+3BCnRW8PYClOL3ZwetM/paq9ObkV8P3AJ26P+SfdZUlu/POA60XE6y6QJkxYcTd1ReHee2FxX1bs+X/cdS4EXnc7a+7E6X9eWLi/UNVv3cc/Ad5x+6QfwumvEqjBOBf6WOG2nhiM06IY4BhQ2C4iHWfYBZwPocIOlK9VEH+Rqh5U1VycfjpnVSI3Y4qEdG8ZE1IKx93PwxmW+R74FXAIeMldR8p5/+FTnle174YA01X1f0t57bie6OeRT9X+fx0t9riqMYyxPXdTZ3wGXAXsc/fM9wGNcfaKl7nrLMUZyogQkWY4V3P6opRYS4HhIhLrdrW8uhJ5LAJGFuvW2EREKtq7Xg6McB8Xv/xeNhBXiW0bEzAr7qauWIszS2b5KcsOquoe9/k7wBqci1osBn7ttg8+iXuJxDeADJye/J+Us93fi0hm4U1V1wO/x7n61BqcC2m0LOf9ABOBe0XkC3fdg+7yNTjHElYXO6BqjCesK6QxQea2Gf5RVVVERgE3qOowv/Myoc3G84wJvmTgGfdCHgeAW33Ox4QB23M3xpgQZGPuxhgTgqy4G2NMCLLibowxIciKuzHGhCAr7sYYE4L+PxJLzELwc0fnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(xA1, xB1, color='green')\n",
    "plt.plot(xA1, xB1, marker='*' , color='red')\n",
    "plt.legend()\n",
    "plt.xlabel('Word Length')\n",
    "plt.xticks(range(1,11))\n",
    "plt.grid(True)\n",
    "plt.legend(['average frequency', 'count'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# English document version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "linesDF_E = spark.read \\\n",
    "        .format('csv') \\\n",
    "        .option('encoding', 'EUC-kr') \\\n",
    "        .option('header', 'False') \\\n",
    "        .option('sep', '\\n') \\\n",
    "        .load('t_bbe.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(_c0='field,field,field,field,field'),\n",
       " Row(_c0='\"01001001\",\"1\",\"1\",\"1\",\"At the first God made the heaven and the earth.\"'),\n",
       " Row(_c0='\"01001002\",\"1\",\"1\",\"2\",\"And the earth was waste and without form; and it was dark on the face of the deep: and the Spirit of God was moving on the face of the waters.\"'),\n",
       " Row(_c0='\"01001003\",\"1\",\"1\",\"3\",\"And God said, Let there be light: and there was light.\"'),\n",
       " Row(_c0='\"01001004\",\"1\",\"1\",\"4\",\"And God, looking on the light, saw that it was good: and God made a division between the light and the dark,\"')]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linesDF_E.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformation lines to RDD\n",
    "linesRDD_E = linesDF_E.rdd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "wcRDD_E = linesRDD_E.map(lambda line: line[0]) \\\n",
    "        .flatMap(lambda line : re.split('\\\\W+', line)) \\\n",
    "        .filter(lambda word : len(word)>0) \\\n",
    "        .map(lambda word : (word,1)) \\\n",
    "        .reduceByKey(lambda v, w : v+w) \\\n",
    "        .sortBy(lambda vw : vw[1], False) \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformation RDD to DF\n",
    "wcDF_E = spark.createDataFrame(wcRDD_E)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _1: string (nullable = true)\n",
      " |-- _2: long (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDF_E.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-----+\n",
      "| _1|   _2|\n",
      "+---+-----+\n",
      "|the|63800|\n",
      "| of|37714|\n",
      "|and|33405|\n",
      "+---+-----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDF_E.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "wcDF_E_named = wcDF_E.withColumnRenamed('_1', 'words')\\\n",
    "                 .withColumnRenamed('_2', 'freq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+\n",
      "|words| freq|\n",
      "+-----+-----+\n",
      "|  the|63800|\n",
      "|   of|37714|\n",
      "|  and|33405|\n",
      "+-----+-----+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDF_E_named.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "wcDF_E_length = wcDF_E_named.withColumn('wordLen', F.length('words'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+-------+\n",
      "|words| freq|wordLen|\n",
      "+-----+-----+-------+\n",
      "|  the|63800|      3|\n",
      "|   of|37714|      2|\n",
      "|  and|33405|      3|\n",
      "+-----+-----+-------+\n",
      "only showing top 3 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDF_E_length.show(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+-------+\n",
      "|words| freq|wordLen|\n",
      "+-----+-----+-------+\n",
      "|  the|63800|      3|\n",
      "|   of|37714|      2|\n",
      "|  and|33405|      3|\n",
      "|   to|26327|      2|\n",
      "|   in|14782|      2|\n",
      "|  you|12231|      3|\n",
      "|  And|11577|      3|\n",
      "| will|10596|      4|\n",
      "|    a|10436|      1|\n",
      "|  for|10327|      3|\n",
      "|   is|10127|      2|\n",
      "|  his| 9257|      3|\n",
      "|   be| 8972|      2|\n",
      "|   he| 8500|      2|\n",
      "|    I| 7811|      1|\n",
      "| your| 7655|      4|\n",
      "| Lord| 7633|      4|\n",
      "|   on| 6890|      2|\n",
      "| have| 6890|      4|\n",
      "|  him| 6119|      3|\n",
      "+-----+-----+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wcDF_E_length.filter(wcDF_E_length.wordLen >= 1).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plus argument 'F.sum'\n",
    "wordLengthFrequency_E = wcDF_E_length\\\n",
    "                     .groupby(F.col('wordLen'))\\\n",
    "                     .agg(F.count(F.col('freq')), F.avg(F.col('freq')), F.sum(F.col('freq')))\\\n",
    "                     .orderBy(F.col('wordLen'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------+------------------+---------+\n",
      "|wordLen|count(freq)|         avg(freq)|sum(freq)|\n",
      "+-------+-----------+------------------+---------+\n",
      "|      1|         15|            3467.0|    52005|\n",
      "|      2|        169|1363.4674556213017|   230426|\n",
      "|      3|        351| 717.4843304843305|   251837|\n",
      "|      4|        817|232.86658506731945|   190252|\n",
      "|      5|       1265| 72.68695652173913|    91949|\n",
      "|      6|       1358| 37.24889543446245|    50584|\n",
      "|      7|       1144|  32.5979020979021|    37292|\n",
      "|      8|      31874| 1.604787601179645|    51151|\n",
      "|      9|        521|18.927063339731287|     9861|\n",
      "|     10|        271|14.118081180811808|     3826|\n",
      "|     11|        116|18.594827586206897|     2157|\n",
      "|     12|         39| 9.128205128205128|      356|\n",
      "|     13|         10|              41.7|      417|\n",
      "|     14|          3|32.333333333333336|       97|\n",
      "|     15|          3|              11.0|       33|\n",
      "|     17|          1|               1.0|        1|\n",
      "+-------+-----------+------------------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "wordLengthFrequency_E.show(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = wordLengthFrequency_E.select(F.col('wordLen')).collect()\n",
    "b = wordLengthFrequency_E.select(F.col('count(freq)')).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = np.array(a)\n",
    "B = np.array(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "A1 = np.ravel(xA)\n",
    "B1 = np.ravel(xB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEGCAYAAACevtWaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfXgV9Zn/8fdNoKCEgijNggFBzQ95DgQQHyrEIFLhJ6KuYkGxdottsdquv7a4dtfQrZd2pStSbde2KKgoRW0Xr4qKQlysgkJE5EkWRcCoPD9IxMjT/ftjJjGQk+QkOSeTnHxe1zXXmZnznfvc5yS5z+Q7M98xd0dERFJLs6gTEBGRxFNxFxFJQSruIiIpSMVdRCQFqbiLiKSg5lEnAHDaaad5165dkxb/888/p3Xr1orbyOImM7biJjeu1I/CwsJd7t4h5pPuHvmUk5PjyVRQUKC4jTBuMmMrbnLjSv0AVngldVXdMiIiKUjFXUQkBam4i4ikoAZxQFVEKnf48GHS09NZv359wmO3bds2KXElsVq1akVmZiYtWrSIexsVd5EGrqioiIyMDDIzMzGzhMY+cOAAbdq0SWhMSSx3Z/fu3RQVFdGtW7e4t1O3jEgDV1JSQtu2bRNe2KVxMDNOPfVUSkpKarSdinsDlMxz/qVxUmFv2mrz81dxb4BU3EWkrlTcRSSlzJgxgx49ejB+/PioU4mUinsEzCpO+fmx2+bnx24v0pgdPXo0abF/97vfsWDBAubMmXPc+iNHjiTtNRsiFfcGYupU+NrXYP/+YPnNN4MiPnVqtHmJXHHFFeTk5NCrVy/+8Ic/APD73/+en/3sZ2VtZs2axY9+9CMAnnjiCQYPHkx2djY333xzWSFPT0/n3/7t3zj33HNZunQpv/zlLxk0aBC9e/dm0qRJeHhXuOXLl9O3b1/OO+88fvrTn9K7d28g+EL46U9/yqBBg+jbty8PP/xwhVy///3vs2nTJi6//HLuv/9+8vPzmTRpEiNGjOCGG26oNIa7c8stt9CzZ09GjRrFZZddxjPPPAME3aS7du0CYMWKFQwbNgwIxuW56aabGDRoEP3792f+/Plln8WVV17JyJEjycrKOu5zevHFFxkwYAD9+vUjLy+PY8eOkZWVxc6dOwE4duwYZ599dtnr1Ull4xKUTkAr4C1gFbAWmBqu7wa8CWwE/gx8LVzfMlx+P3y+a3Wv0dTGloHYU27u8e3OPrvytvWZb1Rxkxm7McVdt26df/bZZwmP6+5++PDhatvs3r3b3d0PHjzovXr18l27dvmOHTv8rLPOKmszcuRIf+2113zdunU+evRoP3TokLu7/+AHP/DZs2e7uzvgf/7znyvEdXefMGGCP/fcc+7u3qtXL3/99dfd3f3nP/+59+rVy93dH374Yf/3f/93d3cvKSnxnJwc37RpU4V8zzjjDN+5c6e7u991110+YMAAP3jwYJUxnn32WR8+fLgfOXLEP/74Y2/btq0//fTTFeItX77chw4d6u7ud9xxhz/++OPu7r53717Pysry4uJif/TRR71bt26+b98+/+KLL7xLly6+detW37Fjh2dmZpblXPr+8/Pz/f7773d395deesmvvPLKmD+HdevWVVhHHceW+RK42N37AdnASDMbAvwauN/ds4C9wHfD9t8F9rr72cD9YTuJw6hRUFICK1cWcewYfPvbUWckEvRh9+vXjyFDhvDRRx+xceNGOnTowJlnnsmyZcvYvXs3GzZs4IILLmDRokUUFhYyaNAgsrOzWbRoEZs2bQIgLS2Nq666qixuQUEB5557Ln369GHx4sWsXbuWffv2ceDAAc4//3wAvl3uj2DhwoU89thjZGdnc+6557J79242btxYbf6XX345J510UpUxlixZwnXXXUdaWhqdOnXi4osvrjbuwoULuffee8nOzmbYsGGUlJSwdetWAPLy8mjbti2tWrWiZ8+ebNmyhWXLlnHRRReVnavevn17AG666SYee+wxAB555BG+853vVPva8aj2Iqbw26E4XGwRTg5cDJR+8rOBfOD3wJhwHuAZ4EEzszCOVGH0aHj1VVi79n327s1kwgT45S+jzkpS2RdffFHlRUyvvvoqr7zyCkuXLuXkk08uK2IA1157LfPmzeOcc85h7NixmBnuzsSJE7nnnnsqxGrVqhVpaWlAcO7+D3/4Q1asWEHnzp3Jz8+npKSEqsqEu/Pb3/6WSy+9tEbvsfyQxpXFWLBgQaWnGzZv3pxjx46V5V0+1rPPPkv37t2Pa//mm2/SsmXLsuW0tDSOHDmCu8d8jc6dO5ORkcHixYt58803KxwrqK24rlA1szSgEDgbeAj4ANjn7qVHKIqA08P504GPANz9iJntB04Fdp0QcxIwCSAjI4NXX321Tm+kKsXFxUmJX9u406ZVXNeu3Ul0734uRUUbycws5vDh98jKOocnnihk27YDFdrX5u00tM8hytiNKW7btm05evQoBw5U/D2oSps2bardprq427Zto02bNhw9epTCwkKWLVvGwYMHOXDgAJdccgm/+tWv6Ny5M1OnTuXAgQMMGTKEcePG8b3vfY8OHTqwZ88eiouL6dKlC0DZa+3btw93p2XLlnz66afMmzePMWPG0Lx5c1q3bs2iRYsYPHgwjz32GMeOHePAgQMMHTqU3/72twwaNIgWLVqwceNGOnXqVGE8enenuLiYli1b8uWXX9KiRYuy160sxqBBg3jkkUcYO3YsO3fupKCggLFjx3LgwAE6d+7Ma6+9xogRI3jqqafKPrPc3Fx+85vfMG3aNMyMVatW0a9fP0pKSjh06FDZax45coSDBw/Sp08ffvjDH7J69Wq6du3Knj17yvbex48fz/jx4xk3bhwHDx6M+bMoKSmp2e9WZf01sSagHVAAfBN4v9z6zsDqcH4tkFnuuQ+AU6uKqz5399tuC57r2tV92rQCb9vW/Ysv3KdPV597U4+bzD736uKWlJT4yJEjvU+fPn711Vf70KFDj3uPo0aN8m7duh23zdy5c71fv37ep08fHzBggC9dutTd3Vu3bn1cuzvvvNPPOussz8vL8xtvvNHvuusud3dftmyZ9+nTx4cMGeJTpkzx888/393djx496nfccYf37t3be/Xq5cOGDfN9+/ZVyPnEPvf77ruv7LnKYhw7dswnT57sPXr08DFjxviYMWPK+tyXLFniWVlZfuGFF/rtt99e1ud+8OBBnzRpUlmsUaNGubv7o48+6pMnTz7uMyr9zBYsWODZ2dnet29fHz58eFmbQ4cOeZs2bXz9+vWV/ixq2ude4xtrAHcBPyXYE28erjsPeCmcfwk4L5xvHrazqmKquLu//LL7mjXB/LRpBQ7uTz/tvm2be1qaintTjhtlcY/CgQMHyubvuecev/XWW+s9h4kTJ5YV9/qwfPlyv/DCC6tsk/ADqmbWwczahfMnAcOB9eEe/NVhs4nA/HD+uXCZ8PnFYRJSiTZtYOhQ+Nvfjl8/Zw5kZMDw4dHkJRKF559/nuzsbHr37s1rr73GL37xi6hTSqp7772Xq666KuZxirqIp8+9IzA77HdvBsxz97+Z2Tpgrpn9ClgJzAzbzwQeN7P3gT3AuIRmnIIuuQRatKhY3BcsgL17Yfx4eOmlaHITqW/XXnst1157baQ5zJo1q95ea8qUKUyZMiXhceM5W+ZdoH+M9ZuAwTHWlwD/mJDsmojRo2HPHli69Pj1hw7B008Hp0SefDJUcpxFRKQCXaEaMTO47DJ48UWIdUX2nDmQng5jxtR/biLSeKm4R2zgwKBf/cQumVKvvQZbtwZdMyIi8VJxj9jo0cEe+4svxn7eHZ58Ei69FE47rX5zE5HGS8U9YqNGwRtvBAdOKzNnDjRvDtdcU395ScMVa5TQ2k5f/3r0t9ibPn16pRfuSO2puEeoUyfIyYHnn6+63Zo18O67MGFC/eQlUp9U3JNDxT1Cl10WPFbW317eE0/AeefBmWcmNyeRWB577DH69u1Lv379uP7669myZQt5eXn07duXvLy8sgGzbrzxxrKhciEY5heCMWqGDRvG1VdfzTnnnMP48eNxd2bMmMEnn3xCbm4uubm5kby3VKXiHqFRo2DzZli7tvq2Tz2FRoqUSKxdu5a7776bxYsXs2rVKh544AFuueUWbrjhBt59913Gjx/PrbfeWm2clStXMn36dNatW8emTZt4/fXXufXWW+nUqRMFBQUUFBTUw7tpOlTcI9KyZXDxUnVdMqWKimDJEnXNSP1bvHgxV199NaeFR/Tbt2/P0qVLy4bjvf766/n73/9ebZzBgweTmZlJs2bNyM7OZvPmzclMu8lTcY/IsGHQunV8XTKl5syB7t2DfnqR+uKVDFVbXunz5YfHdXcOHTpU1ibWMLiSPCruERk1Cj7/HGryn+gzz8CXX+qcd6lfeXl5zJs3j927dwOwZ88ezj//fObOnQvAnDlzuPDCC4HglnSFhYUAzJ8/n8OHD1cbP56hiaXm4hrPXRJv9GhYtCgo1vHaty/oxhmn0XqatEQOwxcU1apPh+zVqxd33nknQ4cOJS0tjf79+zNjxgxuuukm7rvvPjp06MCjjz4KwPe+9z3GjBnD4MGDycvLqzDWeiyTJk3iW9/6Fh07dlS/ewKpuEegZ0/o1g1qMwjcnDlw5ZWJz0mkKhMnTmTixInHrVu8eHGFdhkZGSxbtqxsuXSkw2HDhpXdWBrgwQcfLJv/0Y9+VHZzbUkcdctEYNSo4DHeg6nlPf98sAcvIlIVFfcIjB4NK1fCJ5/UfNsvvwz63kVEqqLiHoELLqjZWTInStD9c6UR0f1umrba/PxV3COQlla34v4//5O4XKTha9WqFfv371eBb6Lcnd27d9OqVasabacDqhHYsQOWL6/99vobb1oyMzNZtWoVxcXFCY9dUlJS46Ih9a9Vq1ZkZmbWaBsV9wgsWKACLfFr0aIFxcXFDBw4MOGxX331Vfr3r3CjNUkB6paJQF26ZERE4qHiHoGXX446AxFJdSruEfjss6gzEJFUp+IuIpKCVNxFRFKQiruISApScRcRSUHVFncz62xmBWa23szWmtlt4fp8M/vYzN4Jp8vKbXOHmb1vZhvM7NJkvgEREakonouYjgC3u/vbZtYGKDSz0pP57nf3aeUbm1lPYBzQC+gEvGJm/8fdjyYycRERqVy1e+7u/qm7vx3OHwDWA6dXsckYYK67f+nuHwLvA4MTkayIiMTHajIYkZl1BZYAvYF/Bm4EPgNWEOzd7zWzB4Fl7v5EuM1M4AV3f+aEWJOASQAZGRk5pbfsSobi4mLS09MjiZuens7AgQN577332LZtGwDhXcgqlZlZTFFR1XFPvI9qu3btyM7OZu3atezcubPW+dZGsuImM7biJjeu1I/c3NxCd489LoW7xzUB6UAhcGW4nAGkEez93w08Eq5/CJhQbruZwFVVxc7JyfFkKigoaFBxg5FlKp+mTSuotk195htV3GTGVtzkxpX6AazwSupqXGfLmFkL4Flgjrv/JfxS2O7uR939GPBHvup6KQI6l9s8E6jFbSlERKS24jlbxgj2vte7+3+WW9+xXLOxwJpw/jlgnJm1NLNuQBbwVuJSFhGR6sRztswFwPXAajN7J1z3L8B1ZpYNOLAZuBnA3dea2TxgHcGZNpNdZ8qIiNSraou7u/8dsBhPLahim7sJ+uFFRCQCukJVRCQFqbiLiKQgFXcRkRSk4i4ikoJU3EVEUpCKu4hIClJxFxFJQSruIiIpSMVdRCQFqbiLiKQgFXcRkRSk4i4ikoJU3EVEUpCKexPSsmXLqFMQkXqi4t6EdO/ePeoURKSeqLg3Ie3bt486BRGpJyruIiIpSMU9hZhVnPLzY7fNz4/dXkRSg4p7ips69fjCvX07tGgRrBeR1KXi3sRkZMCll0adhYgkm4p7E7J06WZ27ICJE6PORESSTcW9CXnjjc3MmQOXXw6nnBJ1NiKSTCruTczs2dCyJYwbF3UmIpJMKu5NzKpV8M47cOONUWciIslUbXE3s85mVmBm681srZndFq5vb2Yvm9nG8PGUcL2Z2Qwze9/M3jWzAcl+E1Fp3rx51CnUyuzZMHgw9OgRdSYikizx7LkfAW539x7AEGCymfUEpgCL3D0LWBQuA3wLyAqnScDvE551A9FYr/icMwcOH9aBVZFUVm1xd/dP3f3tcP4AsB44HRgDzA6bzQauCOfHAI95YBnQzsw6JjzzBqBLly5Rp1ArO3fCCy/AhAnQTB1zIinJ3D3+xmZdgSVAb2Cru7cr99xedz/FzP4G3Ovufw/XLwJ+7u4rTog1iWDPnoyMjJy5c+fW8a1Urri4mPT09ITGNDOGDh3Ktm3beO+992q0bWFh1c9nZhZTVFR1vjk5dYt79tmnMWZMb555ZhVbtuytMm51kvH5Jju24iY3rtSP3NzcQncfGPNJd49rAtKBQuDKcHnfCc/vDR+fBy4st34RkFNV7JycHE+mgoKCpMavKah6mjatoNo2dY37ta+579rl/uST1cetTjI/32TFVtzkxpX6AazwSupqXP+Um1kL4Flgjrv/JVy9vbS7JXzcEa4vAjqX2zwT+CSe12lIYo27kkpjtRw6BE89BVdcAW3bRp2NiCRaPGfLGDATWO/u/1nuqeeA0kNyE4H55dbfEJ41MwTY7+6fJjDnSJWO1fLf/x0slxbxxjhWy6xZcNJJcM01UWciIokWz577BcD1wMVm9k44XQbcC1xiZhuBS8JlgAXAJuB94I/ADxOfdvSys6POoO4KC2HtWp01I5KKqj1R24MDo5V1MOTFaO/A5Drm1aCdcgp07QpbtuwBGufpkKVmzYL77oOsLNi4MepsRCRRdCJcLfTrFzwuX/5RtIkkwBNPwNGjcMMNUWciIomk4l4LpV0yO3cWR5tIAmzbBgsXBsW9IR8AFpGaUXGvhexs+OQTOHjwcNSpJMSsWdClC+TmRp2JiCSKinst9O8fDL6VKubPh337dGBVJJWouNdQy5bBgFupVNy//BLmzoWrroo6ExFJFBX3GurZM7gH6cqVUWeSWLNnQ+vWUWchIomi4l5DpQdTU2nPHWDZMtiwIeosRCRRVNxrqH9/KC6GDz6IOpPEmz27+jYi0jiouNdQdnZwN6MaDKbZaDz+eNQZiEiiqLjXgFlwAVOq9beXKiqKOgMRSRQV9xro1g2+/vXU628XkdSj4l4D/fsHjyruItLQqbjXQHY2HDkCa9ZEnYmISNVU3GsgOxvWrw8u+hERachU3Gsg1YYdEJHUpeIepw4d4PTTVdxFpHFQcY9T6RjuqXoapIikFhX3OJUOO7BqVbR5iIjEQ8U9Tv37w9atsGdP1JmIiFRPxT1O2dnqbxeRxkPFPQ4nnQTdu6u/XUQaDxX3OPTuDWlp2nMXkcZDxT0OGnZARBobFfc4ZGcH9xjdvDnqTERE4qPiHgcdTBWRxqba4m5mj5jZDjNbU25dvpl9bGbvhNNl5Z67w8zeN7MNZnZpshKvL82aQd++Ku4i0rjEs+c+CxgZY/397p4dTgsAzKwnMA7oFW7zOzNLS1SyUcjKCm4creIuIo1JtcXd3ZcA8V66MwaY6+5fuvuHwPvA4DrkF7lUvSG2iKQ28zhuBmpmXYG/uXvvcDkfuBH4DFgB3O7ue83sQWCZuz8RtpsJvODuz8SIOQmYBJCRkZEzd+7cBLyd2IqLi0lPT6/RNoWFweM3v3kmOTmZzJjxGseOHf9ZZWYWU1RUddycnMpjV6ahxS3Vo0cP2rdvzxtvvEH535vafL7xSlZsxU1uXKkfubm5he4+MOaT7l7tBHQF1pRbzgDSCPb87wYeCdc/BEwo124mcFV18XNycjyZCgoKarxNcAts9xdecH/77a+Wy0/TphXEXF9+qip2ZVNDi1ud2ny+UcdW3OTGlfoBrPBK6mqtzpZx9+3uftTdjwF/5KuulyKgc7mmmcAntXmNhkJjuItIY1Sr4m5mHcstjgVKz6R5DhhnZi3NrBuQBbxVtxSj8w//ABkZKu4i0vg0r66BmT0FDANOM7Mi4C5gmJllAw5sBm4GcPe1ZjYPWAccASa7+9HkpJ58pQdTNaaMiDQ21RZ3d78uxuqZVbS/m6AfvtErHXZAY7iLSGOjK1SrkJ0NmzbBZ59FnYmISM2ouFchO1tdMiLSOKm4VyI9Hc4+WwdTRaRxUnGvRN++wbgyKu4i0hipuFdCww6ISGOm4l6J7GzYtQuKiqLOpOFr0aJF1CmIyAlU3CuhMdzjd84550SdgoicQMW9En36qLjH69RTT406BRE5gYp7JVq1UnEXkcZLxb0KOsc9YFZxys+P3TY/P3Z7EalfKu6VKCmBDRuizqLhmjr1+MJdUgIdOgTrRSR6Ku6VWL0ajjbaIc/qX6tWcPPNUWchIqVU3Cuh/vb4vfHGZl54ASZPBp0VKdIwqLhXQv3t8Vu6dDPTp0PHjnDNNVFnIyKg4l4p7bnXzMKFsG4d/OQnUWciIqDiXqnVq6POoPF54IHgBtsXXBB1JiKi4l6J4uKoM2h8Hn8c9uyBH/846kxERMVdEuaLL+Dhh2HsWDjjjKizEWnaVNwloR56CNzhlluizkSkaVNxl4T6+GN4+mn4p38KbngiItFQcZeEmz4d2rWDiROjzkSk6VJxl4R76y1YuhRuu03jyohERcVdkmL6dMjKgssuizoTkaZJxV2S4i9/gY8+0mmRIlGptrib2SNmtsPM1pRb197MXjazjeHjKeF6M7MZZva+mb1rZgOSmbw0XEeOwIMPwvDh0Lt31NmIND3x7LnPAkaesG4KsMjds4BF4TLAt4CscJoE/D4xaUpj9Mc/wsGDQd+7iNSvaou7uy8B9pywegwwO5yfDVxRbv1jHlgGtDOzjolKVhqXvXth9myYMCHqTESantr2uWe4+6cA4eM3wvWnAx+Va1cUrpMmasaMYKx3Ealf5u7VNzLrCvzN3XuHy/vcvV255/e6+ylm9jxwj7v/PVy/CPiZuxfGiDmJoOuGjIyMnLlz5ybg7cRWXFxMejVX1Fx44YVs376djRs3AlBYIeOKMjOLKSqqOm5OTsV11cVOtbhXXtmHTp3SWbZsGfH8vpUXz8+uNhQ3uXGlfuTm5ha6+8CYT7p7tRPQFVhTbnkD0DGc7whsCOcfBq6L1a6qKScnx5OpoKCgxtsEF9FXPU2bVlBtm9rETrW4I0bU+OMvU5ufneJGH1fqB7DCK6mrte2WeQ4ovf5wIjC/3PobwrNmhgD7Pey+kaZr4cKoMxBpeppX18DMngKGAaeZWRFwF3AvMM/MvgtsBf4xbL4AuAx4HzgIfCcJOYuISDWqLe7ufl0lT+XFaOvA5LomJSIidaMrVEVEUpCKu4hIClJxFxFJQSruIiIpSMVdRCQFqbiLiKQgFXcRkRSk4i4ikoJU3EVEUpCKu4hIClJxFxFJQSruIiIpSMVdRCQFqbiLiKQgFXcRkRSk4i4ikoJU3EVEUpCKu4hIClJxFxFJQSruIiIpSMVdRCQFqbiLiKQgFXcRkRSk4i4NWteuXaNOQaRRUnGXBk3FXaR2mtdlYzPbDBwAjgJH3H2gmbUH/gx0BTYD17j73rqlKSIiNZGIPfdcd89294Hh8hRgkbtnAYvCZZEKzGJP+fmx2+fnV2wrIrElo1tmDDA7nJ8NXJGE15AUNnVqULi7dPlq3SuvwKOPRpeTSGNj7l77jc0+BPYCDjzs7n8ws33u3q5cm73ufkqMbScBkwAyMjJy5s6dW+s8qlNcXEx6ejonn3wygwcPZv369Wzfvr3KbQoLq4+bmVlMUVF6lW1ycmoeW3G/cvvtw1i4cAPDhp0FwKuvfsDq1Z9WGTcepb8Tiaa4Up9yc3MLy/WaHM/daz0BncLHbwCrgIuAfSe02VtdnJycHE+mgoKCGm8D1U/TphVU26Y2sRX3q+n11z90cD/jDPdXXgm2e+kl98zMyuPGoza/E4orDQ2wwiupq3XqlnH3T8LHHcBfgcHAdjPrCBA+7qjLa0jTtnTpZgC2bIFLLoEf/ADOPx/WrIGbboo0NZEGrdbF3cxam1mb0nlgBLAGeA6YGDabCMyva5IiEOzL/9d/Qd++sHIlzJwZdUYiDVdd9twzgL+b2SrgLeB5d38RuBe4xMw2ApeEyyIJ8+GHcPHFcMstUWci0nDV+jx3d98E9IuxfjeQV5ekRKrjDg89BA8+GHUmIg2TrlAVEUlBKu4iIilIxV1EJAWpuIuIpKAmUdzT0tKiTkEaGI02KamuSRR3XV4tJ1Jxl1Sn4i4ikoJU3CXl1HQo4Q8/hDvugAEDoFkzDSUsqUHFXZqM0qGES4t3Xh78+tewfz/ccw+8/TZs3w5z5kSbp0gi1OlOTI2FirvEsnhxME2ZAhkZMHw4jBgRTCKNXZMo7qb/s+UEb7yxmeBOkIHSPfbSvfba3uZAB2qloWgS3TIiJyodSjjRVNyloWj0xT0R9+HUjr2IpJpGX9wrU3rwrFn4DkuL+NSp0eYljVesnYJmzeDhh2O3/9//hdtugx49tBMh9S9li3upOtwiVqRa7vD97x9fvEePhhkzguceeADWrw/uJPXHP0abqzQtTeKA6okHz0SS6fnngwngjDOC2wOOGAFXXRVtXrHoGEHqSvk9d0jewTOREwU7El/ZsgX+9Ce45hro0KH2cZNVhFXcU1eTKO4i9aWqHYmjR2sfV0VYaqpJdMuINEUlJbBvXzDt3//V/NlnQ05O7G2OHYO1a+Gtt+DNN4PHd96p37wlMVTcRSIW6yya5s2Ds3Buuqnic2+9BQsXHl+w9+8P1pV30knVv7Z78Prt28PgwcF07rkwZgx897u1ez/SMKi4izRAR44ExbW0wJYW4WTZswdefDGYSp15ZlDon3wyea8ryaM+d5Em7MQDwOVt2gRPPVV/uUhiqbiLNAJVFeG6qM2ZZOUv4jrllOA0z3/9V9iwoertjhyB3bvhgw+gsLBW6UoNqFtGpBEIinDXaJOIYd8+ePnlYPrVr4J1XboEp4COGAHt2kHbtsHjifOSXCruIpJQW7cGjy+/XHW7E68eT0uDr3+94hfBjTfCFVdU3Ecc4EAAAAhKSURBVH7TJti79/i2zVXRyiTtozCzkcADQBrwJ3e/N1mvJSINS2UD91Xl2LGvzv4pb/78r+arO7B88snw+efHr7vuuopfGG3bwsCBkJVV8zwbi6QUdzNLAx4CLgGKgOVm9py7r0vG64lIwxLVAH0HD1ZcN3du9duVfmm0aBEU/vJfAqVfDDNnHr/N/PkV27Vt+9VghSfKz4/9uSRr/Ktk7bkPBt53900AZjYXGAOouItIrdXmP4KaOHwYdu0KphOdWNxjdRWZQXr6V18I774b3Wig5kn42jCzq4GR7v5P4fL1wLnufku5NpOASeFid6CaY+11choQ48eluA08bjJjK25y4zYqmZmZnYqKij5JdNycnJycwsKknht0hrvHHLUoWXvusb6rjvsWcfc/AH9I0usfn4zZCncfqLiNK24yYytucuNK9JJ1nnsR0LncciaQ8G9FERGJLVnFfTmQZWbdzOxrwDjguSS9loiInCAp3TLufsTMbgFeIjgV8hF3X5uM14pTsrp/FDe5cZMZW3GTG1cilpQDqiIiEi2NLSMikoJU3EVEUlBKF3cze8TMdpjZmgTH7WxmBWa23szWmtltCYrbyszeMrNVYdyEXudnZmlmttLM/pbAmJvNbLWZvWNmKxIYt52ZPWNm74Wf83kJiNk9zLN0+szMfpygfH8S/szWmNlTZtYqQXFvC2OurWuusf4ezKy9mb1sZhvDx1PqnrU0BCld3IFZwMgkxD0C3O7uPYAhwGQz65mAuF8CF7t7PyAbGGlmQxIQt9RtwPoExiuV6+7ZCT5f+gHgRXc/B+hHAvJ29w1hntlADnAQ+Gtd45rZ6cCtwEB3701wEsG4BMTtDXyP4IrvfsBoM6vLaCizqPj3MAVY5O5ZwKJwWVJAShd3d18C7ElC3E/d/e1w/gBB4Tk9AXHd3YvDxRbhlJAj3maWCYwC/pSIeMlkZl8HLgJmArj7IXffV/VWNZYHfODuWxIUrzlwkpk1B04mMdd19ACWuftBdz8C/A8wtrbBKvl7GAPMDudnAzEuqpfGKKWLe30ws65Af+DNBMVLM7N3gB3Ay+6ekLjAdOBnwLEExSvlwEIzKwyHlEiEM4GdwKNhN9KfzKx1gmKXGgck5D5D7v4xMA3YCnwK7Hf3hVVvFZc1wEVmdqqZnQxcxvEXByZChrt/CsFOC/CNBMeXiKi414GZpQPPAj92988SEdPdj4bdBpnA4PBf8zoxs9HADndPxhgXF7j7AOBbBN1TFyUgZnNgAPB7d+8PfE4CuwvCC+suB55OULxTCPaAuwGdgNZmNqGucd19PfBr4GXgRWAVQZegSLVU3GvJzFoQFPY57v6XRMcPuyFeJTHHDC4ALjezzcBc4GIzeyIBcXH3T8LHHQT914MTELYIKCr3X8szBMU+Ub4FvO3u2xMUbzjwobvvdPfDwF+A8xMR2N1nuvsAd7+IoEtlYyLilrPdzDoChI87EhxfIqLiXgtmZgT9wevd/T8TGLeDmbUL508iKBrv1TWuu9/h7pnu3pWgO2Kxu9d5z9LMWptZm9J5YARBV0KduPs24CMz6x6uyiOxw0VfR4K6ZEJbgSFmdnL4u5FHgg5cm9k3wscuwJUkNm8IhgWZGM5PBOZX0VYakZS+KZWZPQUMA04zsyLgLnefWfVWcbkAuB5YHfaPA/yLuy+oY9yOwOzwZifNgHnunrDTFpMgA/hrUM9oDjzp7i8mKPaPgDlhF8om4DuJCBr2XV8C3JyIeADu/qaZPQO8TdBtspLEXdb/rJmdChwGJrv73toGivX3ANwLzDOz7xJ8Sf1j3VOWhkDDD4iIpCB1y4iIpCAVdxGRFKTiLiKSglTcRURSkIq7iEgKUnGXBs/M7i8/IqKZvWRmfyq3/Bsz++c6xM83s/8X7/pEMbOuZvbtcss3mtmDyXo9aVpU3KUxeIPwik8zawacBvQq9/z5wOvxBAqvIWgougLfrq6RSG2ouEtj8DpfXc7fi+Aq2ANmdoqZtSQYPXGlBe4Lxz9fbWbXApjZsHD8/SeB1eG6O81sg5m9AnSv+JKVM7MJFoy7/46ZPVz6hWFmxWZ2twXj8S8zs4xw/Vnh8nIz+6WZlY78eS/wzTDOT8J1nczsxXB89f+o9ScmTZ6KuzR44fg1R8JL8M8HlhKMwnkeMBB4190PEVyen00w9vlw4L7ScVMIxry50917mlkOwTAM/cNtBsWbi5n1AK4lGDAtGzgKjA+fbk0wRG8/YAnBWOwQjE3/gLsP4vihgKcAr4VjzN8frssO4/cBrjWzRI8CKU2Eirs0FqV776XFfWm55TfCNhcCT4Uja24nGP+8tHC/5e4fhvPfBP4ajpP+GcH4KvHKI7jRx/Jw6Ik8giGKAQ4BpcNFFBJ0u0DwJVQ6AuWT1cRf5O773b2EYDydM2qQm0iZlB5bRlJKab97H4JumY+A24HPgEfCNlbF9p+fsFzbcTcMmO3ud8R47rB/NZ7HUWr39/VlufnaxhDRnrs0Gq8Do4E94Z75HqAdwV7x0rDNEoKujDQz60BwN6e3YsRaAow1s5PCUS3/bw3yWARcXW60xvZmVt3e9TLgqnC+/O33DgBtavDaInFTcZfGYjXBWTLLTli33913hct/Bd4luKnFYuBn4fDBxwlvkfhn4B2CMflfq+J1f2FmRaWTu68DfkFw96l3CW6k0bGK7QF+DPyzmb0Vtt0frn+X4FjCqnIHVEUSQqNCiiRZOMzwF+7uZjYOuM7dx0Sdl6Q29eeJJF8O8GB4I499wE0R5yNNgPbcRURSkPrcRURSkIq7iEgKUnEXEUlBKu4iIilIxV1EJAX9f0IRo91yc3JzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualization\n",
    "plt.bar(A1, B1, color='blue')\n",
    "plt.plot(A1, B1, marker='+' , color='white')\n",
    "plt.legend()\n",
    "plt.xlabel('Word Length')\n",
    "plt.xticks(range(1,11))\n",
    "plt.grid(True)\n",
    "plt.legend(['average frequency', 'count'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "top20Words=linesRDD.map(lambda line: line[0]) \\\n",
    "        .flatMap(lambda line : re.split('\\\\W+', line)) \\\n",
    "        .filter(lambda word : len(word)>0) \\\n",
    "        .map(lambda word : (word,1)) \\\n",
    "        .reduceByKey(lambda v, w : v+w) \\\n",
    "        .sortBy(lambda vw : vw[1], False) \\\n",
    "        .take(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('및', 41),\n",
       " ('학습한다', 15),\n",
       " ('Business', 14),\n",
       " ('등을', 13),\n",
       " ('대한', 12),\n",
       " ('분석', 11),\n",
       " ('수', 11),\n",
       " ('이를', 11),\n",
       " ('Management', 11),\n",
       " ('공부한다', 10),\n",
       " ('다양한', 9),\n",
       " ('위해', 9),\n",
       " ('한다', 9),\n",
       " ('IT', 9),\n",
       " ('and', 8),\n",
       " ('본', 8),\n",
       " ('목적으로', 8),\n",
       " ('이론을', 8),\n",
       " ('등', 8),\n",
       " ('디지털', 8)]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top20Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "SQL_top20Words = wcDFlength.filter(wcDFlength.wordLen >= 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----+-------+\n",
      "|     words|freq|wordLen|\n",
      "+----------+----+-------+\n",
      "|        및|  41|      1|\n",
      "|  학습한다|  15|      4|\n",
      "|  Business|  14|      8|\n",
      "|      등을|  13|      2|\n",
      "|      대한|  12|      2|\n",
      "|      분석|  11|      2|\n",
      "|        수|  11|      1|\n",
      "|      이를|  11|      2|\n",
      "|Management|  11|     10|\n",
      "|  공부한다|  10|      4|\n",
      "|    다양한|   9|      3|\n",
      "|      위해|   9|      2|\n",
      "|      한다|   9|      2|\n",
      "|        IT|   9|      2|\n",
      "|       and|   8|      3|\n",
      "|        본|   8|      1|\n",
      "|  목적으로|   8|      4|\n",
      "|    이론을|   8|      3|\n",
      "|        등|   8|      1|\n",
      "|    디지털|   8|      3|\n",
      "+----------+----+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "SQL_top20Words.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = []\n",
    "y = []\n",
    "for (w , c) in top20Words:\n",
    "    x.append(w)\n",
    "    y.append(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import font_manager\n",
    "fontpath = '/home/tteon/바탕화면/NanumBarunGothic.ttf'\n",
    "fontprop = font_manager.FontProperties(fname=fontpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/home/tteon/바탕화면/NanumBarunGothic.ttf'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/IPython/core/formatters.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    339\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mprinter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    342\u001b[0m             \u001b[0;31m# Finally look for special method names\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    343\u001b[0m             \u001b[0mmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_real_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_method\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(fig)\u001b[0m\n\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m'png'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mformats\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 248\u001b[0;31m         \u001b[0mpng_formatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFigure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'png'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    249\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m'retina'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mformats\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m'png2x'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mformats\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m         \u001b[0mpng_formatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfor_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFigure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mretina_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/IPython/core/pylabtools.py\u001b[0m in \u001b[0;36mprint_figure\u001b[0;34m(fig, fmt, bbox_inches, **kwargs)\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0mFigureCanvasBase\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m     \u001b[0mfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanvas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_figure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbytes_io\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbytes_io\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfmt\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'svg'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/backend_bases.py\u001b[0m in \u001b[0;36mprint_figure\u001b[0;34m(self, filename, dpi, facecolor, edgecolor, orientation, format, bbox_inches, **kwargs)\u001b[0m\n\u001b[1;32m   2063\u001b[0m                         \u001b[0morientation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morientation\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2064\u001b[0m                         \u001b[0mdryrun\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2065\u001b[0;31m                         **kwargs)\n\u001b[0m\u001b[1;32m   2066\u001b[0m                     \u001b[0mrenderer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cachedRenderer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2067\u001b[0m                     \u001b[0mbbox_artists\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"bbox_extra_artists\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py\u001b[0m in \u001b[0;36mprint_png\u001b[0;34m(self, filename_or_obj, metadata, pil_kwargs, *args, **kwargs)\u001b[0m\n\u001b[1;32m    525\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    526\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 527\u001b[0;31m             \u001b[0mFigureCanvasAgg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    528\u001b[0m             \u001b[0mrenderer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_renderer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mcbook\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_setattr_cm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdpi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdpi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py\u001b[0m in \u001b[0;36mdraw\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrenderer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_renderer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcleared\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    387\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mRendererAgg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 388\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    389\u001b[0m             \u001b[0;31m# A GUI class may be need to update a window using this draw, so\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    390\u001b[0m             \u001b[0;31m# don't forget to call the superclass.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mdraw_wrapper\u001b[0;34m(artist, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0martist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_agg_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/figure.py\u001b[0m in \u001b[0;36mdraw\u001b[0;34m(self, renderer)\u001b[0m\n\u001b[1;32m   1707\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1708\u001b[0m             mimage._draw_list_compositing_images(\n\u001b[0;32m-> 1709\u001b[0;31m                 renderer, self, artists, self.suppressComposite)\n\u001b[0m\u001b[1;32m   1710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1711\u001b[0m             \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'figure'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/image.py\u001b[0m in \u001b[0;36m_draw_list_compositing_images\u001b[0;34m(renderer, parent, artists, suppress_composite)\u001b[0m\n\u001b[1;32m    133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnot_composite\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhas_images\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0martists\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m             \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;31m# Composite any adjacent images together\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mdraw_wrapper\u001b[0;34m(artist, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0martist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_agg_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/axes/_base.py\u001b[0m in \u001b[0;36mdraw\u001b[0;34m(self, renderer, inframe)\u001b[0m\n\u001b[1;32m   2645\u001b[0m             \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_rasterizing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2646\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2647\u001b[0;31m         \u001b[0mmimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_draw_list_compositing_images\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0martists\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2648\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2649\u001b[0m         \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'axes'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/image.py\u001b[0m in \u001b[0;36m_draw_list_compositing_images\u001b[0;34m(renderer, parent, artists, suppress_composite)\u001b[0m\n\u001b[1;32m    133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnot_composite\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhas_images\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0martists\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m             \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrenderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m         \u001b[0;31m# Composite any adjacent images together\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/artist.py\u001b[0m in \u001b[0;36mdraw_wrapper\u001b[0;34m(artist, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0mrenderer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mdraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0martist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrenderer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0martist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_agg_filter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36mdraw\u001b[0;34m(self, renderer, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1203\u001b[0m         \u001b[0mticks_to_draw\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_ticks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1204\u001b[0m         ticklabelBoxes, ticklabelBoxes2 = self._get_tick_bboxes(ticks_to_draw,\n\u001b[0;32m-> 1205\u001b[0;31m                                                                 renderer)\n\u001b[0m\u001b[1;32m   1206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1207\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtick\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mticks_to_draw\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m_get_tick_bboxes\u001b[0;34m(self, ticks, renderer)\u001b[0m\n\u001b[1;32m   1148\u001b[0m         \u001b[0;34m\"\"\"Return lists of bboxes for ticks' label1's and label2's.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1149\u001b[0m         return ([tick.label1.get_window_extent(renderer)\n\u001b[0;32m-> 1150\u001b[0;31m                  for tick in ticks if tick.label1.get_visible()],\n\u001b[0m\u001b[1;32m   1151\u001b[0m                 [tick.label2.get_window_extent(renderer)\n\u001b[1;32m   1152\u001b[0m                  for tick in ticks if tick.label2.get_visible()])\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/axis.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1148\u001b[0m         \u001b[0;34m\"\"\"Return lists of bboxes for ticks' label1's and label2's.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1149\u001b[0m         return ([tick.label1.get_window_extent(renderer)\n\u001b[0;32m-> 1150\u001b[0;31m                  for tick in ticks if tick.label1.get_visible()],\n\u001b[0m\u001b[1;32m   1151\u001b[0m                 [tick.label2.get_window_extent(renderer)\n\u001b[1;32m   1152\u001b[0m                  for tick in ticks if tick.label2.get_visible()])\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/text.py\u001b[0m in \u001b[0;36mget_window_extent\u001b[0;34m(self, renderer, dpi)\u001b[0m\n\u001b[1;32m    888\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Cannot get window extent w/o renderer'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    889\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 890\u001b[0;31m         \u001b[0mbbox\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdescent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_layout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_renderer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    891\u001b[0m         \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_unitless_position\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    892\u001b[0m         \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/text.py\u001b[0m in \u001b[0;36m_get_layout\u001b[0;34m(self, renderer)\u001b[0m\n\u001b[1;32m    289\u001b[0m         _, lp_h, lp_d = renderer.get_text_width_height_descent(\n\u001b[1;32m    290\u001b[0m             \u001b[0;34m\"lp\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fontproperties\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m             ismath=\"TeX\" if self.get_usetex() else False)\n\u001b[0m\u001b[1;32m    292\u001b[0m         \u001b[0mmin_dy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlp_h\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlp_d\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_linespacing\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py\u001b[0m in \u001b[0;36mget_text_width_height_descent\u001b[0;34m(self, s, prop, ismath)\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0mflags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_hinting_flag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m         \u001b[0mfont\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_agg_font\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    211\u001b[0m         \u001b[0mfont\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m         \u001b[0mw\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfont\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_width_height\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# width and height of unrotated string\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py\u001b[0m in \u001b[0;36m_get_agg_font\u001b[0;34m(self, prop)\u001b[0m\n\u001b[1;32m    244\u001b[0m         \"\"\"\n\u001b[1;32m    245\u001b[0m         \u001b[0mfname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfindfont\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 246\u001b[0;31m         \u001b[0mfont\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_font\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m         \u001b[0mfont\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/matplotlib/font_manager.py\u001b[0m in \u001b[0;36mget_font\u001b[0;34m(filename, hinting_factor)\u001b[0m\n\u001b[1;32m   1339\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhinting_factor\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1340\u001b[0m         \u001b[0mhinting_factor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrcParams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text.hinting_factor'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1341\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_get_font\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhinting_factor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/tteon/바탕화면/NanumBarunGothic.ttf'"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(x,y)\n",
    "plt.title('wordscount_visualization_barsection')\n",
    "plt.xlabel('wordcount', color='blue')\n",
    "plt.ylabel('volume' , color='gray')\n",
    "plt.grid(True)\n",
    "plt.xticks(rotation='vertical', fontproperties=fontprop)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "top20Words_E=linesRDD_E.map(lambda line: line[0]) \\\n",
    "            .flatMap(lambda line : re.split('\\\\W+', line)) \\\n",
    "            .filter(lambda word : len(word)>0) \\\n",
    "            .map(lambda word : (word,1)) \\\n",
    "            .reduceByKey(lambda v, w : v+w) \\\n",
    "            .sortBy(lambda vw : vw[1], False) \\\n",
    "            .take(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 63800),\n",
       " ('of', 37714),\n",
       " ('and', 33405),\n",
       " ('to', 26327),\n",
       " ('in', 14782),\n",
       " ('you', 12231),\n",
       " ('And', 11577),\n",
       " ('will', 10596),\n",
       " ('a', 10436),\n",
       " ('for', 10327),\n",
       " ('is', 10127),\n",
       " ('his', 9257),\n",
       " ('be', 8972),\n",
       " ('he', 8500),\n",
       " ('I', 7811),\n",
       " ('your', 7655),\n",
       " ('Lord', 7633),\n",
       " ('on', 6890),\n",
       " ('have', 6890),\n",
       " ('him', 6119)]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top20Words_E"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+-------+\n",
      "|words| freq|wordLen|\n",
      "+-----+-----+-------+\n",
      "|  the|63800|      3|\n",
      "|   of|37714|      2|\n",
      "|  and|33405|      3|\n",
      "|   to|26327|      2|\n",
      "|   in|14782|      2|\n",
      "|  you|12231|      3|\n",
      "|  And|11577|      3|\n",
      "| will|10596|      4|\n",
      "|    a|10436|      1|\n",
      "|  for|10327|      3|\n",
      "|   is|10127|      2|\n",
      "|  his| 9257|      3|\n",
      "|   be| 8972|      2|\n",
      "|   he| 8500|      2|\n",
      "|    I| 7811|      1|\n",
      "| your| 7655|      4|\n",
      "| Lord| 7633|      4|\n",
      "|   on| 6890|      2|\n",
      "| have| 6890|      4|\n",
      "|  him| 6119|      3|\n",
      "+-----+-----+-------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "SQL_top20words_E = wcDF_E_length.filter(wcDF_E_length.wordLen >= 1).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "SQL_top20words_E = wcDF_E_length.filter(wcDF_E_length.wordLen >= 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(words='the', freq=63800, wordLen=3),\n",
       " Row(words='of', freq=37714, wordLen=2),\n",
       " Row(words='and', freq=33405, wordLen=3),\n",
       " Row(words='to', freq=26327, wordLen=2),\n",
       " Row(words='in', freq=14782, wordLen=2),\n",
       " Row(words='you', freq=12231, wordLen=3),\n",
       " Row(words='And', freq=11577, wordLen=3),\n",
       " Row(words='will', freq=10596, wordLen=4),\n",
       " Row(words='a', freq=10436, wordLen=1),\n",
       " Row(words='for', freq=10327, wordLen=3),\n",
       " Row(words='is', freq=10127, wordLen=2),\n",
       " Row(words='his', freq=9257, wordLen=3),\n",
       " Row(words='be', freq=8972, wordLen=2),\n",
       " Row(words='he', freq=8500, wordLen=2),\n",
       " Row(words='I', freq=7811, wordLen=1),\n",
       " Row(words='your', freq=7655, wordLen=4),\n",
       " Row(words='Lord', freq=7633, wordLen=4),\n",
       " Row(words='on', freq=6890, wordLen=2),\n",
       " Row(words='have', freq=6890, wordLen=4),\n",
       " Row(words='him', freq=6119, wordLen=3)]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SQL_top20words_E.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'df'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-62-ddb8a9a1f0c2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mSQL_frequency_2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSQL_top20words_E\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'freq'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/softwares/spark-2.4.5-bin-hadoop2.7/python/pyspark/sql/dataframe.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   1302\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1303\u001b[0m             raise AttributeError(\n\u001b[0;32m-> 1304\u001b[0;31m                 \"'%s' object has no attribute '%s'\" % (self.__class__.__name__, name))\n\u001b[0m\u001b[1;32m   1305\u001b[0m         \u001b[0mjc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1306\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mColumn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'df'"
     ]
    }
   ],
   "source": [
    "SQL_frequency_2 = SQL_top20words_E.df['freq']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(type(SQL_frequency_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SQL_frequency_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for bb1 in SQL_frequency_2:\n",
    "    bb.append(bb1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(type(SQL_frequency))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SQL_frequency.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = []\n",
    "b = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for (w , c) in top20Words_E:\n",
    "    a.append(w)\n",
    "    b.append(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar(a,b)\n",
    "plt.title('bible_frequency')\n",
    "plt.xlabel('wordcount', color='blue')\n",
    "plt.ylabel('volume' , color='gray')\n",
    "plt.grid(True)\n",
    "plt.xticks(rotation='vertical')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(a,b)\n",
    "plt.title('bible_frequency')\n",
    "plt.xlabel('wordcount', color= 'gray')\n",
    "plt.ylabel('volume' , color = 'blue')\n",
    "plt.grid(True)\n",
    "plt.xticks(rotation='vertical')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(1)\n",
    "plt.bar(a,b)\n",
    "plt.title('bible_frequency')\n",
    "plt.xlabel('wordcount', color='blue')\n",
    "plt.ylabel('volume' , color='gray')\n",
    "plt.grid(True)\n",
    "plt.xticks(rotation='vertical')\n",
    "plt.show()\n",
    "\n",
    "plt.figure(2)\n",
    "plt.bar(x,y)\n",
    "plt.title('wordscount_visualization_barsection')\n",
    "plt.xlabel('wordcount', color='blue')\n",
    "plt.ylabel('volume' , color='gray')\n",
    "plt.grid(True)\n",
    "plt.xticks(rotation='vertical', fontproperties=fontprop)\n",
    "plt.show()\n",
    "\n",
    "plt.figure(3)\n",
    "plt.bar(xA1, xB1, color='green')\n",
    "plt.plot(xA1, xB1, marker='*' , color='red')\n",
    "plt.legend()\n",
    "plt.xlabel('Word Length')\n",
    "plt.xticks(range(1,11))\n",
    "plt.grid(True)\n",
    "plt.legend(['average frequency', 'count'])\n",
    "plt.show()\n",
    "\n",
    "plt.figure(4)\n",
    "plt.bar(A1, B1, color='blue')\n",
    "plt.plot(A1, B1, marker='+' , color='white')\n",
    "plt.legend()\n",
    "plt.xlabel('Word Length')\n",
    "plt.xticks(range(1,11))\n",
    "plt.grid(True)\n",
    "plt.legend(['average frequency', 'count'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(2, 2)\n",
    "axs[0, 0].plot(a, b)\n",
    "axs[0, 0].set_title('original_english')\n",
    "axs[0, 1].plot(A1, B1, 'tab:orange')\n",
    "axs[0, 1].set_title('SQL_english')\n",
    "axs[1, 0].plot(x, y, 'tab:green')\n",
    "axs[1, 0].set_title('original_korean')\n",
    "axs[1, 1].plot(xA1, xB1, 'tab:red')\n",
    "axs[1, 1].set_title('SQL_korean')\n",
    "\n",
    "for ax in axs.flat:\n",
    "    ax.set(xlabel='x-label', ylabel='y-label')\n",
    "\n",
    "# Hide x labels and tick labels for top plots and y ticks for right plots.\n",
    "for ax in axs.flat:\n",
    "    ax.label_outer()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}