regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "subject-surgery",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
    "os.environ[\"SPARK_HOME\"] = \"/home/ebrahim/Downloads/spark-3.0.1-bin-hadoop3.2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "faced-mechanism",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Line magic function `%` not found.\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "possible-shaft",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import findspark\n",
    "from pyspark.sql.session import SparkSession\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.sql import types as T\n",
    "from pyspark.sql import Window\n",
    "import seaborn as sns\n",
    "\n",
    "findspark.init()\n",
    "os.environ[\"PYSPARK_DRIVER_PYTHON\"] = os.environ[\"PYSPARK_PYTHON\"] = \"/usr/bin/python3.7\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "substantial-habitat",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.feature import OneHotEncoder\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "from pyspark.ml.feature import HashingTF\n",
    "from pyspark.ml.feature import IDF\n",
    "from pyspark.ml.feature import Tokenizer\n",
    "from pyspark.ml.feature import CountVectorizer\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "from pyspark.ml.feature import StopWordsRemover"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "based-grocery",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "employed-segment",
   "metadata": {},
   "source": [
    "## Read all profiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "occupied-weight",
   "metadata": {},
   "outputs": [],
   "source": [
    "profiles_df = spark.read.parquet('./translated/gsheet/*.parquet').withColumn(\n",
    "    \"translated_description\",\n",
    "    F.regexp_replace(F.col(\"translated_description\"), r\"\\.|\\'|\\\"|\\[|\\]|\\,|\\-|\\+|\\(|\\)|\\`|\\*|\\^\", \" \")\n",
    ").withColumn(\n",
    "    \"translated_description\",\n",
    "    F.regexp_replace(F.col(\"translated_description\"), r\"\\s+\", \" \")\n",
    ").withColumn(\n",
    "    \"translated_description\",\n",
    "    F.lower(\"translated_description\"),\n",
    ").withColumn(\n",
    "    \"status\",\n",
    "    F.when(\n",
    "        F.col(\"status\").startswith(\"wi\"),\n",
    "        F.lit(\"widowed\"),\n",
    "    ).when(\n",
    "        F.col(\"status\").startswith(\"di\"),\n",
    "        F.lit(\"divorced\"),\n",
    "    ).when(\n",
    "        F.col(\"status\").startswith(\"sep\"),\n",
    "        F.lit(\"separated\"),\n",
    "    ).when(\n",
    "        F.col(\"status\").startswith(\"married\"),\n",
    "        F.lit(\"married\"),\n",
    "    ).when(\n",
    "        F.col(\"status\") == \"in relationship\",\n",
    "        F.col(\"status\"),\n",
    "    ).when(\n",
    "        F.col(\"status\").startswith(\"sing\"),\n",
    "        F.lit(\"single\"),\n",
    "    ).otherwise(F.lit(\"other\")),\n",
    ").select(\n",
    "    F.col(\"age\").cast(T.IntegerType()).alias(\"age\"),\n",
    "    F.col(\"ethnicity\"),\n",
    "    F.when(F.col(\"gender\") == \"male\", F.lit(0)).otherwise(F.lit(1)).alias(\"gender\"),\n",
    "    F.col(\"scam\"),\n",
    "    F.col(\"status\"),\n",
    "    F.col(\"translated_description\").alias(\"description\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "inappropriate-nirvana",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-----+\n",
      "|scam|count|\n",
      "+----+-----+\n",
      "|   0|36276|\n",
      "|   1| 7987|\n",
      "+----+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "profiles_df.groupBy(\"scam\").count().show(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "painted-belly",
   "metadata": {},
   "source": [
    "### Correlation between independent cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "moderate-companion",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexer = StringIndexer(inputCols=[\"ethnicity\", \"status\"], outputCols=[\"ethnicity_index\", \"status_index\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cheap-campus",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexed = indexer.setHandleInvalid(\"skip\").fit(profiles_df.drop(\"scam\")).transform(profiles_df.drop(\"scam\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "opened-vintage",
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = indexed.toPandas().corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "subjective-favor",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>ethnicity_index</th>\n",
       "      <th>status_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>age</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.158107</td>\n",
       "      <td>-0.113374</td>\n",
       "      <td>0.351142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>-0.158107</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.021581</td>\n",
       "      <td>-0.077619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ethnicity_index</th>\n",
       "      <td>-0.113374</td>\n",
       "      <td>0.021581</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.042973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>status_index</th>\n",
       "      <td>0.351142</td>\n",
       "      <td>-0.077619</td>\n",
       "      <td>-0.042973</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      age    gender  ethnicity_index  status_index\n",
       "age              1.000000 -0.158107        -0.113374      0.351142\n",
       "gender          -0.158107  1.000000         0.021581     -0.077619\n",
       "ethnicity_index -0.113374  0.021581         1.000000     -0.042973\n",
       "status_index     0.351142 -0.077619        -0.042973      1.000000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "female-corporation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnEAAAINCAYAAABRbSmCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAt0klEQVR4nO3deZwsdX3v/9d7DoIrAlEJIlEkRCQBURE1uKEYMTcKKiremKBXfycm4hIf+lNjIqjhxi3uy81RiWiMqCiKiqIiaOLKQREEFwiogChXQTSiIOd87h9dB/t0z0x3nZmeqeK8no9HPbqr6ltdn5lm+cynvkuqCkmSJPXL3GoHIEmSpPZM4iRJknrIJE6SJKmHTOIkSZJ6yCROkiSph0ziJEmSesgkTpIkaQpJjktyRZJvLnA+Sd6Q5MIk5yS5x9C5I5Nc0GxHLkc8JnGSJEnTeSdwyCLnHw7s2WxrgbcCJNkJOBq4N3AAcHSSHZcajEmcJEnSFKrq88CVizQ5FHhXDXwZ2CHJLsDDgE9X1ZVVdRXwaRZPBqdiEidJkrQ8dgUuGdq/tDm20PEl2WapHzCNBx3zJtf22gqceMQDVzsErYC5m91itUPQCvnCFb9c7RC0Ah5xr32y2jFMYyVyic+95Bl/xeAx6CbrqmrdrO+7pVYkiZMkSeq6JmFbStJ2GbDb0P4dmmOXAQ8aOX7GEu4D+DhVkiRpuZwM/GUzSvU+wNVVdTlwKvAnSXZsBjT8SXNsSazESZIkTSHJexlU1G6T5FIGI05vAlBV/wc4BfhT4ELgGuDJzbkrk7wMOLP5qJdW1WIDJKZiEidJkjovWf2ue1X1hAnnC3j6AueOA45bznh8nCpJktRDVuIkSVLnzXWgEtc1VuIkSZJ6yEqcJEnqPAtx46zESZIk9ZBJnCRJUg+ZxEmSJPWQfeIkSVLnrZmz7jTK34gkSVIPWYmTJEmd14UVG7rGJE6SJHXenDncGB+nSpIk9ZCVOEmS1Hlzse40yt+IJElSD5nESZIk9ZBJnCRJUg/ZJ06SJHXenFOMjLESJ0mS1ENW4iRJUudZiBtnJU6SJKmHrMRJkqTOs0/cOCtxkiRJPWQSJ0mS1EMmcZIkST1knzhJktR5c3PWnUb5G5EkSeohK3GSJKnz5hycOsZKnCRJUg9ZiZMkSZ0X54kbYyVOkiSph6zESZKkzlsT606j/I1IkiT1kEmcJElSD/k4VZIkdZ7jGsZZiZMkSeohK3GSJKnz5izFjbESJ0mS1ENW4iRJUuc52e84K3GSJEk9ZCVOkiR13po5K3GjrMRJkiT1kEmcJElSD5nESZIk9ZB94iRJUuc5OnWclThJkqQeshInSZI6zxUbxlmJkyRJ6iErcZIkqfOsxI2zEidJktRDJnGSJEk9ZBInSZLUQ/aJkyRJnWefuHEmcZIkqfO6MNlvkkOA1wNrgLdX1ctHzr8WOKjZvTlwu6raoTm3ATi3OfeDqnrkUuMxiZMkSZogyRrgzcBDgUuBM5OcXFXnb2pTVX871P4ZwN2HPuJXVbXfcsbUuk9ckpsvZwCSJEmTJJn5NsEBwIVVdVFVXQecABy6SPsnAO9dph9/XlMncUn+OMn5wLeb/bslecvMIpMkSeqOXYFLhvYvbY6NSXJHYHfgs0OHb5pkfZIvJzlsOQJqU4l7LfAw4KcAVfUN4AELNU6ytgl2/Q/P+sLSopQkSVu1NXOZ+TacuzTb2i0M9wjgxKraMHTsjlW1P/A/gdcl2WOpv5NWj1Or6pKRQxvmbThou66q9q+q/W9/zwO3KDhJkqSVMpy7NNu6odOXAbsN7d+hOTafIxh5lFpVlzWvFwFnsHl/uS3SJom7JMkfA5XkJkmeC3xrqQFIkiT1wJnAnkl2T7Itg0Tt5NFGSfYCdgS+NHRsxyTbNe9vAxwInD96bVttRqc+jcGw2l0ZZJ6fAp6+1AAkSZK6rqquT3IUcCqDKUaOq6rzkrwUWF9VmxK6I4ATqqqGLr8r8C9JNjIooL18eFTrlpo6iauqnwB/vtQbSpIktdWFyX6r6hTglJFjLx7ZP2ae674I7LPc8UydxCV5wzyHr2aQfX5k+UKSJEnSJG0ep94U2Av4QLP/GOBi4G5JDqqqZy9zbJIkSUA3VmzomjZJ3L7AgZuGyyZ5K/AfwP347TISkiRJWgFtkrgdgVsyeIQKcAtgp6rakOTaZY9MkiSpsWbOStyoNkncK4Gzk5wBhMFEv/87yS2Az8wgNkmSJC2gzejUdyT5BPAXDOaH+xRwaVX9EnjejOKTJEkiWIkb1WZ06lOBZzGYofhs4D4MJrJ78EwikyRJ0oLarNjwLOBewPer6iAGy0X8bBZBSZIkaXFtkrhfV9WvAZJsV1XfBu4ym7AkSZK0mDYDGy5NsgPwYeDTSa4Cvj+LoCRJkoZ1YcWGrmkzsOFRzdtjkpwO3Br45EyikiRJ0qLaVOJuUFWfW+5AJEmSFmIhblybPnGSJEnqiC2qxEmSJK2kNXPWnUb5G5EkSeohkzhJkqQe8nGqJEnqvDiyYYyVOEmSpB6yEidJkjrPQtw4K3GSJEk9ZCVOkiR1nlOMjPM3IkmS1ENW4iRJUufNYae4UVbiJEmSesgkTpIkqYdM4iRJknrIPnGSJKnznCdunJU4SZKkHrISJ0mSOs+1U8dZiZMkSeohK3GSJKnz1sxZiRtlJU6SJKmHTOIkSZJ6yCROkiSph+wTJ0mSOs/RqeNM4iRJUufNmcSN8XGqJElSD1mJkyRJnecUI+OsxEmSJPWQlThJktR5DmwYZyVOkiSph0ziJEmSesgkTpIkqYfsEydJkjov2CdulJU4SZKkHrISJ0mSOs8VG8ZZiZMkSeohK3GSJKnzXLFhnJU4SZKkHrISJ0mSOs8VG8ZZiZMkSeohkzhJkqQeMomTJEnqIZM4SZLUeUlmvk0RwyFJvpPkwiQvmOf8k5L83yRnN9tTh84dmeSCZjtyOX4nDmyQJEmaIMka4M3AQ4FLgTOTnFxV5480fV9VHTVy7U7A0cD+QAFnNddetZSYViSJO/GIB67EbbTKDj/hc6sdglbASc9Zlj8g1QP333a71Q5BukEH5ok7ALiwqi4CSHICcCgwmsTN52HAp6vqyubaTwOHAO9dSkA+TpUkSZ03l8x8S7I2yfqhbe1QCLsClwztX9ocG/WYJOckOTHJbi2vbcXHqZIkSUBVrQPWLeEjPgq8t6quTfJXwPHAg5cluHlYiZMkSZrsMmC3of07NMduUFU/raprm923A/ec9totYRInSZI02ZnAnkl2T7ItcARw8nCDJLsM7T4S+Fbz/lTgT5LsmGRH4E+aY0vi41RJktR5c6u87FZVXZ/kKAbJ1xrguKo6L8lLgfVVdTLwzCSPBK4HrgSe1Fx7ZZKXMUgEAV66aZDDUpjESZIkTaGqTgFOGTn24qH3LwReuMC1xwHHLWc8JnGSJKnzppmMd2tjnzhJkqQeshInSZI6rwOT/XaOlThJkqQeshInSZI6zz5x46zESZIk9ZBJnCRJUg+ZxEmSJPWQfeIkSVLnBfvEjbISJ0mS1ENW4iRJUuc5T9w4K3GSJEk9ZCVOkiR13pzzxI2xEidJktRDVuIkSVLnuWLDOCtxkiRJPWQSJ0mS1EM+TpUkSZ3n49RxVuIkSZJ6yEqcJEnqPCf7HWclTpIkqYesxEmSpM6zT9w4K3GSJEk9ZBInSZLUQyZxkiRJPWSfOEmS1Hlz2CdulJU4SZKkHrISJ0mSOs954sZZiZMkSeohK3GSJKnznCdunJU4SZKkHrISJ0mSOs9K3DgrcZIkST1kEidJktRDJnGSJEk9ZJ84SZLUeU4TN85KnCRJUg9ZiZMkSZ23Zs660yiTOEmS1HlOMTLOtFaSJKmHrMRJkqTOsxA3zkqcJElSD5nESZIk9ZBJnCRJUg/ZJ06SJHXemlh3GuVvRJIkqYesxEmSpM5znrhxVuIkSZJ6yEqcJEnqvDkLcWOsxEmSJPWQSZwkSVIPmcRJkiT1kEmcJEnqvCQz36aI4ZAk30lyYZIXzHP+OUnOT3JOktOS3HHo3IYkZzfbycvxO5kqiUuyJsnfLscNJUmS+ibJGuDNwMOBvYEnJNl7pNnXgf2ral/gROCVQ+d+VVX7NdsjlyOmqZK4qtoAPGE5bihJktTWmrm5mW8THABcWFUXVdV1wAnAocMNqur0qrqm2f0ycIdl/0UMafM49QtJ3pTk/knusWlbqHGStUnWJ1n/rvefuAyhSpIkzc5w7tJsa4dO7wpcMrR/aXNsIU8BPjG0f9PmM7+c5LDliLfNPHH7Na8vHTpWwIPna1xV64B1AD/59rm1JcFJkiQBrMSCDcO5y1IkeSKwP/DAocN3rKrLktwZ+GySc6vqv5Zyn6mTuKo6aCk3kiRJ6rHLgN2G9u/QHNtMkoOBFwEPrKprNx2vqsua14uSnAHcHVhSEjf149QkOyd5R5JPNPt7J3nKUm4uSZI0jblk5tsEZwJ7Jtk9ybbAEcBmo0yT3B34F+CRVXXF0PEdk2zXvL8NcCBw/pJ/Jy3avhM4Fbh9s/9d4NlLDUCSJKnrqup64CgGudC3gPdX1XlJXppk02jTVwG3BD4wMpXIXYH1Sb4BnA68vKqWnMS16RN3m6p6f5IXbvphkmxYagCSJEl9UFWnAKeMHHvx0PuDF7jui8A+yx1PmyTul0l+h8FgBpLcB7h6uQOSJEkaNRfXJxjVJol7DoNnv3sk+QJwW+DwmUQlSZKkRbUZnfq1JA8E7gIE+E5V/WZmkUmSJDXmVmCKkb6ZmMQlefQCp/4gCVX1oWWOSZIkSRNMU4l7RPN6O+CPgc82+wcBXwRM4iRJ0kxNs0D91mZiEldVTwZI8ilg76q6vNnfhcG0I5IkSVphbYZ67LYpgWv8GPi9ZY5HkiRJU2gzOvW0JKcC7232Hw98ZvlDkiRJ0iRtRqce1QxyuH9zaF1VnTSbsCRJkn5rimWxtjptKnGbRqI6kEGSJGmVTZ3ENVW4VzAYpZpmq6rafkaxSZIkATDnRHFj2lTiXgk8oqq+NatgJEmSNJ02SdyPTeAkSdJqsE/cuDZJ3Pok7wM+DFy76aArNkiSJK28Nknc9sA1wJ8MHSsc6CBJkmbMFRvGtZli5MmzDESSJEnTm3rFhiR/kOS0JN9s9vdN8vezC02SJEkLabPs1tuAFwK/Aaiqc4AjZhGUJEmSFtemT9zNq+qrI8+kr1/meCRJksassU/cmDZJ3E+S7MFgMANJDgcun0lUkiRJQxzYMK5NEvd0YB2wV5LLgIuBP59JVJIkSVpUmyTuMOAU4HQGfel+CRyc5KyqOnv5Q5MkSRpwst9xbQY27A88DdgR2AH4K+AQ4G1J/v/lD02SJEkLaVOJuwNwj6r6b4AkRwMfBx4AnMVgbVVJkqRlZ5+4cW0qcbdjaLktBlON7FxVvxo5LkmSpBlrU4l7D/CVJB9p9h8B/HuSWwDnL3tkkiRJWlCbZbdeluQTwIHNoadV1frmvaNUJUmSVlCbShxN0rZ+YkNJkqRltGbOPnGj2vSJkyRJUke0qsRJkiStBkenjrMSJ0mS1ENW4iRJUue5YsM4K3GSJEk9ZBInSZLUQyZxkiRJPWSfOEmS1HlzzhM3xkqcJElSD1mJkyRJnTeHlbhRVuIkSZJ6yEqcJEnqPFdsGGcSJ0mSOs9xDeN8nCpJktRDJnGSJEk9ZBInSZLUQ/aJkyRJnTc3Z91plL8RSZKkHrISJ0mSOm/OKUbGWImTJEnqIStxkiSp8yzEjbMSJ0mS1ENW4iRJUuetcXTqGH8jkiRJPWQSJ0mSNIUkhyT5TpILk7xgnvPbJXlfc/4rSe40dO6FzfHvJHnYcsRjEidJkjRBkjXAm4GHA3sDT0iy90izpwBXVdXvA68FXtFcuzdwBPCHwCHAW5rPWxKTOEmS1Hlzmf02wQHAhVV1UVVdB5wAHDrS5lDg+Ob9icBDkqQ5fkJVXVtVFwMXNp+3JCsysGHuZrdYidtolZ30nCNXOwStgEe95vjJjXSjcPJRj1vtEKQVlWQtsHbo0LqqWte83xW4ZOjcpcC9Rz7ihjZVdX2Sq4HfaY5/eeTaXZcar6NTJUlS581VzfweTcK2bmLDjvBxqiRJ0mSXAbsN7d+hOTZvmyTbALcGfjrlta2ZxEmSpO6rjbPfFncmsGeS3ZNsy2CgwskjbU4GNvUtOhz4bFVVc/yIZvTq7sCewFeX+ivxcaokSdIETR+3o4BTgTXAcVV1XpKXAuur6mTgHcC7k1wIXMkg0aNp937gfOB64OlVtWGpMZnESZIkTaGqTgFOGTn24qH3vwYeu8C1xwLHLmc8Pk6VJEnqIStxkiSp82rDkp8+3uiYxEmSpO5bgSlG+sbHqZIkST1kJU6SJHXf5ClAtjpW4iRJknrISpwkSeq82mAlbpSVOEmSpB4yiZMkSeohkzhJkqQesk+cJEnqvHJ06hgrcZIkST1kJU6SJHWfKzaMsRInSZLUQ1biJElS99knboyVOEmSpB6yEidJkjqvNmxY7RA6x0qcJElSD5nESZIk9ZBJnCRJUg/ZJ06SJHWf88SNsRInSZLUQ1biJElS55WVuDEmcZIkqfOcYmScj1MlSZJ6yCROkiSph0ziJEmSesg+cZIkqftq42pH0DlW4iRJknrISpwkSeo+pxgZYyVOkiSph6zESZKkziv7xI2xEidJktRDVuIkSVL3uWLDGCtxkiRJPWQSJ0mS1EMmcZIkST1knzhJktR55TxxY6zESZIk9ZCVOEmS1H1W4sZYiZMkSeohK3GSJKnzynnixliJkyRJ6iGTOEmSpB7ycaokSeq+2rjaEXSOlThJkqQeshInSZI6z8l+x1mJkyRJ6iErcZIkqfvsEzfGSpwkSVIPWYmTJEmdVxusxI2yEidJktRDJnGSJElLlGSnJJ9OckHzuuM8bfZL8qUk5yU5J8njh869M8nFSc5utv0m3dMkTpIkaeleAJxWVXsCpzX7o64B/rKq/hA4BHhdkh2Gzj+vqvZrtrMn3XDqJC7Jy5JsM7S/fZJ/nfZ6SZKkLVYbZ78tzaHA8c3744HDxn6Equ9W1QXN+x8CVwC33dIbtqnEbQN8Jcm+SR4KnAmctVDjJGuTrE+y/vh/f++WxidJkrQihnOXZlvb4vKdq+ry5v2PgJ0n3OsAYFvgv4YOH9s8Zn1tku0m3XDq0alV9cIknwG+AlwFPKCqLlyk/TpgHcCV37/IaZYlSdIWW4kVG4Zzl/k0edDvznPqRSOfU0kWDDjJLsC7gSOrbigBvpBB8rdtE8PzgZcuFu/USVySBwBvaD5wH+CNSZ7SlAMlSZJu1Krq4IXOJflxkl2q6vImSbtigXbbAx8HXlRVXx767E1VvGub7mrPnRRPm3niXg08tqrOb4J4NPBZYK8WnyFJktTehg2rHcEkJwNHAi9vXj8y2iDJtsBJwLuq6sSRc5sSwDDoT/fNSTds0yfuvpsSOICq+hBwYIvrJUmSbqxeDjw0yQXAwc0+SfZP8vamzeOABwBPmmcqkfckORc4F7gN8I+TbtimErdHkrcy6Lj3R0n2BR45zU0kSZKWojq+dmpV/RR4yDzH1wNPbd7/G/BvC1z/4Lb3bFOJexuDTne/aW52DnBE2xtKkiRp6dokcTevqq+OHLt+OYORJEnSdNokcT9JsgdQAEkOBy5f/BJJkiTNQps+cU9nMG/JXkkuAy4GnjiTqCRJkoatwDxxfdNmst+LgIOT3AKYq6pfzC4sSZKkIR0f2LAaJiZxSZ6zwHEAquo1yxyTJEmSJpimEner5vUuwL0YTGYH8AhgdKCDJEnSsqsNVuJGTUziquolAEk+D9xj02PUJMcwWDZCkiRJK6zN6NSdgeuG9q9rjkmSJGmFtRmd+i7gq0lOavYPA45f9ogkSZI0UZvRqccm+SRwv+bQk6vq67MJS5Ik6be6vuzWamhTiQM4m8EEv9sAJPm9qvrBcgclSZKkxU2dxCV5BnA08GNgAxAGqzfsO5vQJEmSGk72O6ZNJe5ZwF2q6qezCkaSJEnTaZPEXQJcPatAJEmSFlIbNqx2CJ3TJom7CDgjyceBazcddMUGSZKkldcmiftBs23bbJIkSSvD0alj2kwx8pJZBiJJkqTpTUzikryuqp6d5KMMRqNupqoeOZPIJEmStKBpKnHvbl5fPctAJEmSNL2JSVxVndW8fm6xdkk+WFWPWa7AJEmSbuA8cWPmlvGz7ryMnyVJkqRFtF12azGmyJIkaSbKStyY5azESZIkaYUsZyUuy/hZkiRJN3DFhnFTV+KSPCLJYu2fvwzxSJIkaQptHqc+HrggySuT7DV6sqo+tXxhSZIkDamNs996ZuokrqqeCNwd+C/gnUm+lGRtklvNLDpJkiTNq9XAhqr6OXAicAKwC/Ao4GtJnjGD2CRJkrSAqQc2JDkUeBLw+8C7gAOq6ookNwfOB944kwglSZKcYmRMm9GpjwZeW1WfHz5YVdckecryhiVJkqTFtEnifjSawCV5RVU9v6pOW+a4JEmSbuAUI+Pa9Il76DzHHr5cgUiSJGl6EytxSf4a+BtgjyTnDJ26FfCFWQUmSZJ0A/vEjZnmceq/A58A/gl4wdDxX1TVlTOJSpIkSYuaJomrqvpekqePnkiyk4mcJEnSypu2EvdnwFlAsfkaqQXceQZxSZIkaRETk7iq+rPmdffZhyNJkjSuergs1qxNPTo1yaOS3Hpof4ckh80kKkmSJC2qzRQjR1fV1Zt2qupnwNHLHpEkSdKoqtlvPdMmiZuvbZvJgiVJkrRM2iRh65O8Bnhzs/90BoMdJEmSZsoVG8a1qcQ9A7gOeF+zXcsgkZMkSdIKm7oSV1W/ZPPJfiVJklaGo1PHTLPs1uuq6tlJPspgXrjNVNUjZxKZJEmSFjRNJe7dzeurZxmIJEmSpjfNZL9nNa+fm304kiRJmsbUfeKSHAgcA9yxuS4M1lV12S1JkjRT1cN53GatzRQj7wD+lsG0Io7zlSRJK8cpRsa0SeKurqpPzCwSSZIkTa1NEnd6klcBH2IwRxwAVfW1ZY9KkiRpiI9Tx7VJ4u7dvO4/dKyABy9fOJIkSZpGm8l+D5plIJIkSQvq+GS/SXZisKLVnYDvAY+rqqvmabcBOLfZ/cGm+XaT7A6cAPwOg/EHf1FV1y12z6mX3UqyXZL/meTvkrx40zbt9ZIkSTdiLwBOq6o9gdNYeJWrX1XVfs02vGDCK4DXVtXvA1cBT5l0wzZrp34EOBS4Hvjl0CZJkrS1OxQ4vnl/PHDYtBcmCYPuaSe2ub5Nn7g7VNUhLdrf4AtXmOttDe6/7XarHYJWwMlHPW61Q9AKeeSb3r/aIWgFnHHMUasdwo3FzlV1efP+R8DOC7S7aZL1DIpiL6+qDzN4hPqzqrq+aXMpsOukG7ZJ4r6YZJ+qOndyU0mSpOVTG2ffJy7JWmDt0KF1VbVu6PxngN+d59IXDe9UVSVZaDjtHavqsiR3Bj6b5Fzg6i2Jd2IS13x4NW2fnOQiBlOMbFqxYd8tubEkSVKXNAnbukXOH7zQuSQ/TrJLVV2eZBfgigU+47Lm9aIkZwB3Bz4I7JBkm6YadwfgsknxTlOJ+7Mp2kiSJM3Oxs6v2HAycCTw8ub1I6MNkuwIXFNV1ya5DXAg8Mqmcnc6cDiDEarzXj9q4sCGqvp+VX0f+MdN74ePtfjhJEmSbqxeDjw0yQXAwc0+SfZP8vamzV2B9Um+AZzOoE/c+c255wPPSXIhgz5y75h0wzZ94v5weCfJNsA9W1wvSZK0Rbq+YkNV/RR4yDzH1wNPbd5/EdhngesvAg5oc8+JlbgkL0zyC2DfJD9vtl8AP2aKUp8kSZKW3zSPU/+pqm4FvAZ4OvDPzf7dgZNmHJ8kSZLm0Way3+2B+wBHNPu/AN687BFJkiRpojZ94g6oqnsk+TpAVV2VZNsZxSVJkvRbKzBPXN+0qcT9JskaBnPGkeS2gL9RSZKkVdCmEvcGBn3gbpfkWAZzmfz9TKKSJEkaUt2fJ27FTZ3EVdV7kpzFYPhsgMOq6lszi0ySJEkLalOJo6q+DXx7RrFIkiTNr+PzxK2GVkmcJEnSaqiNJnGj2gxskCRJUkeYxEmSJPWQSZwkSVIP2SdOkiR1Xzk17SgrcZIkST1kJU6SJHWek/2OsxInSZLUQ1biJElS9znZ7xgrcZIkST1kEidJktRDJnGSJEk9ZJ84SZLUebXReeJGWYmTJEnqIStxkiSp+5wnboyVOEmSpB6yEidJkjqvnCdujJU4SZKkHrISJ0mSum+jlbhRVuIkSZJ6yCROkiSph3ycKkmSuq+c7HeUlThJkqQeshInSZI6r5zsd4yVOEmSpB6yEidJkjrPyX7HWYmTJEnqIStxkiSp+zY6OnWUlThJkqQeMomTJEnqIZM4SZKkHrJPnCRJ6ryyT9wYK3GSJEk9ZCVOkiR1n2unjrESJ0mS1ENW4iRJUvdtdMWGUVbiJEmSesgkTpIkqYdM4iRJknrIPnGSJKnzytGpY6zESZIk9ZCVOEmS1Hm1ccNqh9A5VuIkSZJ6yEqcJEnqvnKeuFFW4iRJUvdtrNlvS5BkpySfTnJB87rjPG0OSnL20PbrJIc1596Z5OKhc/tNuqdJnCRJ0tK9ADitqvYETmv2N1NVp1fVflW1H/Bg4BrgU0NNnrfpfFWdPemGJnGSJElLdyhwfPP+eOCwCe0PBz5RVdds6Q2nTuKSvCzJNkP72yf51y29sSRJ0o3IzlV1efP+R8DOE9ofAbx35NixSc5J8tok2026YZtK3DbAV5Lsm+ShwJnAWQs1TrI2yfok6z950oktbiNJkrS52rhh5ttw7tJsa4djSPKZJN+cZzt0s1irCliwk12SXYB9gFOHDr8Q2Au4F7AT8PxJv5OpR6dW1QuTfAb4CnAV8ICqunCR9uuAdQAfPfNch5RIkqROG85dFjh/8ELnkvw4yS5VdXmTpF2xyK0eB5xUVb8Z+uxNVbxrmyedz50Ub5vHqQ8A3gC8FDgDeGOS2097vSRJ0harjbPfluZk4Mjm/ZHARxZp+wRGHqU2iR9JwqA/3Tcn3bDNPHGvBh5bVec3N3k08FkGpT9JkqSt2cuB9yd5CvB9BtU2kuwPPK2qntrs3wnYDfjcyPXvSXJbIMDZwNMm3bBNEnffqrphzYuq+lCS0QAkSZKWXXV8st+q+inwkHmOrweeOrT/PWDXedo9uO092wxs2CPJaUm+CZBkX+Cv295QkiRJS9cmiXsbg5ETvwGoqnMYDI+VJEmarY0bZ7/1TJsk7uZV9dWRY9cvZzCSJEmaTpsk7idJ9qCZ9yTJ4cDli18iSZKkWWgzsOHpDOZO2SvJZcDFwBNnEpUkSZIW1Way34uAg5PcApirql/MLixJkqTfqo0bJjfaykxM4pI8Z4HjAFTVa5Y5JkmSJE0wTSXuVs3rXRis53Vys/8IYHSggyRJ0vLr+Dxxq2FiEldVLwFI8nngHpseoyY5Bvj4TKOTJEnSvNoMbNgZuG5o/7rmmCRJ0kzVRitxo9okce8CvprkpGb/MOD4ZY9IkiRJE7UZnXpskk8C92sOPbmqvj6bsCRJkrSYNpU4gLMZTPC7DUCS36uqHyx3UJIkSZtxipExUydxSZ4BHA38GNgAhMHqDfvOJjRJkiQtpE0l7lnAXarqp7MKRpIkaT5V/VugftbarJ16CXD1rAKRJEnS9NpU4i4CzkjyceDaTQddsUGSJM2ck/2OaZPE/aDZtm02SZIkrZI2U4y8ZJaBSJIkLaQ22idu1MQkLsnrqurZST7KYDTqZqrqkTOJTJIkSQuaphL37ub11bMMRJIkSdObmMRV1VnN6+cWa5fkg1X1mOUKTJIkSQtru2LDYu68jJ8lSZL0W/aJG9NmnrhJHPsrSZK0QpazEidJkjQTrtgwbjkrcVnGz5IkSdIitqgSl2RHYLeqOmfo8POXJyRJkqQRG+21NWrqSlySM5Jsn2Qn4GvA25LcsORWVX1qFgFKkiRpXJtK3K2r6udJngq8q6qOTnLOxKskSZKWqDZuWO0QOqdNn7htkuwCPA742IzikSRJ0hTaJHEvBU4FLqyqM5PcGbhgNmFJkiRpMVM/Tq2qDwAfGNq/CHCFBkmSpFUwdRKX5F+ZZ0LfqvpfyxqRJEnSKOeJG9NmYMNwP7ibAo8Cfri84UiSJI2rcoqRUW0ep35weD/Je4H/XPaIJEmSNNFSlt3aE7jdcgUiSZK0ICf7HdOmT9wv2LxP3I9wlQZJkqRV0eZx6q1mGYgkSZKm12bZrdOmOSZJkqTZm1iJS3JT4ObAbZqF79Oc2h7YdYaxSZIkAS67NZ9pHqf+FfBs4PbAWfw2ifs58KbZhCVJkqTFTEziqur1wOuTPKOq3rgCMUmSJG3OyX7HtBnY8MYkfwTszWCy303H3zWLwCRJkrSwNlOMHA08iEESdwrwcAaT/ZrESZKk2XKeuDFTj04FDgceAvyoqp4M3A249UyikiRJ0qLarNjwq6ramOT6JNsDVwC7zSguSZKkGzg6dVybJG59kh2AtzEYpfrfwJdmEZQkSZIW12Zgw980b/9Pkk8C21fVObMJS5IkSYvZohUbqup7VXWOKzZIkiStDldskCRJnVfOEzdmS1dsKOAXgJP/SpIkrYKJj1Or6vVVtTtwLLBf8/5fgYtwYIMkSVoJVbPfeqbVPHFV9fMk9wMeDLwdeOtswpIkSeqPJI9Ncl6SjUn2X6TdIUm+k+TCJC8YOr57kq80x9+XZNtJ92yTxG2aoOV/AG+rqo8DE28gSZK0ZBtr9tvSfBN4NPD5hRokWQO8mcGqV3sDT0iyd3P6FcBrq+r3gauAp0y6YZsk7rIk/wI8HjglyXYtr5ckSbpRqqpvVdV3JjQ7ALiwqi6qquuAE4BDk4TBU84Tm3bHA4dNumebJOxxwKnAw6rqZ8BOwPNaXC9JkrQ12xW4ZGj/0ubY7wA/q6rrR44vqs1kv9cAHxravxy4fJprH3GvfTK51Y1LkrVVtW6149Ds+V1vHbbG7/mMY45a7RBW3Nb4PffF/d7/hZnnEknWAmuHDq0b/uchyWeA353n0hdV1UdmHd+oNstuqZ21gP8h2Dr4XW8d/J63Dn7PW7EmYVvw+6+qg5d4i8vYfN35OzTHfgrskGSbphq36fii7NMmSZK0Ms4E9mxGom4LHAGcXFUFnA4c3rQ7EphY2TOJkyRJWqIkj0pyKXBf4ONJTm2O3z7JKQBNle0oBmMMvgW8v6rOaz7i+cBzklzIoI/cOybes3o4uV0f2K9i6+F3vXXwe946+D2rT0ziJEmSesjHqZIkST1kEictsyTvTHL45JaSJG05kzhplSVxqp8pJPm7ofd3SvLNltfvn+QNE9qckmSHZvubLYxz4n3mueaYJM/dkvttLZI8O8nNl6tdi/v6faqzTOKWIMmHk5zVLHi7tjn2lCTfTfLVJG9L8qbm+G2TfDDJmc124OpGL4Ak/9AsRPyfSd6b5LlJ9kjyyea7/Y8kezVt35nkDUm+mOSiTdW2DLyp+ZzPALcb+vx7Jvlc81mnJtmlOX5GktclWQ88azV+9h76u8lNFlZV66vqmRPa/GmzIs0OwBYlcdPcR1vk2cA0ydm07abi96kuM4lbmv9VVfcE9geemWRX4B+A+wAHAnsNtX09g4Vt7wU8Bnj7SgerzSXZ9F3cjcFixPs3p9YBz2i+2+cCbxm6bBfgfsCfAS9vjj0KuAuDxYz/Evjj5vNvArwROLz5rOOAY4c+a9uq2r+q/nn5f7p+S/LE5g+hs5P8S5JXATdr9t/TNFvT/KF0XpJPJblZc+0ZSV7RXP/dJPdvjj8oycea97dM8q9Jzk1yTpLHNMe/l+Q2DL7bPZr7vSrJu5IcNhTfe5IcukDsw/c5JslxTUwXJXnmULsXNfH9J4N/fjYdH/sjIsk2zR9/D2ra/FOSY7mRSnKLJB9P8o0k30xyNHB74PQkpzdt3ppkffP9v6Q59sx52v330OcenuSdzfvHNp/9jSSLLVju96nO8jHO0jwzyaOa97sBfwF8rqquBEjyAeAPmvMHA3snN6wasn2SW1bVf6PVciDwkar6NfDrJB8FbsogCfvA0He13dA1H66qjcD5SXZujj0AeG9VbQB+mOSzzfG7AH8EfLr5rDVsvlTd+2bwM/VekrsCjwcOrKrfJHkLcC7wq6rar2lzJ2BP4AlV9f8leT+DhPzfmo/ZpqoOSPKnwNEM/v0b9g/A1VW1T/N5O46cfwHwR0P3eyDwt8CHk9yawT8jR075I+0FHATcCvhOkrcC+zKY5HM/Bv8d/hpwVtN+HfC0qrogyb2Bt1TVg5M8CTgxyTOAQ4B7T3n/PjoE+GFV/Q+A5nf+ZOCgqvpJ0+ZFVXVlkjXAaUn2rao3JHnOSLuFvJjBWuCXJdmhRWx+n+oMk7gt1PwFdTBw36q6JskZwLeBuy5wyRxwnyZhUHfNMViEeL8Fzl879H7SOn4Bzquq+y5w/pctY9taPAS4J3Bmk/zeDLhinnYXV9XZzfuzgDsNnfvQAsc3OZjB/3QBqKqrFguoqj6X5C1JbssgWfzg0ELVk3y8qq4Frk1yBbAzcH/gpGZNapKc3LzekgX+iKiq85K8G/gYg//uXDfl/fvoXOCfk7wC+FhV/cfQ72OTx2XQjWUbBhXyvYFzWtzjC8A7mz8APjSp8RC/T3WGj1O33K2Bq5oEbi8Gj1BvATwwyY4ZdFZ/zFD7TwHP2LSTZL+VDFbz+gLwiCQ3bf5j+2fANcDFSR4LN/R3u9uEz/k88PgkazLo83ZQc/w7wG2T3Lf5rJsk+cOZ/CQ3LgGOr6r9mu0uVXXMPO2GE+oNbP5H6bULHF+KdwFPZFAROq7FdYvFOeqGPyKGtuE/DPcBfsZQv8sbo6r6LnAPBsncPyZ58fD5JLsz6OrwkKraF/g4gyr6vB839P6GNlX1NODvGTxFOSvJ70wZnt+nOsMkbst9EtgmybcY9J/5MoPFav838FUGCcL3gKub9s8E9s+g/835wNNWPGJtpqrOBE5m8Nf7Jxj8D+Nq4M+BpyT5BnAeMG/fpyEnARcA5zP4H/2Xms+/jsE6eK9oPutsmv5yWtRpwOFJbgeQZKckdwR+k0E/w+XwaeDpm3bmeZz6CwaPy4a9k0Gnearq/CXe//PAYUluluRWwCOaz/05C/wRkeTRwE4MHt+/seUjwF5Jcnvgmqr6N+BVDBK64e9kewaV7Kubbg0PH7p89Lv7cZK7Jplj0H910z32qKqvVNWLgf/L5ouSt+X3qVXh49Qt1JTTHz56PMn6qlrXVOJOAj7ctP8Jg34+6pZXV9UxGUxJ8HngrKq6mEEflc1U1ZNG9m/ZvBaDtfDGNI/7HjDP8QctNfAbq6o6P8nfA59q/sf7GwYJ1zrgnCRfA160xNv8I/DmDKYp2QC8hKFHalX10yRfaM5/oqqeV1U/bv5o+/AS701VfS3J+4BvMHhUfObQ6T8H3tr8Dm4CnJDkMgZ/LD6kqi7JYNT765m+X17f7AO8KslGBt//XzNYj/KTSX5YVQcl+TqDLiyXMPijeZN1w+0Y9G/8GINEbT1wy6bdq5LsyaDyexqD72KL+H1qtbjs1jJL8moG/W1uyuAR6rPKX3JnJfl3Bn1pbsrgEd4/rXJI6qgm0T8XuEdVXT2pvSTNmkmcJE2Q5GDgHQymCXrdKocjSYBJnCRtkSQPA14xcvjiqnrUfO3VbX6f6iOTOEmSpB5ydKokSVIPmcRJkiT1kEmcJElSD5nESZIk9ZBJnCRJUg/9PzqJK8Js1wYAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x648 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(15, 9)) \n",
    "\n",
    "ax = sns.heatmap(\n",
    "    corr, \n",
    "    vmin=-1, vmax=1, center=0,\n",
    "    cmap=sns.diverging_palette(20, 220, n=200),\n",
    "    square=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "planned-entity",
   "metadata": {},
   "source": [
    "> Lets pick all of them"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "complicated-tanzania",
   "metadata": {},
   "source": [
    "### Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "coordinate-surge",
   "metadata": {},
   "outputs": [],
   "source": [
    "stages = []\n",
    "categoral_cols = [\"ethnicity\", \"status\"]\n",
    "\n",
    "num_cols = [\"age\", \"gender\",]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "colored-automation",
   "metadata": {},
   "outputs": [],
   "source": [
    "for categoricalCol in categoral_cols:\n",
    "    stringIndexer = StringIndexer(inputCol = categoricalCol, outputCol = categoricalCol + 'Index').setHandleInvalid(\"skip\")\n",
    "    encoder = OneHotEncoder(inputCols=[stringIndexer.getOutputCol()], outputCols=[categoricalCol + \"classVec\"])\n",
    "    stages += [stringIndexer, encoder]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hispanic-terrain",
   "metadata": {},
   "source": [
    "> Normal cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "brutal-blake",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(inputCol=\"description\", outputCol=\"words\")\n",
    "remover = StopWordsRemover(inputCol=\"words\", outputCol=\"filtered\")\n",
    "\n",
    "cv = CountVectorizer(vocabSize=2**16, inputCol=\"filtered\", outputCol='cv')\n",
    "idf = IDF(inputCol='cv', outputCol=\"idf_features\", minDocFreq=5) #minDocFreq: remove sparse terms\n",
    "\n",
    "stages.append(tokenizer)\n",
    "stages.append(remover)\n",
    "stages.append(cv)\n",
    "stages.append(idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "altered-replica",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_stringIdx = StringIndexer(inputCol = 'scam', outputCol = 'label').setHandleInvalid(\"skip\")\n",
    "\n",
    "stages.append(label_stringIdx)\n",
    "\n",
    "assemblerInputs = [c + \"classVec\" for c in categoral_cols] + num_cols + [\"idf_features\"]\n",
    "assembler = VectorAssembler(inputCols=assemblerInputs, outputCol=\"features\").setHandleInvalid(\"skip\")\n",
    "\n",
    "stages.append(assembler)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "clear-refrigerator",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[age: int, ethnicity: string, gender: int, scam: bigint, status: string, description: string]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "profiles_df.withColumn(\"description\", F.when(F.col(\"description\").isNull(), F.lit(\"\")).otherwise(F.col(\"description\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "pressing-intro",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- age: integer (nullable = true)\n",
      " |-- ethnicity: string (nullable = true)\n",
      " |-- gender: integer (nullable = false)\n",
      " |-- scam: long (nullable = true)\n",
      " |-- status: string (nullable = true)\n",
      " |-- description: string (nullable = true)\n",
      " |-- label: double (nullable = false)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pipeline = Pipeline(stages=stages)\n",
    "pipelineModel = pipeline.fit(profiles_df.withColumn(\"description\", F.when(F.col(\"description\").isNull(), F.lit(\"\")).otherwise(F.col(\"description\"))))\n",
    "df = pipelineModel.transform(profiles_df.withColumn(\"description\", F.when(F.col(\"description\").isNull(), F.lit(\"\")).otherwise(F.col(\"description\"))))\n",
    "df = df.drop(\"ethnicityIndex\", \"ethnicityclassVec\", \"statusclassVec\", \"words\", \"statusIndex\", \"tf\", \"idf\", \"filtered\", \"cv\", \"idf_features\")\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "monetary-durham",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Dataset Count: 29761\n",
      "Test Dataset Count: 12704\n"
     ]
    }
   ],
   "source": [
    "train, test = df.randomSplit([0.7, 0.3], seed = 2000)\n",
    "print(\"Training Dataset Count: \" + str(train.count()))\n",
    "print(\"Test Dataset Count: \" + str(test.count()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "level-genre",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-----+\n",
      "|scam|count|\n",
      "+----+-----+\n",
      "|   0|25075|\n",
      "|   1| 4686|\n",
      "+----+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train.groupBy(\"scam\").count().show(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bridal-elevation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----+-----+\n",
      "|scam|count|\n",
      "+----+-----+\n",
      "|   0|10754|\n",
      "|   1| 1950|\n",
      "+----+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.groupBy(\"scam\").count().show(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "confirmed-sheep",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression\n",
    "lr = LogisticRegression(featuresCol='features', labelCol='label', maxIter=10)\n",
    "lrModel = lr.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "suburban-wilderness",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------+------+----+------+-----------+-----+--------------------+--------------------+--------------------+----------+\n",
      "|age|ethnicity|gender|scam|status|description|label|            features|       rawPrediction|         probability|prediction|\n",
      "+---+---------+------+----+------+-----------+-----+--------------------+--------------------+--------------------+----------+\n",
      "| 19|    asian|     1|   0|single|           |  0.0|(18304,[4,12,18,1...|[4.32864819825129...|[0.98698623177156...|       0.0|\n",
      "| 19|    asian|     1|   0|single|           |  0.0|(18304,[4,12,18,1...|[4.32864819825129...|[0.98698623177156...|       0.0|\n",
      "+---+---------+------+----+------+-----------+-----+--------------------+--------------------+--------------------+----------+\n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = lrModel.transform(test)\n",
    "predictions.show(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "discrete-nylon",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.9652153521885705\n"
     ]
    }
   ],
   "source": [
    "evaluator = BinaryClassificationEvaluator()\n",
    "print('Accuracy', evaluator.evaluate(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "parallel-performance",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "warming-procedure",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "conscious-bride",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "motivated-conservation",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}