CSP_572 v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# Balance classes\n",
        "# outliers, scaling,\n",
        "# EDA\n",
        "#"
      ],
      "metadata": {
        "id": "U7oPCw7jLG1k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FuRXIO8jTrbi",
        "outputId": "d11fd98e-9fb3-4790-c2c6-01cd459bd1b6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "# Mount to google drive where data is stored\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## PySpark Setup"
      ],
      "metadata": {
        "id": "lU1YuEycWy28"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# If you encounter any problems visit the apache.org link and ensure you have selected the right tgz file\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://www.apache.org/dist/spark/spark-3.2.4/spark-3.2.4-bin-hadoop2.7.tgz\n",
        "!tar xf spark-3.2.4-bin-hadoop2.7.tgz\n"
      ],
      "metadata": {
        "id": "zCPeiUI9AQtn"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#import shutil\n",
        "\n",
        "#shutil.rmtree('/content/gdrive/MyDrive/CSP 572/spark-3.2.4-bin-hadoop2.7')"
      ],
      "metadata": {
        "id": "QKVtxvWDkBU2"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Setting the appropriate paths for Spark\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.2.4-bin-hadoop2.7\"\n",
        "\n",
        "!pip install -q findspark\n",
        "import findspark\n",
        "findspark.init()"
      ],
      "metadata": {
        "id": "_EbDt5I71w23"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import libraries"
      ],
      "metadata": {
        "id": "2IhdtDu6W4uG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "8km2Nvkc4Pgr"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from pyspark.sql.functions import *\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import col, lit, to_timestamp, log\n",
        "from pyspark.sql import Window\n",
        "import pyspark.sql.functions as f\n",
        "from pyspark.sql import Window\n",
        "from pyspark.sql import functions as F\n",
        "from pyspark.sql.types import StringType, DateType, FloatType\n",
        "import pandas as pd\n",
        "from pyspark.sql.functions import mean as _mean, stddev as _stddev, col"
      ],
      "metadata": {
        "id": "msPS2l2x4C7N"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.stat import Correlation\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "from pyspark.ml.classification import RandomForestClassifier\n",
        "#!pip install pyspark_dist_explore\n",
        "#from pyspark_dist_explore import hist\n",
        "#import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "so2vChh4Ui5f"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix, accuracy_score, f1_score, precision_score, recall_score, ConfusionMatrixDisplay"
      ],
      "metadata": {
        "id": "69RHxCCBx0iC"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Change directory\n",
        "import os\n",
        "os.chdir(\"/content/gdrive/MyDrive/CSP 572/\")"
      ],
      "metadata": {
        "id": "gPEJS7FD4HH0"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Create a spark session\n",
        "spark = SparkSession.builder\\\n",
        "        .master(\"local\")\\\n",
        "        .appName(\"Colab\")\\\n",
        "        .config('spark.ui.port', '4050')\\\n",
        "        .getOrCreate()\n"
      ],
      "metadata": {
        "id": "ib2wLRBD4J1w"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Read Data"
      ],
      "metadata": {
        "id": "1h6L8C0ZXEgR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Read 2021 data\n",
        "data = spark.read.csv(\"2021.rpt\", header=None, sep=\"|\")"
      ],
      "metadata": {
        "id": "9VYaBrvX4RFc"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Read the header file\n",
        "header = pd.read_csv(\"Header.rpt\", sep=\"|\")\n",
        "cols = header.columns\n",
        "data = data.toDF(*cols)\n"
      ],
      "metadata": {
        "id": "HVR-a9Em84jH"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Read all other data files\n",
        "#data_2021 = spark.read.csv(\"2022.rpt\", header=None, sep=\"|\")\n",
        "data_2020 = spark.read.csv(\"2020.rpt\", header=None, sep=\"|\")\n",
        "data_2019 = spark.read.csv(\"2019.rpt\", header=None, sep=\"|\")\n",
        "data_2018 = spark.read.csv(\"2018.rpt\", header=None, sep=\"|\")\n",
        "data_2017 = spark.read.csv(\"2017.rpt\", header=None, sep=\"|\")"
      ],
      "metadata": {
        "id": "xZLp4QWi4weP"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Combine all data\n",
        "#data = data.union(data_2022)\n",
        "data = data.union(data_2020)\n",
        "data = data.union(data_2019)\n",
        "data = data.union(data_2018)\n",
        "data = data.union(data_2017)"
      ],
      "metadata": {
        "id": "fXy29qJa5EWo"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Numeric columns list - add to this as you see fit\n",
        "numeric_cols = [\"AdjustedAmountDue1\", \"TaxAmountDue1\", \"InterestAmountDue1\", \"CostAmountDue1\", \"TotalAmountDue1\", \"OriginalTaxDue2\",\n",
        "        \"AdjustedTaxDue2\", \"TaxAmountDue2\", \"InterestAmountDue2\", \"CostAmountDue2\", \"TotalAmountDue2\", \"AssessedValuation\",\n",
        "        \"EqualizedEvaluation\", \"TaxRate\", \"LastPaymentReceivedAmount1\", \"LastPaymentReceivedAmount2\"\n",
        "        ] + list(data.columns)[70:81]"
      ],
      "metadata": {
        "id": "LSDbhNsZ81CN"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert to numeric columns\n",
        "for col_name in numeric_cols:\n",
        "    data = data.withColumn(col_name, col(col_name).cast('float'))\n",
        "\n",
        "\n",
        "# Remove 3 rows which have garbage values\n",
        "data = data.where(data.SegmentCode==\"PH\")"
      ],
      "metadata": {
        "id": "DShj7KV56qi2"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Target Variable\n",
        "Our target variable is Late Payment in 2nd Installment Y/N. We get this from the InstDates.csv (available in 'Helper datasets' folder in the shared drive) which contains the payment due dates of each year for each installment."
      ],
      "metadata": {
        "id": "CfS-US7ytEcv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "instDates = spark.read.csv(\"InstDates.csv\", header=True)\n",
        "instDates = instDates.withColumn(\"Year\", col(\"Year\").cast(\"int\"))\n",
        "instDates = instDates.where(instDates.Year>=2017)\n",
        "instDates = instDates.withColumnRenamed(\"Inst1\", \"PmtDue1\")\n",
        "instDates = instDates.withColumnRenamed(\"Inst2\", \"PmtDue2\")"
      ],
      "metadata": {
        "id": "O7ItNlcvLIEY"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = data.join(instDates, data.TaxYear==instDates.Year, \"left\")\n",
        "data = data.withColumn(\"LastPaymentDate1\", to_date(col(\"LastPaymentDate1\"), \"MMddyyyy\"))\n",
        "data = data.withColumn(\"LastPaymentDate2\", to_date(col(\"LastPaymentDate2\"), \"MMddyyyy\"))\n",
        "data = data.withColumn(\"PmtDue1\", to_date(col(\"PmtDue1\"), \"yyyy-MM-dd\"))\n",
        "data = data.withColumn(\"PmtDue2\", to_date(col(\"PmtDue2\"), \"yyyy-MM-dd\"))\n",
        "data = data.withColumn(\"LatePmt1\", when(col(\"LastPaymentDate1\")>col(\"PmtDue1\"), 1).otherwise(0))\n",
        "data = data.withColumn(\"LatePmt2\", when(col(\"LastPaymentDate2\")>col(\"PmtDue2\"), 1).otherwise(0))\n"
      ],
      "metadata": {
        "id": "cPU8y9a8Z0ZP"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Persist data in disk/memory\n",
        "# Until this point Spark does not actually execute any of the above statements (Lazy loading). It only creates a plan\n",
        "# Persist does not get executed until an 'action' command is called on the dataframe\n",
        "#data1 = data.persist()"
      ],
      "metadata": {
        "id": "GZpf0TBf95Kc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# The below action with persist called above will ensure that all the steps until data.persist() is run and stored in memory/disk\n",
        "# Only the first run takes approx 10 mins. If you run it again, it will only take 3 secs as the data is now stored in memory/disk\n",
        "# It helps in using the modified dataframe directly\n",
        "# If you do not persist, Spark is going to run all the steps right from read_csv whenever you call an action\n",
        "#print([data1.count(), len(data1.columns)])"
      ],
      "metadata": {
        "id": "88QBnsW84a7y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#data1.where(col(\"LatePmt2\")==1).count()"
      ],
      "metadata": {
        "id": "6zXcXwuiL7Z9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data1 = data"
      ],
      "metadata": {
        "id": "IAPqTq0lcJe8"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Payment Master data\n",
        "Please do not run this section as it is not used yet. Proceed to 'Drop columns' part if you want to run the whole model."
      ],
      "metadata": {
        "id": "J-JvxAvOZaJp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "pmt_2021 = spark.read.csv(\"TY2021.rpt\", sep=\",\", header=True)"
      ],
      "metadata": {
        "id": "UcgfAi4nP7Pd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print([pmt_2021.count(), len(pmt_2021.columns)])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fAlxz58mQcLi",
        "outputId": "b844f0e1-18cd-4d30-8cd8-ea7c9c1eeec9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[3339581, 17]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pmt_2021.columns"
      ],
      "metadata": {
        "id": "1o4s58AeQgCh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pmt_2021.where(col(\"PIN\")==\"09143170550000\").show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f0P2hzF0QOmn",
        "outputId": "4127b0e4-7563-454c-a1fa-309890f2b2db"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------+--------------+-------+--------------------+-------+----------+-------+------------+--------+----------------+----------+-------------------+------------+---------+--------+-----------+\n",
            "|Payment|Volume|           PIN|TaxType|            TaxPayer|TaxYear|  DatePaid|TaxPaid|InterestPaid|CostPaid|    SerialNumber|  SourceID|        DateUpdated|RefundNumber|TotalPaid|TaxYear4|WarrantYear|\n",
            "+-------+------+--------------+-------+--------------------+-------+----------+-------+------------+--------+----------------+----------+-------------------+------------+---------+--------+-----------+\n",
            "|     P1|   088|09143170550000|      0|DIANE S FAGIANO  ...|   21  |230209    |2422.03|      435.96|    0.00|021023500337    |750       |2023-02-11 05:31:00|        NULL|  2857.99|    2021|       2021|\n",
            "|     P2|   088|09143170550000|      0|DIANE S FAGIANO  ...|   21  |230209    |2234.76|       67.04|    0.00|021023500337    |750       |2023-02-11 05:31:00|        NULL|  2301.80|    2021|       2021|\n",
            "+-------+------+--------------+-------+--------------------+-------+----------+-------+------------+--------+----------------+----------+-------------------+------------+---------+--------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Analysis\n",
        "Just some EDA on the dataset. TProceed to 'Drop columns' part if you want to run the whole model."
      ],
      "metadata": {
        "id": "MvzAaYzNXLjl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data1 = data1.withColumn(\"TaxSale\", when(data1.LastPaymentDate2==\"00000000\", 1).otherwise(0))"
      ],
      "metadata": {
        "id": "INani_fbiuSa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tot = data1.count()"
      ],
      "metadata": {
        "id": "z3ZJW88uGszN"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Univariate analysis\n",
        "var = \"LastPaymentSource1\"\n",
        "data1.groupBy(var).agg(count(var), count(var)/tot*100).show()"
      ],
      "metadata": {
        "id": "kqUkJeCx5-Re",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e9c877c7-af23-424a-9f4b-865a9a52f2db"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------------------+-----------------------------+-------------------------------------------------+\n",
            "|TaxpayerPropertySuffix|count(TaxpayerPropertySuffix)|((count(TaxpayerPropertySuffix) / 9325500) * 100)|\n",
            "+----------------------+-----------------------------+-------------------------------------------------+\n",
            "|                     1|                      1452585|                               15.576483834646936|\n",
            "|                     2|                      6899714|                                73.98760388182939|\n",
            "|                     0|                       973201|                                10.43591228352367|\n",
            "+----------------------+-----------------------------+-------------------------------------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where(data1.PastTaxSaleStatus==\"Y\").show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C5XBCqzqFqTw",
        "outputId": "49e2a93c-9b8f-43dd-a544-97fc35b0d525"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|SegmentCode|           PIN|Volume|Classification|        TaxpayerName|TaxpayerMailingAddress| TaxpayerMailingCity|TaxpayerMailingState|TaxpayerMailingZip|TaxpayerPropertyHouse|TaxpayerPropertyDirection|TaxpayerPropertyStreet|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyState|TaxpayerPropertyZip|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|LongtimeHomeownersExempt|TaxInfoType|TaxType|TaxYear|BillYear|AccountStatus|BillType|SegmentCode2|InstallmentNumber1|AdjustedAmountDue1|TaxAmountDue1|RefundTaxAmountDueIndicator1|InterestAmountDue1|RefundInterestDueIndicator1|CostAmountDue1|RefundCostDueIndicator1|TotalAmountDue1|RefundTotalDueIndicator1|LastPaymentDate1|LastPaymentSource1|InstallmentNumber2|OriginalTaxDue2|AdjustedTaxDue2|TaxAmountDue2|RefundTaxAmountDueIndicator2|InterestAmountDue2|RefundInterestDueIndicator2|CostAmountDue2|RefundCostDueIndicator2|TotalAmountDue2|RefundTaxDueIndicator2|LastPaymentDate2|LastPaymentSource2|CofENumber|PastTaxSaleStatus|AssessedValuation|EqualizedEvaluation|TaxRate|RecordCount|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|LastPaymentReceivedAmount1|LastPaymentReceivedAmount2|EndMarker|TaxDueEstimated1|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|VeteranExempt|AdjustedAmountDue1_Org|TaxSale|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|         PH|12264020260000|   067|           202|CHARLES EVANS    ...|  2116 KENTON LN   ...|LIBERTYVILLE     ...|                  IL|         600481500|           0000002718|                       N |       BUDD           |                  ST  |RIVER GROVE      ...|                   IL|          601711516|     LEYDEN         |  20254|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          205311.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04242023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|           9928.0|            29811.0|    0.0|  000483460|                 0|                         0|                0|           0|             0|           0|                  205311.0|                       0.0|        X|        205311.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              205311.0|      1|\n",
            "|         PH|12283140050000|   069|           204|NORMA MARCONI    ...|  2724 SARAH       ...|FRANKLIN PK      ...|                  IL|         601313156|           0000002724|                         |       SARAH          |                  ST  |FRANKLIN PARK    ...|                   IL|          601313156|     LEYDEN         |  20029|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          671295.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04242023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          32358.0|            97161.0|    0.0|  000488609|                 0|                         0|                0|           0|             0|           0|                  671295.0|                       0.0|        X|        671295.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              671295.0|      1|\n",
            "|         PH|13011020360000|   316|           241|TAXPAYER OF      ...|  6307 N ALBANY AVE...|CHICAGO          ...|                  IL|         606591403|           0000006307|                       N |       ALBANY         |                  AVE |CHICAGO          ...|                   IL|          606591403|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|           41143.0|      41143.0|                           P|            1851.0|                          P|           0.0|                      P|        42994.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|           3720.0|            11170.0|    0.0|  000501012|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|         41143.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               41143.0|      1|\n",
            "|         PH|13011060371003|   316|           299|CHICAGO TITLE LAN...|  10 S LA SALLE ST#...|CHICAGO          ...|                  IL|         606031108|           0000006333|                       N |       FRANCISCO      |                  AVE |CHICAGO          ...|                   IL|          606591527|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          176937.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03232023|               444|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          15998.0|            48037.0|    0.0|  000501272|                 0|                         0|                0|           0|             0|           0|                  176937.0|                       0.0|        X|        176937.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              176937.0|      1|\n",
            "|         PH|13011080551004|   316|           299|ALLEN GREEN      ...|  6239 N KEDZIE  1S...|CHICAGO          ...|                  IL|         606591309|           0000006239|                       N |       KEDZIE         |                  AVE |CHICAGO          ...|                   IL|          606591309|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          188020.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04252023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          17000.0|            51046.0|    0.0|  000501330|                 0|                         0|                0|           0|             0|           0|                  188020.0|                       0.0|        X|        188020.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              188020.0|      1|\n",
            "|         PH|13021050310000|   318|           205|JOSEPHINE LUPU   ...|  6322 N MONTICELLO...|CHICAGO          ...|                  IL|         606591210|           0000006322|                       N |       MONTICELLO     |                  AVE |CHICAGO          ...|                   IL|          606591210|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          355177.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04252023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          38003.0|            96112.0|    0.0|  000505935|                 0|                         0|                0|           0|             0|           0|                  355177.0|                       0.0|        X|        355177.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              355177.0|      1|\n",
            "|         PH|13031190080000|   319|           204|DAVID & MARIE CHI...|  6171 N LEMONT AVE...|CHICAGO          ...|                  IL|         606464954|           0000006167|                       N |       LEMONT         |                  AVE |CHICAGO          ...|                   IL|          606464954|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          411230.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04242023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          37060.0|           111280.0|    0.0|  000508927|                 0|                         0|                0|           0|             0|           0|                  411230.0|                       0.0|        X|        411230.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              411230.0|      1|\n",
            "|         PH|13021080270000|   318|           203|ILYAS A LAKADA   ...|  4001 W DEVON AVE#...|CHICAGO          ...|                  IL|         606464523|           0000006220|                       N |       SPRINGFIELD    |                  AVE |CHICAGO          ...|                   IL|          606591016|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          388418.0|     388418.0|                           P|           17478.0|                          P|           0.0|                      P|       405896.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          35004.0|           105107.0|    0.0|  000506022|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        388418.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              388418.0|      1|\n",
            "|         PH|13021150280000|   318|           278|LAKADA 6142 HARDI...|  5700 N LINCOLN #2...|CHICAGO          ...|                  IL|         606594731|           0000006142|                       N |       HARDING        |                  AVE |CHICAGO          ...|                   IL|          606593108|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          151717.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04242023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          17003.0|            41055.0|    0.0|  000506218|                 0|                         0|                0|           0|             0|           0|                  151717.0|                       0.0|        X|        151717.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              151717.0|      1|\n",
            "|         PH|13032120240000|   319|           203|CHONG W & YUN S P...|  6226 N KEELER AV ...|CHICAGO          ...|                  IL|         606465110|           0000006226|                       N |       KEELER         |                  AVE |CHICAGO          ...|                   IL|          606465110|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          410623.0|     410623.0|                           P|           18477.0|                          P|           0.0|                      P|       429100.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          43000.0|           111116.0|    0.0|  000509597|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        410623.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              410623.0|      1|\n",
            "|         PH|13032040270000|   319|           203|LOLITA M STANTON ...|  3336 S PRAIRIE AV...|BELOIT           ...|                  WI|         535118843|           0000006336|                       N |       KEELER         |                  AVE |CHICAGO          ...|                   IL|          606464508|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          377276.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04252023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          34000.0|           102092.0|    0.0|  000509346|                 0|                         0|                0|           0|             0|           0|                  377276.0|                       0.0|        X|        377276.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              377276.0|      1|\n",
            "|         PH|13051020301024|   321|           299|JAZZ JASPER      ...|  6331 W HIGHLAND A...|CHICAGO          ...|                  IL|         606463739|           0000006333|                       N |       MILWAUKEE      |                  AVE |CHICAGO          ...|                   IL|          606463744|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          195476.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02142023|               501|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          17674.0|            53070.0|    0.0|  000512974|                 0|                         0|                0|           0|             0|           0|                  195476.0|                       0.0|        X|        195476.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              195476.0|      1|\n",
            "|         PH|13091240290000|   328|           234|STANLEY A LASKOWS...|  5454 N LYNCH AVE ...|CHICAGO          ...|                  IL|         606301416|           0000005305|                       W |       CATALPA        |                  AVE |CHICAGO          ...|                   IL|          606301313|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          144288.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04212023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          14711.0|            39173.0|    0.0|  000527962|                 0|                         0|                0|           0|             0|           0|                  144288.0|                       0.0|        X|        144288.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              144288.0|      1|\n",
            "|         PH|11194010451017|   059|           299|M NUR & H MOHAMED...|  811 CHICAGO AVE #...|EVANSTON         ...|                  IL|         602022392|           0000000811|                         |       CHICAGO        |                  AVE |EVANSTON         ...|                   IL|          602022391|     EVANSTON       |  17041|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          441568.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04212023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          29613.0|            88919.0|    0.0|  000422807|                 0|                         0|                0|           0|             0|           0|                  441568.0|                       0.0|        X|        441568.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              441568.0|      1|\n",
            "|         PH|13121150481035|   332|           299|BERWYN SUMMERDALE...|  2910-20 W BERWYN ...|CHICAGO          ...|                  IL|         606254025|           0000002910|                       W |       BERWYN         |                  AVE |CHICAGO          ...|                   IL|          606254025|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|           11050.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04252023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|            999.0|             3000.0|    0.0|  000537744|                 0|                         0|                0|           0|             0|           0|                   11050.0|                       0.0|        X|         11050.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               11050.0|      1|\n",
            "|         PH|13314230531001|   365|           299|PHOENIX BOND & IN...|  161 N CLARK #3040...|CHICAGO          ...|                  IL|         606013335|           0000001614|                       N |       NAGLE          |                  AVE |CHICAGO          ...|                   IL|          607074015|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|           11039.0|      11039.0|                           P|             498.0|                          P|           0.0|                      P|        11537.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|            998.0|             2997.0|    0.0|  000623788|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|         11039.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               11039.0|      1|\n",
            "|         PH|13113140030000|   331|           202|ZACHARY A GOTHARD...|  4923 N HARDING AV...|CHICAGO          ...|                  IL|         606256104|           0000004921|                       N |       HARDING        |                  AVE |CHICAGO          ...|                   IL|          606256104|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          321781.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03102023|               802|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          28999.0|            87075.0|    0.0|  000535349|                 0|                         0|                0|           0|             0|           0|                  321781.0|                       0.0|        X|        321781.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              321781.0|      1|\n",
            "|         PH|13114190301012|   332|           299|JOSEPH K WHARRAM ...|  4948 NORTH KIMBAL...|CHICAGO          ...|                  IL|         606255176|           0000004956|                       N |       KIMBALL        |                  AVE |CHICAGO          ...|                   IL|          606255111|     JEFFERSON      |  71128|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|           11075.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04162023|               700|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|            998.0|             2997.0|    0.0|  000536720|                 0|                         0|                0|           0|             0|           0|                   11075.0|                       0.0|        X|         11075.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               11075.0|      1|\n",
            "|         PH|13124100070000|   333|           204|CHRIST MOURLAS   ...|  2539 W WINNEMAC A...|CHICAGO          ...|                  IL|         606252615|           0000002539|                       W |       WINNEMAC       |                  AVE |CHICAGO          ...|                   IL|          606252615|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          704175.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04202023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          66999.0|           191178.0|    0.0|  000541686|                 0|                         0|                0|           0|             0|           0|                  704175.0|                       0.0|        X|        704175.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              704175.0|      1|\n",
            "|         PH|13124111310000|   333|           100|5035 LINCOLN, LLC...|  5035 N LINCOLN AV...|CHICAGO          ...|                  IL|         606252611|           0000005029|                       N |       LINCOLN        |                  AVE |CHICAGO          ...|                   IL|          606252611|     JEFFERSON      |  71158|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          240516.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04242023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          20835.0|            62561.0|    0.0|  000541932|                 0|                         0|                0|           0|             0|           0|                  240516.0|                       0.0|        X|        240516.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              240516.0|      1|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where(data1.PastTaxSaleStatus==\"Y\").count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5j9SlYFITa93",
        "outputId": "d3986eb0-715e-441b-9427-556451e4907b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "272729"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where((data1.PastTaxSaleStatus==\"Y\") & (data1.TaxSale==1)).count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7YP0s10zbYJV",
        "outputId": "a6d855e6-e209-4e88-822e-d17f1f2bd985"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "195448"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where((data1.PastTaxSaleStatus==\"Y\") & (data1.TaxSale==0)).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r-lSNepqbhfi",
        "outputId": "2d45b55a-fd68-4a4b-ea28-83457df9396a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|SegmentCode|           PIN|Volume|Classification|        TaxpayerName|TaxpayerMailingAddress| TaxpayerMailingCity|TaxpayerMailingState|TaxpayerMailingZip|TaxpayerPropertyHouse|TaxpayerPropertyDirection|TaxpayerPropertyStreet|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyState|TaxpayerPropertyZip|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|LongtimeHomeownersExempt|TaxInfoType|TaxType|TaxYear|BillYear|AccountStatus|BillType|SegmentCode2|InstallmentNumber1|AdjustedAmountDue1|TaxAmountDue1|RefundTaxAmountDueIndicator1|InterestAmountDue1|RefundInterestDueIndicator1|CostAmountDue1|RefundCostDueIndicator1|TotalAmountDue1|RefundTotalDueIndicator1|LastPaymentDate1|LastPaymentSource1|InstallmentNumber2|OriginalTaxDue2|AdjustedTaxDue2|TaxAmountDue2|RefundTaxAmountDueIndicator2|InterestAmountDue2|RefundInterestDueIndicator2|CostAmountDue2|RefundCostDueIndicator2|TotalAmountDue2|RefundTaxDueIndicator2|LastPaymentDate2|LastPaymentSource2|CofENumber|PastTaxSaleStatus|AssessedValuation|EqualizedEvaluation|TaxRate|RecordCount|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|LastPaymentReceivedAmount1|LastPaymentReceivedAmount2|EndMarker|TaxDueEstimated1|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|VeteranExempt|AdjustedAmountDue1_Org|TaxSale|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|         PH|08322000171119|   050|           299|SANDRA TOMBAUGH  ...|  2075 E IL RT 18  ...|STREATOR         ...|                  IL|         613649028|           0000000120|                         |       BOARDWALK      |                  ST  |ELK GROVE VILLAGE...|                   IL|          600073683|     ELK GROVE      |  16035|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          124765.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02222023|               801|                02|       110724.0|       110724.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02222023|               801|   0000000|                Y|           8483.0|            25472.0| 9245.0|  000284831|                 0|                         0|                0|           0|             0|           0|                  124765.0|                  110724.0|        X|        124765.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              124765.0|      0|\n",
            "|         PH|09143170550000|   088|           295|DIANE S FAGIANO  ...|  9011 N GRACE     ...|NILES            ...|                  IL|         607141424|           0000009011|                       N |       GRACE          |                  AVE |NILES            ...|                   IL|          607141424|     MAINE          |  22021|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          242203.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02092023|               750|                02|       223476.0|       223476.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02092023|               750|   0000000|                Y|          16048.0|            48187.0| 9664.0|  000300826|                 0|                         0|                0|           0|             0|           0|                  242203.0|                  223476.0|        X|        242203.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              242203.0|      0|\n",
            "|         PH|09174080120000|   089|           592|RDLD BUILD LLC 13...|  5126 N MISSION DR...|NORRIDGE         ...|                  IL|         607063216|           0000001326|                         |       JEFFERSON      |                  ST  |DES PLAINES      ...|                   IL|          600164371|     MAINE          |  22028|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|         1256305.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02162023|               801|                02|      3054424.0|      3054424.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02162023|               801|   0000000|                Y|         140238.0|           421093.0|10237.0|  000310618|                 0|                         0|                0|           0|             0|           0|                 1256305.0|                 3054424.0|        X|       1256305.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|             1256305.0|      0|\n",
            "|         PH|09174080140000|   089|           592|RDLD BUILD LLC 13...|  5126 N MISSION DR...|NORRIDGE         ...|                  IL|         607063216|           0000001338|                         |       JEFFERSON      |                  ST  |DES PLAINES      ...|                   IL|          600164371|     MAINE          |  22028|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|         1256305.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02162023|               801|                02|      3054424.0|      3054424.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02162023|               801|   0000000|                Y|         140238.0|           421093.0|10237.0|  000310620|                 0|                         0|                0|           0|             0|           0|                 1256305.0|                 3054424.0|        X|       1256305.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|             1256305.0|      0|\n",
            "|         PH|09174080160000|   089|           590|RDLD BUILD LLC 13...|  5126 N MISSION DR...|NORRIDGE         ...|                  IL|         607063216|           0000001352|                         |       JEFFERSON      |                  ST  |DES PLAINES      ...|                   IL|          600164371|     MAINE          |  22028|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          185868.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02162023|               801|                02|       165445.0|       165445.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02162023|               801|   0000000|                Y|          11429.0|            34318.0|10237.0|  000310622|                 0|                         0|                0|           0|             0|           0|                  185868.0|                  165445.0|        X|        185868.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              185868.0|      0|\n",
            "|         PH|09174080170000|   089|           590|RDLD BUILD LLC 13...|  5126 N MISSION DR...|NORRIDGE         ...|                  IL|         607063216|           0000001358|                         |       JEFFERSON      |                  ST  |DES PLAINES      ...|                   IL|          600164371|     MAINE          |  22028|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          185868.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03012023|               801|                02|       165445.0|       165445.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        03012023|               801|   0000000|                Y|          11429.0|            34318.0|10237.0|  000310623|                 0|                         0|                0|           0|             0|           0|                  185868.0|                  165445.0|        X|        185868.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              185868.0|      0|\n",
            "|         PH|09174080180000|   089|           590|1300 JEFFERSON GE...|  P O BOX 42794    ...|EVERGREEN PK     ...|                  IL|         608050794|           0000001364|                         |       JEFFERSON      |                  ST  |DES PLAINES      ...|                   IL|          600164371|     MAINE          |  22028|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          185868.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03022023|               801|                02|       165445.0|       165445.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        03022023|               801|   0000000|                Y|          11429.0|            34318.0|10237.0|  000310624|                 0|                         0|                0|           0|             0|           0|                  185868.0|                  165445.0|        X|        185868.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              185868.0|      0|\n",
            "|         PH|09174060331028|   089|           299|SAM & MARIA KONDI...|  1378 PERRY ST #40...|DES PLAINES      ...|                  IL|         600164389|           0000001378|                         |       PERRY          |                  ST  |DES PLAINES      ...|                   IL|          600164389|     MAINE          |  22028|       03|              1|           1|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          190137.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02222023|               801|                02|       148892.0|       148892.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02222023|               801|   0000000|                Y|          17024.0|            33118.0|10237.0|  000310589|                 0|                         0|                0|           1|             0|           0|                  190137.0|                  148892.0|        X|        190137.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|              190137.0|      0|\n",
            "|         PH|09194110050000|   090|           202|WILLIAM & J THOMP...|  1433 S WOLF RD   ...|DES PLAINES      ...|                  IL|         600181303|           0000001433|                       S |       WOLF           |                  RD  |DES PLAINES      ...|                   IL|          600181303|     MAINE          |  22028|       03|              1|           1|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          237471.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        01242023|               700|                02|       189105.0|       189105.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        01242023|               700|   0000000|                Y|          19872.0|            41670.0|10237.0|  000314229|                 0|                         0|                0|           1|             0|           0|                  237471.0|                  189105.0|        X|        237471.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|              237471.0|      0|\n",
            "|         PH|09202030170000|   090|           593|KYUWON & MICHA LE...|  984 LEE STREET   ...|DES PLAINES      ...|                  IL|         600166513|           0000001000|                         |       LEE            |                  ST  |DES PLAINES      ...|                   IL|          600166515|     MAINE          |  22028|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|         1788466.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02162023|               801|                02|      1519170.0|      1519170.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02162023|               801|   0000000|                Y|         107605.0|           323106.0|10237.0|  000315406|                 0|                         0|                0|           0|             0|           0|                 1788466.0|                 1519170.0|        X|       1788466.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|             1788466.0|      0|\n",
            "|         PH|07152020250000|   187|           203|JOSE TORRES      ...|  960 APRICOT ST   ...|HOFFMAN ESTS     ...|                  IL|         601694902|           0000000960|                         |       APRICOT        |                  ST  |HOFFMAN ESTATES  ...|                   IL|          601694902|     SCHAUMBURG     |  35013|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          399808.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        01302023|               700|                02|       344520.0|       344520.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        01302023|               700|   0000000|                Y|          22216.0|            66708.0|11158.0|  000218550|                 0|                         0|                0|           0|             0|           0|                  399808.0|                  344520.0|        X|        399808.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              399808.0|      0|\n",
            "|         PH|09204140310000|   090|           241|HELENA M MICHALAK...|  1509 CENTER STREE...|DES PLAINES      ...|                  IL|         600181722|           0000001509|                         |       CENTER         |                  ST  |DES PLAINES      ...|                   IL|          600181722|     MAINE          |  22028|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|           31113.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03012023|               801|                02|        26429.0|        26429.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        03012023|               801|   0000000|                Y|           1872.0|             5621.0|10237.0|  000317502|                 0|                         0|                0|           0|             0|           0|                   31113.0|                   26429.0|        X|         31113.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               31113.0|      0|\n",
            "|         PH|09223000160000|   091|           203|WEDDINGTON & DOYL...|  1301 LUNDERGAN AV...|PARK RIDGE       ...|                  IL|         600681307|           0000001301|                         |       LUNDERGAN      |                  AVE |PARK RIDGE       ...|                   IL|          600681307|     MAINE          |  22031|       00|              1|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          202212.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02212023|               801|                02|       177170.0|       177170.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02212023|               801|   0000000|                Y|          14139.0|            37455.0|10129.0|  000319590|                 0|                         0|                0|           0|             0|           0|                  202212.0|                  177170.0|        X|        202212.0|                        0.0|                          0.0|                       0.0|                             0.0|               5000.0|               0.0|                     0.0|                           0.0|          0.0|              202212.0|      0|\n",
            "|         PH|09223000170000|   091|           203|TAXPAYER OF      ...|  1301 LUNDERGAN AV...|PARK RIDGE       ...|                  IL|         600681307|           0000001301|                         |       LUNDERGAN      |                  AVE |PARK RIDGE       ...|                   IL|          600681307|     MAINE          |  22031|       00|              1|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          202258.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02212023|               801|                02|       177215.0|       177215.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02212023|               801|   0000000|                Y|          14142.0|            37464.0|10129.0|  000319591|                 0|                         0|                0|           0|             0|           0|                  202258.0|                  177215.0|        X|        202258.0|                        0.0|                          0.0|                       0.0|                             0.0|               5000.0|               0.0|                     0.0|                           0.0|          0.0|              202258.0|      0|\n",
            "|         PH|09223120170000|   091|           211|LAURA I BALDWIN T...|  1930 WALNUT ST   ...|PARK RIDGE       ...|                  IL|         600681756|           0000001212|                         |       RAND           |                  RD  |PARK RIDGE       ...|                   IL|          600680000|     MAINE          |  22031|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          468185.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        12282022|               600|                02|       418244.0|       418244.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        12282022|               600|   0000000|                Y|          29145.0|            87514.0|10129.0|  000319848|                 0|                         0|                0|           0|             0|           0|                  468185.0|                  418244.0|        X|        468185.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              468185.0|      0|\n",
            "|         PH|09233230070000|   092|           203|KATARZYNA PIETRZA...|  8529 W NORMAL AVE...|NILES            ...|                  IL|         607142327|           0000008529|                       W |       NORMAL         |                  AVE |NILES            ...|                   IL|          607142327|     MAINE          |  22023|       00|              1|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          238724.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02222023|               801|                02|       218879.0|       218879.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02222023|               801|   0000000|                Y|          18181.0|            44592.0|10262.0|  000321745|                 0|                         0|                0|           0|             0|           0|                  238724.0|                  218879.0|        X|        238724.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|              238724.0|      0|\n",
            "|         PH|09244110370000|   092|           206|CHRIS DELANNOY   ...|  7539 CLEVELAND ST...|NILES            ...|                  IL|         607142953|           0000007539|                       W |       CLEVELAND      |                  ST  |NILES            ...|                   IL|          607142953|     MAINE          |  22021|       00|              1|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          634483.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03072023|               801|                02|       578813.0|       578813.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        03072023|               801|   0000000|                Y|          45142.0|           125548.0| 9664.0|  000324540|                 0|                         0|                0|           0|             0|           0|                  634483.0|                  578813.0|        X|        634483.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|              634483.0|      0|\n",
            "|         PH|09261100020000|   093|           203|ANASTASIA MARSHAL...|  841 N DELPHIA    ...|PARK RIDGE       ...|                  IL|         600682517|           0000000841|                       N |       DELPHIA        |                  AVE |PARK RIDGE       ...|                   IL|          600682517|     MAINE          |  22031|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          679587.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02212023|               801|                02|       607090.0|       607090.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02212023|               801|   0000000|                Y|          42305.0|           127029.0|10129.0|  000328329|                 0|                         0|                0|           0|             0|           0|                  679587.0|                  607090.0|        X|        679587.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              679587.0|      0|\n",
            "|         PH|09261120230000|   093|           203|PRETTY WOMAN TRUS...|  1406 LAHON ST    ...|PARK RIDGE       ...|                  IL|         600682522|           0000001405|                         |       LAHON          |                  ST  |PARK RIDGE       ...|                   IL|          600682521|     MAINE          |  22031|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          414706.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02222023|               801|                02|       370474.0|       370474.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02222023|               801|   0000000|                Y|          25816.0|            77518.0|10129.0|  000328373|                 0|                         0|                0|           0|             0|           0|                  414706.0|                  370474.0|        X|        414706.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              414706.0|      0|\n",
            "|         PH|09271200170000|   094|           203|DENNIS WODKA     ...|  728 SYLVIAWOOD AV...|PARK RIDGE       ...|                  IL|         600682248|           0000000728|                         |       SYLVIAWOOD     |                  AVE |PARK RIDGE       ...|                   IL|          600682248|     MAINE          |  22031|       00|              1|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          393979.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04302023|               801|                02|       345023.0|       345023.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        04302023|               801|   0000000|                Y|          27628.0|            72959.0|10129.0|  000330581|                 0|                         0|                0|           0|             0|           0|                   65010.0|                  345023.0|        X|        393979.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|              393979.0|      0|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where(data.PIN==\"09143170550000\").show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m3gpGSsG_4Yh",
        "outputId": "0b66b22b-3e31-43e6-97f1-07f5521a747f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|SegmentCode|           PIN|Volume|Classification|        TaxpayerName|TaxpayerMailingAddress| TaxpayerMailingCity|TaxpayerMailingState|TaxpayerMailingZip|TaxpayerPropertyHouse|TaxpayerPropertyDirection|TaxpayerPropertyStreet|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyState|TaxpayerPropertyZip|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|LongtimeHomeownersExempt|TaxInfoType|TaxType|TaxYear|BillYear|AccountStatus|BillType|SegmentCode2|InstallmentNumber1|AdjustedAmountDue1|TaxAmountDue1|RefundTaxAmountDueIndicator1|InterestAmountDue1|RefundInterestDueIndicator1|CostAmountDue1|RefundCostDueIndicator1|TotalAmountDue1|RefundTotalDueIndicator1|LastPaymentDate1|LastPaymentSource1|InstallmentNumber2|OriginalTaxDue2|AdjustedTaxDue2|TaxAmountDue2|RefundTaxAmountDueIndicator2|InterestAmountDue2|RefundInterestDueIndicator2|CostAmountDue2|RefundCostDueIndicator2|TotalAmountDue2|RefundTaxDueIndicator2|LastPaymentDate2|LastPaymentSource2|CofENumber|PastTaxSaleStatus|AssessedValuation|EqualizedEvaluation|TaxRate|RecordCount|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|LastPaymentReceivedAmount1|LastPaymentReceivedAmount2|EndMarker|TaxDueEstimated1|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|VeteranExempt|AdjustedAmountDue1_Org|TaxSale|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|         PH|09143170550000|   088|           295|DIANE S FAGIANO  ...|  9011 N GRACE     ...|NILES            ...|                  IL|         607141424|           0000009011|                       N |       GRACE          |                  AVE |NILES            ...|                   IL|          607141424|     MAINE          |  22021|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          256123.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        04272023|               801|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                Y|          16048.0|            48187.0|    0.0|  000300978|                 0|                         0|                0|           0|             0|           0|                  256123.0|                       0.0|        X|        256123.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              256123.0|      1|\n",
            "|         PH|09143170550000|   088|           295|DIANE S FAGIANO  ...|  9011 N GRACE     ...|NILES            ...|                  IL|         607141424|           0000009011|                       N |       GRACE          |                  AVE |NILES            ...|                   IL|          607141424|     MAINE          |  22021|       00|              0|           0|                 0|                       0|          R|     00|   2021|    2021|            P|       1|          PD|                01|          242203.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02092023|               750|                02|       223476.0|       223476.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        02092023|               750|   0000000|                Y|          16048.0|            48187.0| 9664.0|  000300826|                 0|                         0|                0|           0|             0|           0|                  242203.0|                  223476.0|        X|        242203.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              242203.0|      0|\n",
            "|         PH|09143170550000|   088|           295|DIANE S FAGIANO  ...|  9011 N GRACE     ...|NILES            ...|                  IL|         607141424|           0000009011|                       N |       GRACE          |                  AVE |NILES            ...|                   IL|          607141424|     MAINE          |  22021|       00|              0|           0|                 0|                       0|          R|     00|   2020|    2020|            P|       1|          PD|                01|          187403.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02262021|               600|                02|       252966.0|       252966.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        11152022|               823|   0000000|                N|          16048.0|            51729.0| 8513.0|  000300749|                 0|                         0|                0|           0|             0|           0|                  187403.0|                  252966.0|        X|        187403.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              187403.0|      0|\n",
            "|         PH|09143170550000|   088|           295|DIANE S FAGIANO  ...|  9011 N GRACE     ...|NILES            ...|                  IL|         607141424|           0000009011|                       N |       GRACE          |                  AVE |NILES            ...|                   IL|          607141424|     MAINE          |  22021|       03|              1|           1|                 0|                       0|          R|     00|   2019|    2019|            P|       1|          PD|                01|           98857.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02202020|               600|                02|       241876.0|       241876.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        07232020|               600|   0000000|                N|          20314.0|            41236.0| 8263.0|  000300917|                 0|                         0|                0|           1|             0|           0|                   98857.0|                  241876.0|        X|         98857.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|               98857.0|      0|\n",
            "|         PH|09143170550000|   088|           295|DIANE S FAGIANO  ...|  9011 N GRACE     ...|NILES            ...|                  IL|         607141424|           0000009011|                       N |       GRACE          |                  AVE |NILES            ...|                   IL|          607141424|     MAINE          |  22021|       00|              0|           0|                 0|                       0|          R|     00|   2018|    2018|            P|       1|          PD|                01|           96165.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02192019|               600|                02|       446830.0|        83575.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        07232019|               600|   0017118|                N|          20360.0|            59266.0| 9162.0|  000300564|                 0|                         0|                0|           0|             0|           0|                   96165.0|                   83575.0|        X|         96165.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               96165.0|      0|\n",
            "|         PH|09143170550000|   088|           295|DIANE S FAGIANO  ...|  9011 N GRACE     ...|NILES            ...|                  IL|         607141424|           0000009011|                       N |       GRACE          |                  AVE |NILES            ...|                   IL|          607141424|     MAINE          |  22021|       03|              1|           1|                 1|                       0|          R|     00|   2017|    2017|            P|       1|          PD|                01|          123845.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        02142018|               600|                02|        51001.0|        51001.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        07162018|               600|   0000000|                N|          20360.0|            19617.0| 8913.0|  000302261|                 0|                         0|                0|           1|             0|           0|                  123845.0|                   51001.0|        X|        123845.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                 22704.0|                           0.0|          0.0|              123845.0|      0|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "analysis_cols = [\"PIN\", \"TaxYear\", \"LastPaymentDate1\", \"LastPaymentDate2\", \"AdjustedAmountDue1\", \"PastTaxSaleStatus\", \"PastTaxSaleStatusSum\"]\n",
        "#\"TotalAmountDue1\", \"TotalAmountDue2\",\n",
        "data2.where(data2.PIN==\"09143170550000\").select(*analysis_cols).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qgk5N7xGcreb",
        "outputId": "0b41ea30-f520-4b21-9401-d0a321251e83"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-------+----------------+----------------+------------------+-----------------+--------------------+\n",
            "|           PIN|TaxYear|LastPaymentDate1|LastPaymentDate2|AdjustedAmountDue1|PastTaxSaleStatus|PastTaxSaleStatusSum|\n",
            "+--------------+-------+----------------+----------------+------------------+-----------------+--------------------+\n",
            "|09143170550000|   2017|      2018-02-14|      2018-07-16|          123845.0|                0|                   0|\n",
            "|09143170550000|   2018|      2019-02-19|      2019-07-23|           96165.0|                0|                   0|\n",
            "|09143170550000|   2019|      2020-02-20|      2020-07-23|           98857.0|                0|                   0|\n",
            "|09143170550000|   2020|      2021-02-26|      2022-11-15|          187403.0|                0|                   0|\n",
            "|09143170550000|   2021|      2023-02-09|      2023-02-09|          242203.0|                1|                   1|\n",
            "+--------------+-------+----------------+----------------+------------------+-----------------+--------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data2.where(data2.PIN==\"08322000171119\").select(*analysis_cols).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eWoBr0uhexf1",
        "outputId": "02e44b7e-a521-4f78-c6cb-1a3e760eff91"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-------+----------------+----------------+------------------+-----------------+--------------------+\n",
            "|           PIN|TaxYear|LastPaymentDate1|LastPaymentDate2|AdjustedAmountDue1|PastTaxSaleStatus|PastTaxSaleStatusSum|\n",
            "+--------------+-------+----------------+----------------+------------------+-----------------+--------------------+\n",
            "|08322000171119|   2017|      2019-05-03|      2019-05-03|           45910.0|                0|                   0|\n",
            "|08322000171119|   2018|      2019-08-13|      2019-08-27|           35990.0|                1|                   1|\n",
            "|08322000171119|   2019|      2020-03-30|      2020-10-19|           87176.0|                1|                   2|\n",
            "|08322000171119|   2020|      2021-05-18|      2022-11-15|          127151.0|                0|                   2|\n",
            "|08322000171119|   2021|      2023-02-22|      2023-02-22|          124765.0|                1|                   3|\n",
            "+--------------+-------+----------------+----------------+------------------+-----------------+--------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "temp_df = data2.where(data2.PIN==\"08322000171119\").persist()\n"
      ],
      "metadata": {
        "id": "wnL6u-FDLpcD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1x7IkdIhLtPT",
        "outputId": "7a1c78ca-9900-44f1-82dd-7cd7169861a4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+-----------+\n",
            "|           PIN|Volume|Classification|TaxpayerMailingState|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|TaxYear|AccountStatus|BillType|AdjustedAmountDue1|TaxAmountDue1|CostAmountDue1|LastPaymentDate1|LastPaymentSource1|LastPaymentDate2|CofENumber|PastTaxSaleStatus|TaxRate|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|Year|   PmtDue1|   PmtDue2|LatePmt1|LatePmt2|PastTaxSaleStatusSum|LatePmt2Sum|\n",
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+-----------+\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              1|           0|                 0|   2017|            P|       1|           45910.0|          0.0|           0.0|      2019-05-03|               823|      2019-05-03|         0|                0| 9193.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|                     0.0|                           0.0|2017|2018-03-01|2018-08-01|       1|       1|                   0|       null|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2018|            P|       1|           35990.0|          0.0|           0.0|      2019-08-13|               801|      2019-08-27|         0|                1| 9424.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2018|2019-03-01|2019-08-01|       1|       1|                   1|          1|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2019|            P|       1|           87176.0|          0.0|           0.0|      2020-03-30|               801|      2020-10-19|         0|                1| 8349.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2019|2020-03-03|2020-08-03|       1|       1|                   2|          2|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2020|            P|       1|          127151.0|          0.0|           0.0|      2021-05-18|               801|      2022-11-15|         0|                0| 8296.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2020|2021-03-02|2021-10-01|       1|       1|                   2|          3|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2021|            P|       1|          124765.0|          0.0|           0.0|      2023-02-22|               801|      2023-02-22|         0|                1| 9245.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2021|2022-03-01|2023-01-03|       1|       1|                   3|          4|\n",
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gyzO7x4VOPJg",
        "outputId": "a62a9dd8-c9f2-4542-e5ed-130d0ffd918c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+-----------+\n",
            "|           PIN|Volume|Classification|TaxpayerMailingState|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|TaxYear|AccountStatus|BillType|AdjustedAmountDue1|TaxAmountDue1|CostAmountDue1|LastPaymentDate1|LastPaymentSource1|LastPaymentDate2|CofENumber|PastTaxSaleStatus|TaxRate|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|Year|   PmtDue1|   PmtDue2|LatePmt1|LatePmt2|PastTaxSaleStatusSum|LatePmt2Sum|\n",
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+-----------+\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              1|           0|                 0|   2017|            P|       1|           45910.0|          0.0|           0.0|      2019-05-03|               823|      2019-05-03|         0|                0| 9193.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|                     0.0|                           0.0|2017|2018-03-01|2018-08-01|       1|       1|                   0|          0|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2018|            P|       1|           35990.0|          0.0|           0.0|      2019-08-13|               801|      2019-08-27|         0|                1| 9424.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2018|2019-03-01|2019-08-01|       1|       1|                   1|          1|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2019|            P|       1|           87176.0|          0.0|           0.0|      2020-03-30|               801|      2020-10-19|         0|                1| 8349.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2019|2020-03-03|2020-08-03|       1|       1|                   2|          2|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2020|            P|       1|          127151.0|          0.0|           0.0|      2021-05-18|               801|      2022-11-15|         0|                0| 8296.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2020|2021-03-02|2021-10-01|       1|       1|                   2|          3|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2021|            P|       1|          124765.0|          0.0|           0.0|      2023-02-22|               801|      2023-02-22|         0|                1| 9245.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2021|2022-03-01|2023-01-03|       1|       1|                   3|          4|\n",
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "temp_df.dtypes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N8lkXLm0MtzL",
        "outputId": "62764942-fc57-46f0-85b3-90595f887727"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('PIN', 'string'),\n",
              " ('Volume', 'int'),\n",
              " ('Classification', 'string'),\n",
              " ('TaxpayerMailingState', 'int'),\n",
              " ('TaxpayerPropertySuffix', 'string'),\n",
              " ('TaxpayerPropertyCity', 'int'),\n",
              " ('TaxpayerPropertyTown', 'string'),\n",
              " ('TaxCode', 'string'),\n",
              " ('TaxStatus', 'int'),\n",
              " ('HomeownerExempt', 'string'),\n",
              " ('SeniorExempt', 'string'),\n",
              " ('SeniorFreezeExempt', 'string'),\n",
              " ('TaxYear', 'string'),\n",
              " ('AccountStatus', 'string'),\n",
              " ('BillType', 'int'),\n",
              " ('AdjustedAmountDue1', 'float'),\n",
              " ('TaxAmountDue1', 'float'),\n",
              " ('CostAmountDue1', 'float'),\n",
              " ('LastPaymentDate1', 'date'),\n",
              " ('LastPaymentSource1', 'string'),\n",
              " ('LastPaymentDate2', 'date'),\n",
              " ('CofENumber', 'int'),\n",
              " ('PastTaxSaleStatus', 'int'),\n",
              " ('TaxRate', 'float'),\n",
              " ('CondemnationStatus', 'string'),\n",
              " ('MunicipalAcquisitionStatus', 'string'),\n",
              " ('AcquisitionStatus', 'string'),\n",
              " ('ExemptStatus', 'string'),\n",
              " ('BankruptStatus', 'string'),\n",
              " ('RefundStatus', 'string'),\n",
              " ('ReturningVetExemptionAmount', 'float'),\n",
              " ('DisabledPersonExemptionAmount', 'float'),\n",
              " ('DisabledVetExemptionAmount', 'float'),\n",
              " ('DisabledPersonVetExemptionAmount', 'float'),\n",
              " ('HomeownerExemptAmount', 'float'),\n",
              " ('SeniorFreezeExemptAmount', 'float'),\n",
              " ('LongtimeHomeownersExemptAmount', 'float'),\n",
              " ('Year', 'int'),\n",
              " ('PmtDue1', 'date'),\n",
              " ('PmtDue2', 'date'),\n",
              " ('LatePmt1', 'int'),\n",
              " ('LatePmt2', 'int'),\n",
              " ('PastTaxSaleStatusSum', 'bigint')]"
            ]
          },
          "metadata": {},
          "execution_count": 109
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "data2.where(data2.PIN==\"08322000171119\").show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NtvUqfv4JfAU",
        "outputId": "498039f5-a3b0-466f-a03f-523a8a44d755"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+\n",
            "|           PIN|Volume|Classification|TaxpayerMailingState|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|TaxYear|AccountStatus|BillType|AdjustedAmountDue1|TaxAmountDue1|CostAmountDue1|LastPaymentDate1|LastPaymentSource1|LastPaymentDate2|CofENumber|PastTaxSaleStatus|TaxRate|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|Year|   PmtDue1|   PmtDue2|LatePmt1|LatePmt2|PastTaxSaleStatusSum|\n",
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              1|           0|                 0|   2017|            P|       1|           45910.0|          0.0|           0.0|      2019-05-03|               823|      2019-05-03|         0|                0| 9193.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|                     0.0|                           0.0|2017|2018-03-01|2018-08-01|       1|       1|                   0|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2018|            P|       1|           35990.0|          0.0|           0.0|      2019-08-13|               801|      2019-08-27|         0|                1| 9424.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2018|2019-03-01|2019-08-01|       1|       1|                   1|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2019|            P|       1|           87176.0|          0.0|           0.0|      2020-03-30|               801|      2020-10-19|         0|                1| 8349.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2019|2020-03-03|2020-08-03|       1|       1|                   2|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2020|            P|       1|          127151.0|          0.0|           0.0|      2021-05-18|               801|      2022-11-15|         0|                0| 8296.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2020|2021-03-02|2021-10-01|       1|       1|                   2|\n",
            "|08322000171119|    50|           299|                   0|                  ST  |                   0|     ELK GROVE      |  16035|        0|              0|           0|                 0|   2021|            P|       1|          124765.0|          0.0|           0.0|      2023-02-22|               801|      2023-02-22|         0|                1| 9245.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|2021|2022-03-01|2023-01-03|       1|       1|                   3|\n",
            "+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+-------+---------+---------------+------------+------------------+-------+-------------+--------+------------------+-------------+--------------+----------------+------------------+----------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+----+----------+----------+--------+--------+--------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where(data.PIN==\"04201070150000\").select(*analysis_cols).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s2VZ7t8pppgD",
        "outputId": "9bbb14e5-20bd-49fd-fea9-d89f926b95cf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-------+----------------+----------------+------------------+---------------+--------------------------+--------------------------+-----------------+\n",
            "|           PIN|TaxYear|LastPaymentDate1|LastPaymentDate2|AdjustedAmountDue1|AdjustedTaxDue2|LastPaymentReceivedAmount1|LastPaymentReceivedAmount2|PastTaxSaleStatus|\n",
            "+--------------+-------+----------------+----------------+------------------+---------------+--------------------------+--------------------------+-----------------+\n",
            "|04201070150000|   2022|        03032023|        00000000|          654054.0|            0.0|                  654054.0|                       0.0|                N|\n",
            "|04201070150000|   2021|        02012022|        12052022|          626570.0|       562619.0|                  626570.0|                  562619.0|                N|\n",
            "|04201070150000|   2020|        02022021|        08262021|          608158.0|       531061.0|                  608158.0|                  531061.0|                N|\n",
            "|04201070150000|   2019|        01272020|        07062020|          742023.0|       363718.0|                  742023.0|                  363718.0|                N|\n",
            "|04201070150000|   2018|        02032019|        07032019|          721482.0|       627650.0|                  721482.0|                  627650.0|                N|\n",
            "|04201070150000|   2017|        12282017|        07232018|          696365.0|       615421.0|                  696365.0|                  615421.0|                N|\n",
            "|04201070150000|   2017|        00000000|        09142018|               0.0|         3988.0|                       0.0|                    3988.0|                N|\n",
            "+--------------+-------+----------------+----------------+------------------+---------------+--------------------------+--------------------------+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where(data.PIN==\"09204140310000\").select(*analysis_cols).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ED0Bf1_KtV4D",
        "outputId": "4fdcc221-71e9-493a-981c-966f22a55bff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------+-------+----------------+----------------+------------------+---------------+--------------------------+--------------------------+-----------------+\n",
            "|           PIN|TaxYear|LastPaymentDate1|LastPaymentDate2|AdjustedAmountDue1|AdjustedTaxDue2|LastPaymentReceivedAmount1|LastPaymentReceivedAmount2|PastTaxSaleStatus|\n",
            "+--------------+-------+----------------+----------------+------------------+---------------+--------------------------+--------------------------+-----------------+\n",
            "|09204140310000|   2022|        04272023|        00000000|           31648.0|            0.0|                   31648.0|                       0.0|                Y|\n",
            "|09204140310000|   2021|        03012023|        03012023|           31113.0|        26429.0|                   31113.0|                   26429.0|                Y|\n",
            "|09204140310000|   2020|        11152022|        11152022|           27710.0|        28859.0|                   27710.0|                   28859.0|                N|\n",
            "|09204140310000|   2019|        04222020|        06172020|           27237.0|        23144.0|                   27237.0|                   23144.0|                N|\n",
            "|09204140310000|   2018|        06212019|        12272019|           26756.0|        22765.0|                   26756.0|                   22765.0|                N|\n",
            "|09204140310000|   2017|        11292018|        11292018|           25309.0|        23339.0|                   25309.0|                   23339.0|                N|\n",
            "+--------------+-------+----------------+----------------+------------------+---------------+--------------------------+--------------------------+-----------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where((data1.AdjustedAmountDue1!=0)&(data1.TotalAmountDue1==0)).count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yi890JM8TzKu",
        "outputId": "4d69275c-6f8d-4036-9a4c-cb9f67d6cebd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9876762"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where((data1.TaxRate<30)&(data1.TaxRate>0)&(data1.AdjustedTaxDue2>0)).show()"
      ],
      "metadata": {
        "id": "hHkHr7DYozoW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data1.where((data1.TaxRate<30)&(data1.TaxRate>0)&(data1.AdjustedTaxDue2>0)).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NFXxo89BU28Z",
        "outputId": "48d79723-0eec-4c47-cf0f-52772aec42b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+\n",
            "|SegmentCode|           PIN|Volume|Classification|        TaxpayerName|TaxpayerMailingAddress| TaxpayerMailingCity|TaxpayerMailingState|TaxpayerMailingZip|TaxpayerPropertyHouse|TaxpayerPropertyDirection|TaxpayerPropertyStreet|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyState|TaxpayerPropertyZip|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|LongtimeHomeownersExempt|TaxInfoType|TaxType|TaxYear|BillYear|AccountStatus|BillType|SegmentCode2|InstallmentNumber1|AdjustedAmountDue1|TaxAmountDue1|RefundTaxAmountDueIndicator1|InterestAmountDue1|RefundInterestDueIndicator1|CostAmountDue1|RefundCostDueIndicator1|TotalAmountDue1|RefundTotalDueIndicator1|LastPaymentDate1|LastPaymentSource1|InstallmentNumber2|OriginalTaxDue2|AdjustedTaxDue2|TaxAmountDue2|RefundTaxAmountDueIndicator2|InterestAmountDue2|RefundInterestDueIndicator2|CostAmountDue2|RefundCostDueIndicator2|TotalAmountDue2|RefundTaxDueIndicator2|LastPaymentDate2|LastPaymentSource2|CofENumber|PastTaxSaleStatus|AssessedValuation|EqualizedEvaluation|TaxRate|RecordCount|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|LastPaymentReceivedAmount1|LastPaymentReceivedAmount2|EndMarker|TaxDueEstimated1|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|VeteranExempt|AdjustedAmountDue1_Org|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+\n",
            "|         PH|04201070080000|   132|           278|MELVIN D ALEXANDE...|  2734 ASPEN COURT ...|GLENVIEW         ...|                  IL|         600267311|           0000002734|                         |       ASPEN          |                  CT  |GLENVIEW         ...|                   IL|          600267311|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3818.0|         3818.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08012018|               501|   0000000|                N|          62103.0|           165993.0|   23.0|  000128772|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3818.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201070090000|   132|           278|MR & MRS GREENBER...|  4255 MEADOWVIEW D...|GLENVIEW         ...|                  IL|         600267310|           0000004255|                         |       MEADOWVIEW     |                  DR  |GLENVIEW         ...|                   IL|          600267310|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         4082.0|         4082.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        10012018|               501|   0000000|                N|          63286.0|           177497.0|   23.0|  000128774|                 0|                         0|                0|           0|             0|           0|                       0.0|                    4082.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201070100000|   132|           278|SANDRA RANDELL   ...|  2729 SUNFLOWER CT...|GLENVIEW         ...|                  IL|         600267309|           0000002729|                         |       SUNFLOWER      |                  CT  |GLENVIEW         ...|                   IL|          600267309|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3822.0|         3822.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08172018|               501|   0000000|                N|          62145.0|           166117.0|   23.0|  000128776|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3822.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201070110000|   132|           278|MARYANN SCHEIBER ...|  2725 SUNFLOWER CT...|GLENVIEW         ...|                  IL|         600267309|           0000002725|                         |       SUNFLOWER      |                  CT  |GLENVIEW         ...|                   IL|          600267309|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3934.0|         3934.0|         52.0|                           N|               0.0|                          P|           0.0|                      P|           52.0|                     N|        08012018|               501|   0000000|                N|          63815.0|           171065.0|   23.0|  000128778|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3986.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201070120000|   132|           278|MR & MRS MAKI    ...|  2721 SUNFLOWER CT...|GLENVIEW         ...|                  IL|         600267309|           0000002721|                         |       SUNFLOWER      |                  CT  |GLENVIEW         ...|                   IL|          600267309|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3566.0|         3566.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08172018|               501|   0000000|                N|          58389.0|           154989.0|   23.0|  000128780|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3566.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201070130000|   132|           278|FRANK JANECEK    ...|  2720 SUNFLOWER CT...|GLENVIEW         ...|                  IL|         600267309|           0000002720|                         |       SUNFLOWER      |                  CT  |GLENVIEW         ...|                   IL|          600267309|     NORTHFIELD     |  25401|       00|              0|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         4242.0|         4242.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        09262018|               501|   0000000|                N|          62231.0|           184372.0|   23.0|  000128782|                 0|                         0|                0|           0|             0|           0|                       0.0|                    4242.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201070140000|   132|           204|MR & MRS KARFIAS ...|  2724 SUNFLOWER CT...|GLENVIEW         ...|                  IL|         600267309|           0000002724|                         |       SUNFLOWER      |                  CT  |GLENVIEW         ...|                   IL|          600267309|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3560.0|         3560.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08012018|               444|   0000000|                N|          58310.0|           154755.0|   23.0|  000128784|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3560.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201070150000|   132|           278|OTTO LEE & MINGLI...|  2728 SUNFLOWER CT...|GLENVIEW         ...|                  IL|         600267309|           0000002728|                         |       SUNFLOWER      |                  CT  |GLENVIEW         ...|                   IL|          600267309|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3988.0|         3988.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        09142018|               501|   0000000|                N|          61896.0|           173379.0|   23.0|  000128786|                 0|                         0|                0|           0|             0|           0|                       0.0|                    3988.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080010000|   132|           278|MICHAEL TEKIPPE  ...|  2802 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002802|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         4266.0|         4266.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08012018|               501|   0000000|                N|          65970.0|           185449.0|   23.0|  000128792|                 0|                         0|                0|           0|             0|           0|                       0.0|                    4266.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080020000|   132|           278|MIRESSE          ...|  2806 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002806|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3692.0|         3692.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        10012018|               501|   0000000|                N|          60249.0|           160500.0|   23.0|  000128794|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3692.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080030000|   132|           278|MR JUSTIN C PAK &...|  2810 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002810|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3684.0|         3684.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        10012018|               501|   0000000|                N|          57423.0|           160127.0|   23.0|  000128796|                 0|                         0|                0|           0|             0|           0|                       0.0|                    3684.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080040000|   132|           278|ALEXANDER FRANTZI...|  2814 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002814|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3870.0|         3870.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08012018|               501|   0000000|                N|          62871.0|           168268.0|   23.0|  000128798|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3870.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080050000|   132|           278|MR & MRS WEINBERG...|  2818 WILD FLOWER ...|GLENVIEW         ...|                  IL|         600261097|           0000002818|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              0|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         4392.0|         4392.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        07302018|               444|   0000000|                N|          64435.0|           190902.0|   23.0|  000128800|                 0|                         0|                0|           0|             0|           0|                       0.0|                    4392.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080060000|   132|           278|ERIC & JOANNE KIM...|  2822 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002822|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3998.0|         3998.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        11012018|               501|   0000000|                N|          62050.0|           173836.0|   23.0|  000128802|                 0|                         0|                0|           0|             0|           0|                       0.0|                    3998.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080070000|   132|           278|MR & MRS GRIBBEN ...|  2823 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002823|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            O|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3734.0|         3734.0|       3734.0|                           P|            1792.0|                          P|           0.0|                      P|         5526.0|                     P|        00000000|               000|   0000000|                N|          60861.0|           162313.0|   23.0|  000128804|                 0|                         0|                0|           1|             0|           0|                       0.0|                       0.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080080000|   132|           278|MR & MRS CHOE    ...|  2819 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002819|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            O|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         4004.0|         4004.0|       4004.0|                           P|            1920.0|                          P|           0.0|                      P|         5924.0|                     P|        06172022|               000|   0000000|                N|          62119.0|           174040.0|   23.0|  000128806|                 0|                         0|                0|           0|             0|           0|                       0.0|                    4004.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080090000|   132|           278|VIVIAN L TAITEL  ...|  2815 WILDFLOWER  ...|GLENVIEW         ...|                  IL|         600261097|           0000002815|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       03|              1|           1|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3692.0|         3692.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08202018|               501|   0000000|                N|          60249.0|           160500.0|   23.0|  000128808|                 0|                         0|                0|           1|             0|           0|                       0.0|                    3692.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|            8000.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080100000|   132|           278|MR & MRS SCHNEIDE...|  2811 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002811|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3876.0|         3876.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        08162018|               501|   0000000|                N|          60249.0|           168500.0|   23.0|  000128810|                 0|                         0|                0|           0|             0|           0|                       0.0|                    3876.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080110000|   132|           278|SANTIAGO A CANDOC...|  2807 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002807|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3688.0|         3688.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        07312018|               501|   0000000|                N|          57479.0|           160293.0|   23.0|  000128812|                 0|                         0|                0|           0|             0|           0|                       0.0|                    3688.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "|         PH|04201080120000|   132|           204|MURUGESAN & RAGAV...|  2803 WILDFLOWER C...|GLENVIEW         ...|                  IL|         600261097|           0000002803|                         |       WILDFLOWER     |                  CT  |GLENVIEW         ...|                   IL|          600261097|     NORTHFIELD     |  25401|       00|              1|           0|                 0|                       0|          R|     04|   2017|    2017|            P|       1|          PD|                01|               0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        00000000|               000|                02|         3972.0|         3972.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        04102019|               501|   0000000|                N|          61668.0|           172704.0|   23.0|  000128814|                 0|                         0|                0|           0|             0|           0|                       0.0|                    3972.0|        X|             0.0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|               0.0|                     0.0|                           0.0|          0.0|                   0.0|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "data1.where(data1.AdjustedAmountDue1 != data1.LastPaymentReceivedAmount1).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9oeTfasoU0MK",
        "outputId": "a2eb860d-d8de-4e19-9725-c426ff37e61a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|SegmentCode|           PIN|Volume|Classification|        TaxpayerName|TaxpayerMailingAddress| TaxpayerMailingCity|TaxpayerMailingState|TaxpayerMailingZip|TaxpayerPropertyHouse|TaxpayerPropertyDirection|TaxpayerPropertyStreet|TaxpayerPropertySuffix|TaxpayerPropertyCity|TaxpayerPropertyState|TaxpayerPropertyZip|TaxpayerPropertyTown|TaxCode|TaxStatus|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|LongtimeHomeownersExempt|TaxInfoType|TaxType|TaxYear|BillYear|AccountStatus|BillType|SegmentCode2|InstallmentNumber1|AdjustedAmountDue1|TaxAmountDue1|RefundTaxAmountDueIndicator1|InterestAmountDue1|RefundInterestDueIndicator1|CostAmountDue1|RefundCostDueIndicator1|TotalAmountDue1|RefundTotalDueIndicator1|LastPaymentDate1|LastPaymentSource1|InstallmentNumber2|OriginalTaxDue2|AdjustedTaxDue2|TaxAmountDue2|RefundTaxAmountDueIndicator2|InterestAmountDue2|RefundInterestDueIndicator2|CostAmountDue2|RefundCostDueIndicator2|TotalAmountDue2|RefundTaxDueIndicator2|LastPaymentDate2|LastPaymentSource2|CofENumber|PastTaxSaleStatus|AssessedValuation|EqualizedEvaluation|TaxRate|RecordCount|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|LastPaymentReceivedAmount1|LastPaymentReceivedAmount2|EndMarker|TaxDueEstimated1|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|VeteranExempt|AdjustedAmountDue1_Org|TaxSale|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "|         PH|12111190381123|   310|           299|DTS PROPERTIES V ...|  1111 N PLAZA DR#2...|SCHAUMBURG       ...|                  IL|         601734996|           0000005348|                       N |       CUMBERLAND     |                  AVE |CHICAGO          ...|                   IL|          606561409|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|           71881.0|      71881.0|                           P|            3234.0|                          P|           0.0|                      P|        75115.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|           6499.0|            19515.0|    0.0|  000452006|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|         71881.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               71881.0|      1|\n",
            "|         PH|12111190381127|   310|           299|SANTHOSH SAITEJ B...|  207 PRESTON RD   ...|HICKORY CREE     ...|                  TX|         750652614|           0000005348|                       N |       CUMBERLAND     |                  AVE |CHICAGO          ...|                   IL|          606561409|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|           77409.0|      77409.0|                           P|            3483.0|                          P|           0.0|                      P|        80892.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|           6999.0|            21016.0|    0.0|  000452010|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|         77409.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               77409.0|      1|\n",
            "|         PH|12111190381131|   310|           299|JORGE GONZALES RA...|  5358 N CUMBERLAND...|CHICAGO          ...|                  IL|         606561412|           0000005348|                       N |       CUMBERLAND     |                  AVE |CHICAGO          ...|                   IL|          606561409|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          105060.0|     105060.0|                           P|            4728.0|                          P|           0.0|                      P|       109788.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|           9499.0|            28523.0|    0.0|  000452013|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        105060.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              105060.0|      1|\n",
            "|         PH|12111190381139|   310|           299|SALVATORE INTILE ...|  5358 N CUMBERLAND...|CHICAGO          ...|                  IL|         606561412|           0000005358|                       N |       CUMBERLAND     |                  AVE |CHICAGO          ...|                   IL|          606561412|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|           68227.0|      68227.0|                           P|            3069.0|                          P|           0.0|                      P|        71296.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|           9499.0|            18523.0|    0.0|  000452020|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|         68227.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               68227.0|      1|\n",
            "|         PH|12111190381158|   310|           299|ROBERT SUCCAR    ...|  7303 W FULLERTON ...|ELMWOOD PARK     ...|                  IL|         607072618|           0000005348|                       N |       CUMBERLAND     |                  AVE |CHICAGO          ...|                   IL|          606561409|     JEFFERSON      |  71001|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|           82938.0|      82938.0|                           P|            3732.0|                          P|           0.0|                      P|        86670.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|           7499.0|            22517.0|    0.0|  000452039|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|         82938.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               82938.0|      1|\n",
            "|         PH|04234010150000|   133|           206|MICHAEL MATUSCHKA...|  2329 CLOVER LN   ...|NORTHFIELD       ...|                  IL|         600932703|           0000002329|                         |       CLOVER         |                  LN  |NORTHFIELD       ...|                   IL|          600932703|     NORTHFIELD     |  25029|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|         1475833.0|    1475833.0|                           P|           66411.0|                          P|           0.0|                      P|      1542244.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|         113984.0|           332260.0|    0.0|  000133619|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|       1475833.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|             1475833.0|      1|\n",
            "|         PH|12163070300000|   064|           203|NADIA YAGUOB     ...|  10542 STONE HILL ...|ORLAND PARK      ...|                  IL|         604678796|           0000004100|                         |       DENLEY         |                  AVE |SCHILLER PARK    ...|                   IL|          601761825|     LEYDEN         |  20023|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          349414.0|     349414.0|                           P|           15723.0|                          P|           0.0|                      P|       365137.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          16572.0|            49761.0|    0.0|  000465814|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        349414.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              349414.0|      1|\n",
            "|         PH|12163070350000|   064|           523|OHARE SHELL PARTN...|  4111 MANNHEIM RD ...|SCHILLER PK      ...|                  IL|         601761840|           0000004111|                         |       MANNHEIM       |                  RD  |SCHILLER PARK    ...|                   IL|          601761840|     LEYDEN         |  20227|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|       1.6036143E7|  1.6036143E7|                           P|          721626.0|                          P|           0.0|                      P|    1.6757769E7|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|         760565.0|          2283749.0|    0.0|  000465816|                 0|                         0|                0|           0|             1|           0|                       0.0|                       0.0|        X|     1.6036143E7|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|           1.6036143E7|      1|\n",
            "|         PH|12163100080000|   064|           100|FRANK SERAK      ...|  4051 N JUDD AV   ...|SCHILLER PK      ...|                  IL|         601761833|           0000010005|                         |       IRVING PARK    |                  RD  |SCHILLER PARK    ...|                   IL|          601761451|     LEYDEN         |  20023|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|           36226.0|      36226.0|                           P|            1629.0|                          P|           0.0|                      P|        37855.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|           1718.0|             5159.0|    0.0|  000465852|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|         36226.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|               36226.0|      1|\n",
            "|         PH|12163110460000|   064|           203|WOJCIECH KOPENSKI...|  4105 JUDD AVE    ...|SCHILLER PK      ...|                  IL|         601761835|           0000004105|                         |       JUDD           |                  ST  |SCHILLER PARK    ...|                   IL|          601761835|     LEYDEN         |  20023|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          367875.0|     367875.0|                           P|           16554.0|                          P|           0.0|                      P|       384429.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          20778.0|            52390.0|    0.0|  000465877|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        367875.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              367875.0|      1|\n",
            "|         PH|12163110650000|   064|           278|LEROY P PLIER    ...|  4142 N SCOTT ST  ...|SCHILLER PK      ...|                  IL|         601761805|           0000004142|                         |       SCOTT          |                  ST  |SCHILLER PARK    ...|                   IL|          601761805|     LEYDEN         |  20023|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          606758.0|       9101.0|                           P|             274.0|                          P|           0.0|                      P|         9375.0|                       P|        04242023|               486|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          34772.0|            86410.0|    0.0|  000465896|                 0|                         0|                0|           0|             0|           0|                  597657.0|                       0.0|        X|        606758.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              606758.0|      1|\n",
            "|         PH|04234010230000|   133|           204|DAN XU           ...|  140 SUNSET RIDGE ...|NORTHFIELD       ...|                  IL|         600932712|           0000000140|                         |       SUNSET RIDGE   |                  RD  |NORTHFIELD       ...|                   IL|          600932712|     NORTHFIELD     |  25029|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|         1125907.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03262023|               750|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          90412.0|           253480.0|    0.0|  000133623|                 0|                         0|                0|           0|             0|           0|                  155907.0|                       0.0|        X|       1125907.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|             1125907.0|      1|\n",
            "|         PH|12164010350000|   064|           202|DREXEL EMERSON 42...|  8205 W GUNNISON S...|NORRIDGE         ...|                  IL|         607063001|           0000004264|                       N |       EMERSON        |                  AVE |SCHILLER PARK    ...|                   IL|          601761410|     LEYDEN         |  20023|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          344141.0|     344141.0|                           P|           15486.0|                          P|           0.0|                      P|       359627.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          16322.0|            49010.0|    0.0|  000466162|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        344141.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              344141.0|      1|\n",
            "|         PH|12164080050000|   064|           517|KLOTZ ENVIRONMENT...|  9739 W IRVING PK ...|SCHILLER PK      ...|                  IL|         601761938|           0000009739|                         |       IRVING PARK    |                  RD  |SCHILLER PARK    ...|                   IL|          601761938|     LEYDEN         |  20023|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|         1341061.0|    1341061.0|                           P|           60348.0|                          P|           0.0|                      P|      1401409.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          63604.0|           190984.0|    0.0|  000466287|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|       1341061.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|             1341061.0|      1|\n",
            "|         PH|12164080060000|   064|           590|KLOTZ ENVIRONMENT...|  9739 W IRVING PK ...|SCHILLER PK      ...|                  IL|         601761938|           0000009741|                         |       IRVING PARK    |                  RD  |SCHILLER PARK    ...|                   IL|          601761938|     LEYDEN         |  20023|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          303976.0|      43551.0|                           P|             653.0|                          P|           0.0|                      P|        44204.0|                       P|        06012023|               501|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          14417.0|            43290.0|    0.0|  000466288|                 0|                         0|                0|           0|             0|           0|                  260425.0|                       0.0|        X|        303976.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              303976.0|      1|\n",
            "|         PH|12243020380000|   314|           205|EVI THANASENARIS ...|  3518 N OPAL AVE  ...|CHICAGO          ...|                  IL|         606343028|           0000003518|                       N |       OPAL           |                  AVE |CHICAGO          ...|                   IL|          606343028|     JEFFERSON      |  71033|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          217709.0|     217709.0|                           P|            9798.0|                          P|           0.0|                      P|       227507.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          23003.0|            59071.0|    0.0|  000474719|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        217709.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              217709.0|      1|\n",
            "|         PH|12243020500000|   314|           211|FRANCESCA DE LUCA...|  7853 W ADDISON ST...|CHICAGO          ...|                  IL|         606343024|           0000007853|                       W |       ADDISON        |                  ST  |CHICAGO          ...|                   IL|          606343024|     JEFFERSON      |  71033|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          461177.0|     461177.0|                           P|           20754.0|                          P|           0.0|                      P|       481931.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          45003.0|           125131.0|    0.0|  000474727|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        461177.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              461177.0|      1|\n",
            "|         PH|12243090090000|   314|           211|DAVID J RILEY    ...|  3435 N OSAGE AV  ...|CHICAGO          ...|                  IL|         606342954|           0000003435|                       N |       OSAGE          |                  AVE |CHICAGO          ...|                   IL|          606342954|     JEFFERSON      |  71033|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          239845.0|      30000.0|                           N|               0.0|                          P|           0.0|                      P|        30000.0|                       N|        03262023|               750|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          25003.0|            65077.0|    0.0|  000474998|                 0|                         0|                0|           0|             0|           0|                  100000.0|                       0.0|        X|        239845.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              239845.0|      1|\n",
            "|         PH|12243090240000|   314|           205|CARL MEYER       ...|  3450 N ORANGE AVE...|CHICAGO          ...|                  IL|         606342947|           0000003450|                       N |       ORANGE         |                  AVE |CHICAGO          ...|                   IL|          606342947|     JEFFERSON      |  71033|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            P|       1|          PD|                01|          195577.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                       P|        03192023|               700|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          21003.0|            53066.0|    0.0|  000475013|                 0|                         0|                0|           0|             0|           0|                   95577.0|                       0.0|        X|        195577.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              195577.0|      1|\n",
            "|         PH|12243100140000|   314|           205|DONALD M DESALVO ...|  3421 N ORANGE AVE...|CHICAGO          ...|                  IL|         606342948|           0000003421|                       N |       ORANGE         |                  AVE |CHICAGO          ...|                   IL|          606342948|     JEFFERSON      |  71033|       00|              0|           0|                 0|                       0|          R|     00|   2022|    2022|            O|       1|          PD|                01|          317308.0|     317308.0|                           P|           14280.0|                          P|           0.0|                      P|       331588.0|                       P|        00000000|               000|                02|            0.0|            0.0|          0.0|                           P|               0.0|                          P|           0.0|                      P|            0.0|                     P|        00000000|               000|   0000000|                N|          32003.0|            86095.0|    0.0|  000475043|                 0|                         0|                0|           0|             0|           0|                       0.0|                       0.0|        X|        317308.0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|               0.0|                     0.0|                           0.0|          0.0|              317308.0|      1|\n",
            "+-----------+--------------+------+--------------+--------------------+----------------------+--------------------+--------------------+------------------+---------------------+-------------------------+----------------------+----------------------+--------------------+---------------------+-------------------+--------------------+-------+---------+---------------+------------+------------------+------------------------+-----------+-------+-------+--------+-------------+--------+------------+------------------+------------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+------------------------+----------------+------------------+------------------+---------------+---------------+-------------+----------------------------+------------------+---------------------------+--------------+-----------------------+---------------+----------------------+----------------+------------------+----------+-----------------+-----------------+-------------------+-------+-----------+------------------+--------------------------+-----------------+------------+--------------+------------+--------------------------+--------------------------+---------+----------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------+------------------------+------------------------------+-------------+----------------------+-------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Univariate analysis\n",
        "var = \"LastPaymentSource1\"\n",
        "data1.groupBy(var).agg(count(var).alias(\"cnt\"), count(var)/tot*100).sort(col(\"cnt\").desc()).show(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sdJ-5nnrPqi7",
        "outputId": "27483799-3df7-4ca4-ebe8-5c32417ec6be"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------------+-------+---------------------------------------------+\n",
            "|LastPaymentSource1|    cnt|((count(LastPaymentSource1) / 9325500) * 100)|\n",
            "+------------------+-------+---------------------------------------------+\n",
            "|               600|3705416|                            39.73423408932497|\n",
            "|               700|1739914|                            18.65759476703662|\n",
            "|               444|1412040|                           15.141708219398422|\n",
            "|               000| 984367|                            10.55564849069755|\n",
            "|               431| 486952|                           5.2217253766554075|\n",
            "|               481| 320967|                           3.4418208138973783|\n",
            "|               750| 195508|                           2.0964881239611817|\n",
            "|               802| 144734|                           1.5520240201597768|\n",
            "|               501| 102427|                           1.0983539756581415|\n",
            "|               801|  62324|                           0.6683180526513324|\n",
            "+------------------+-------+---------------------------------------------+\n",
            "only showing top 10 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Cleaning and Transformation"
      ],
      "metadata": {
        "id": "owdXDjckXTXo"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Correlation analysis\n",
        "Remove numerically correlated features with Pearson's correlation > 0.5 (the threshold can be discussed). Proceed to 'Drop columns' part if you want to run the whole model."
      ],
      "metadata": {
        "id": "L0UKD6-mtHeO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "c = data4.columns[-6:]\n",
        "#data4 = data4.cast\n",
        "for c1 in c:\n",
        "  data4 = data4.withColumn(c1, data4[c1].cast(\"float\"))"
      ],
      "metadata": {
        "id": "a_Bq7YuQA_EF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data4.dtypes"
      ],
      "metadata": {
        "id": "dqen0BV_BZqL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "numeric_cols = []\n",
        "for f in data4.schema.fields:\n",
        "  if not isinstance(f.dataType, StringType):\n",
        "    if not isinstance(f.dataType, DateType):\n",
        "      numeric_cols.append(f.name)\n",
        "numeric_cols"
      ],
      "metadata": {
        "id": "HIgiU-BWnoji",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ce3b65d8-15ad-42e3-fe29-563778065f62"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Volume',\n",
              " 'TaxpayerMailingState',\n",
              " 'TaxpayerPropertyCity',\n",
              " 'BillType',\n",
              " 'AdjustedAmountDue1',\n",
              " 'TaxAmountDue1',\n",
              " 'CostAmountDue1',\n",
              " 'CofENumber',\n",
              " 'TaxRate',\n",
              " 'ReturningVetExemptionAmount',\n",
              " 'DisabledPersonExemptionAmount',\n",
              " 'DisabledVetExemptionAmount',\n",
              " 'DisabledPersonVetExemptionAmount',\n",
              " 'HomeownerExemptAmount',\n",
              " 'SeniorFreezeExemptAmount',\n",
              " 'LongtimeHomeownersExemptAmount',\n",
              " 'LatePmt1',\n",
              " 'LatePmt2',\n",
              " 'DiffPmt1',\n",
              " 'WOE_TaxpayerPropertySuffix',\n",
              " 'WOE_Classification',\n",
              " 'WOE_TaxpayerPropertyTown',\n",
              " 'WOE_TaxStatus',\n",
              " 'WOE_AccountStatus',\n",
              " 'WOE_LastPaymentSource1']"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vector_col = \"corr_features\"\n",
        "assembler = VectorAssembler(inputCols=numeric_cols, outputCol=vector_col)\n",
        "df_vector = assembler.transform(data4).select(vector_col)\n",
        "\n",
        "# get correlation matrix\n",
        "matrix = Correlation.corr(df_vector, vector_col)"
      ],
      "metadata": {
        "id": "DjWk1KmVnWIz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corr_matrix = matrix.collect()[0][0].toArray().tolist()\n",
        "corr_matrix_df = pd.DataFrame(data=corr_matrix, columns = numeric_cols, index=numeric_cols)"
      ],
      "metadata": {
        "id": "5j3IvAYRvQ-V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "numeric_corr_cols = [\"OriginalTaxDue2\", \"AdjustedTaxDue2\", \"AssessedValuation\", \"EqualizedEvaluation\", \"LastPaymentReceivedAmount2\",\n",
        " \"TaxDueEstimated1\", \"LastPaymentReceivedAmount1\", \"AdjustedAmountDue1_Org\",\n",
        " \"InterestAmountDue1\", \"TotalAmountDue1\", \"InterestAmountDue2\", \"TotalAmountDue2\",\n",
        " \"SeniorExemptAmount\", \"VeteranExempt\"\n",
        " ]"
      ],
      "metadata": {
        "id": "sBvBupkhuNrc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corr_matrix_df = corr_matrix_df.drop(numeric_corr_cols, axis=1)\n",
        "corr_matrix_df = corr_matrix_df.drop(numeric_corr_cols, axis=0)"
      ],
      "metadata": {
        "id": "U0S0JxewvHWq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "corr_matrix_df.style.background_gradient(cmap='coolwarm').set_precision(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 607
        },
        "id": "1a59NGiLrJmk",
        "outputId": "669b544f-49fd-495b-eb12-f58d354efa20"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-80-68a7647adc47>:1: FutureWarning: this method is deprecated in favour of `Styler.format(precision=..)`\n",
            "  corr_matrix_df.style.background_gradient(cmap='coolwarm').set_precision(2)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pandas.io.formats.style.Styler at 0x7fe3356709a0>"
            ],
            "text/html": [
              "<style type=\"text/css\">\n",
              "#T_e74bb_row0_col0, #T_e74bb_row1_col1, #T_e74bb_row2_col2, #T_e74bb_row3_col3, #T_e74bb_row4_col4, #T_e74bb_row5_col5, #T_e74bb_row6_col6, #T_e74bb_row7_col7, #T_e74bb_row8_col8, #T_e74bb_row9_col9, #T_e74bb_row10_col10, #T_e74bb_row11_col11, #T_e74bb_row12_col12, #T_e74bb_row13_col13, #T_e74bb_row15_col15 {\n",
              "  background-color: #b40426;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col1, #T_e74bb_row6_col10 {\n",
              "  background-color: #4e68d8;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col2, #T_e74bb_row1_col0, #T_e74bb_row4_col2, #T_e74bb_row7_col13, #T_e74bb_row8_col2, #T_e74bb_row11_col13, #T_e74bb_row15_col5 {\n",
              "  background-color: #4b64d5;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col3, #T_e74bb_row3_col0, #T_e74bb_row4_col13, #T_e74bb_row10_col8, #T_e74bb_row11_col2 {\n",
              "  background-color: #455cce;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col4, #T_e74bb_row1_col8, #T_e74bb_row1_col12, #T_e74bb_row2_col4, #T_e74bb_row3_col8, #T_e74bb_row3_col12, #T_e74bb_row4_col8, #T_e74bb_row4_col12, #T_e74bb_row6_col5, #T_e74bb_row6_col8, #T_e74bb_row6_col12, #T_e74bb_row7_col4, #T_e74bb_row8_col4, #T_e74bb_row8_col12, #T_e74bb_row9_col5, #T_e74bb_row9_col8, #T_e74bb_row9_col12, #T_e74bb_row11_col8, #T_e74bb_row12_col8, #T_e74bb_row13_col0, #T_e74bb_row13_col3, #T_e74bb_row15_col12 {\n",
              "  background-color: #3e51c5;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col5, #T_e74bb_row0_col11, #T_e74bb_row4_col11, #T_e74bb_row6_col4, #T_e74bb_row7_col12, #T_e74bb_row9_col4, #T_e74bb_row10_col0, #T_e74bb_row10_col15, #T_e74bb_row12_col4, #T_e74bb_row12_col7, #T_e74bb_row13_col1, #T_e74bb_row13_col2 {\n",
              "  background-color: #3f53c6;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col6, #T_e74bb_row0_col9, #T_e74bb_row0_col14, #T_e74bb_row1_col6, #T_e74bb_row1_col9, #T_e74bb_row1_col14, #T_e74bb_row2_col0, #T_e74bb_row2_col6, #T_e74bb_row2_col7, #T_e74bb_row2_col9, #T_e74bb_row2_col10, #T_e74bb_row2_col11, #T_e74bb_row2_col13, #T_e74bb_row2_col14, #T_e74bb_row3_col6, #T_e74bb_row3_col9, #T_e74bb_row3_col14, #T_e74bb_row4_col6, #T_e74bb_row4_col9, #T_e74bb_row4_col14, #T_e74bb_row5_col6, #T_e74bb_row5_col9, #T_e74bb_row5_col12, #T_e74bb_row5_col14, #T_e74bb_row6_col9, #T_e74bb_row6_col14, #T_e74bb_row7_col0, #T_e74bb_row7_col6, #T_e74bb_row7_col9, #T_e74bb_row7_col14, #T_e74bb_row8_col0, #T_e74bb_row8_col6, #T_e74bb_row8_col9, #T_e74bb_row8_col14, #T_e74bb_row9_col6, #T_e74bb_row9_col14, #T_e74bb_row10_col1, #T_e74bb_row10_col2, #T_e74bb_row10_col3, #T_e74bb_row10_col4, #T_e74bb_row10_col6, #T_e74bb_row10_col9, #T_e74bb_row10_col12, #T_e74bb_row10_col14, #T_e74bb_row11_col0, #T_e74bb_row11_col5, #T_e74bb_row11_col6, #T_e74bb_row11_col9, #T_e74bb_row11_col14, #T_e74bb_row12_col5, #T_e74bb_row12_col6, #T_e74bb_row12_col9, #T_e74bb_row12_col14, #T_e74bb_row13_col5, #T_e74bb_row13_col6, #T_e74bb_row13_col9, #T_e74bb_row13_col14, #T_e74bb_row13_col15, #T_e74bb_row14_col14, #T_e74bb_row15_col6, #T_e74bb_row15_col8, #T_e74bb_row15_col9, #T_e74bb_row15_col13, #T_e74bb_row15_col14 {\n",
              "  background-color: #3b4cc0;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col7, #T_e74bb_row2_col15, #T_e74bb_row4_col0, #T_e74bb_row6_col0, #T_e74bb_row7_col3, #T_e74bb_row8_col3, #T_e74bb_row8_col7, #T_e74bb_row9_col0, #T_e74bb_row11_col1, #T_e74bb_row11_col3, #T_e74bb_row12_col0, #T_e74bb_row12_col3, #T_e74bb_row13_col4, #T_e74bb_row13_col12, #T_e74bb_row15_col0, #T_e74bb_row15_col1, #T_e74bb_row15_col3, #T_e74bb_row15_col7, #T_e74bb_row15_col11 {\n",
              "  background-color: #3c4ec2;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col8, #T_e74bb_row0_col12, #T_e74bb_row1_col7, #T_e74bb_row2_col8, #T_e74bb_row2_col12, #T_e74bb_row3_col7, #T_e74bb_row4_col7, #T_e74bb_row5_col0, #T_e74bb_row5_col11, #T_e74bb_row6_col1, #T_e74bb_row6_col3, #T_e74bb_row6_col7, #T_e74bb_row7_col1, #T_e74bb_row7_col8, #T_e74bb_row8_col1, #T_e74bb_row9_col1, #T_e74bb_row9_col3, #T_e74bb_row9_col7, #T_e74bb_row11_col4, #T_e74bb_row11_col12, #T_e74bb_row12_col1, #T_e74bb_row15_col4 {\n",
              "  background-color: #3d50c3;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col10 {\n",
              "  background-color: #506bda;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col13, #T_e74bb_row1_col13, #T_e74bb_row3_col10, #T_e74bb_row3_col13, #T_e74bb_row7_col2, #T_e74bb_row7_col11, #T_e74bb_row10_col7 {\n",
              "  background-color: #4a63d3;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row0_col15, #T_e74bb_row3_col15, #T_e74bb_row4_col10, #T_e74bb_row7_col15 {\n",
              "  background-color: #485fd1;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row1_col2 {\n",
              "  background-color: #7093f3;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row1_col3 {\n",
              "  background-color: #d8dce2;\n",
              "  color: #000000;\n",
              "}\n",
              "#T_e74bb_row1_col4, #T_e74bb_row6_col11, #T_e74bb_row9_col11 {\n",
              "  background-color: #4257c9;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row1_col5, #T_e74bb_row1_col10, #T_e74bb_row5_col1, #T_e74bb_row6_col13, #T_e74bb_row6_col15, #T_e74bb_row9_col13, #T_e74bb_row9_col15, #T_e74bb_row10_col13, #T_e74bb_row12_col10, #T_e74bb_row12_col15 {\n",
              "  background-color: #4961d2;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row1_col11, #T_e74bb_row3_col11, #T_e74bb_row4_col1, #T_e74bb_row8_col11, #T_e74bb_row12_col11, #T_e74bb_row13_col7, #T_e74bb_row15_col2 {\n",
              "  background-color: #4055c8;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row1_col15, #T_e74bb_row3_col4, #T_e74bb_row4_col15, #T_e74bb_row5_col8, #T_e74bb_row8_col5, #T_e74bb_row11_col7, #T_e74bb_row12_col13 {\n",
              "  background-color: #465ecf;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row2_col1 {\n",
              "  background-color: #6282ea;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row2_col3 {\n",
              "  background-color: #4f69d9;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row2_col5 {\n",
              "  background-color: #6b8df0;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row3_col1 {\n",
              "  background-color: #d9dce1;\n",
              "  color: #000000;\n",
              "}\n",
              "#T_e74bb_row3_col2 {\n",
              "  background-color: #5f7fe8;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row3_col5, #T_e74bb_row4_col3, #T_e74bb_row5_col4, #T_e74bb_row5_col13, #T_e74bb_row8_col15, #T_e74bb_row13_col11 {\n",
              "  background-color: #445acc;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row4_col5, #T_e74bb_row5_col3, #T_e74bb_row11_col15, #T_e74bb_row13_col8, #T_e74bb_row15_col10 {\n",
              "  background-color: #4358cb;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row5_col2 {\n",
              "  background-color: #7a9df8;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row5_col7, #T_e74bb_row7_col5 {\n",
              "  background-color: #5977e3;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row5_col10 {\n",
              "  background-color: #6485ec;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row5_col15 {\n",
              "  background-color: #5673e0;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row6_col2, #T_e74bb_row8_col13, #T_e74bb_row9_col2, #T_e74bb_row9_col10, #T_e74bb_row12_col2, #T_e74bb_row13_col10 {\n",
              "  background-color: #4c66d6;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row7_col10 {\n",
              "  background-color: #5a78e4;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row8_col10 {\n",
              "  background-color: #536edd;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row10_col5 {\n",
              "  background-color: #5572df;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row10_col11 {\n",
              "  background-color: #779af7;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row11_col10 {\n",
              "  background-color: #82a6fb;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "#T_e74bb_row14_col0, #T_e74bb_row14_col1, #T_e74bb_row14_col2, #T_e74bb_row14_col3, #T_e74bb_row14_col4, #T_e74bb_row14_col5, #T_e74bb_row14_col6, #T_e74bb_row14_col7, #T_e74bb_row14_col8, #T_e74bb_row14_col9, #T_e74bb_row14_col10, #T_e74bb_row14_col11, #T_e74bb_row14_col12, #T_e74bb_row14_col13, #T_e74bb_row14_col15 {\n",
              "  background-color: #000000;\n",
              "  color: #f1f1f1;\n",
              "}\n",
              "</style>\n",
              "<table id=\"T_e74bb\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th class=\"blank level0\" >&nbsp;</th>\n",
              "      <th id=\"T_e74bb_level0_col0\" class=\"col_heading level0 col0\" >AdjustedAmountDue1</th>\n",
              "      <th id=\"T_e74bb_level0_col1\" class=\"col_heading level0 col1\" >TaxAmountDue1</th>\n",
              "      <th id=\"T_e74bb_level0_col2\" class=\"col_heading level0 col2\" >CostAmountDue1</th>\n",
              "      <th id=\"T_e74bb_level0_col3\" class=\"col_heading level0 col3\" >TaxAmountDue2</th>\n",
              "      <th id=\"T_e74bb_level0_col4\" class=\"col_heading level0 col4\" >CostAmountDue2</th>\n",
              "      <th id=\"T_e74bb_level0_col5\" class=\"col_heading level0 col5\" >TaxRate</th>\n",
              "      <th id=\"T_e74bb_level0_col6\" class=\"col_heading level0 col6\" >ReturningVetExemptionAmount</th>\n",
              "      <th id=\"T_e74bb_level0_col7\" class=\"col_heading level0 col7\" >DisabledPersonExemptionAmount</th>\n",
              "      <th id=\"T_e74bb_level0_col8\" class=\"col_heading level0 col8\" >DisabledVetExemptionAmount</th>\n",
              "      <th id=\"T_e74bb_level0_col9\" class=\"col_heading level0 col9\" >DisabledPersonVetExemptionAmount</th>\n",
              "      <th id=\"T_e74bb_level0_col10\" class=\"col_heading level0 col10\" >HomeownerExemptAmount</th>\n",
              "      <th id=\"T_e74bb_level0_col11\" class=\"col_heading level0 col11\" >SeniorFreezeExemptAmount</th>\n",
              "      <th id=\"T_e74bb_level0_col12\" class=\"col_heading level0 col12\" >LongtimeHomeownersExemptAmount</th>\n",
              "      <th id=\"T_e74bb_level0_col13\" class=\"col_heading level0 col13\" >Year</th>\n",
              "      <th id=\"T_e74bb_level0_col14\" class=\"col_heading level0 col14\" >LatePmt1</th>\n",
              "      <th id=\"T_e74bb_level0_col15\" class=\"col_heading level0 col15\" >LatePmt2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row0\" class=\"row_heading level0 row0\" >AdjustedAmountDue1</th>\n",
              "      <td id=\"T_e74bb_row0_col0\" class=\"data row0 col0\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row0_col1\" class=\"data row0 col1\" >0.05</td>\n",
              "      <td id=\"T_e74bb_row0_col2\" class=\"data row0 col2\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row0_col3\" class=\"data row0 col3\" >0.03</td>\n",
              "      <td id=\"T_e74bb_row0_col4\" class=\"data row0 col4\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row0_col5\" class=\"data row0 col5\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row0_col6\" class=\"data row0 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row0_col7\" class=\"data row0 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row0_col8\" class=\"data row0 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row0_col9\" class=\"data row0 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row0_col10\" class=\"data row0 col10\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row0_col11\" class=\"data row0 col11\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row0_col12\" class=\"data row0 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row0_col13\" class=\"data row0 col13\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row0_col14\" class=\"data row0 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row0_col15\" class=\"data row0 col15\" >-0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row1\" class=\"row_heading level0 row1\" >TaxAmountDue1</th>\n",
              "      <td id=\"T_e74bb_row1_col0\" class=\"data row1 col0\" >0.05</td>\n",
              "      <td id=\"T_e74bb_row1_col1\" class=\"data row1 col1\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row1_col2\" class=\"data row1 col2\" >0.11</td>\n",
              "      <td id=\"T_e74bb_row1_col3\" class=\"data row1 col3\" >0.48</td>\n",
              "      <td id=\"T_e74bb_row1_col4\" class=\"data row1 col4\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row1_col5\" class=\"data row1 col5\" >0.04</td>\n",
              "      <td id=\"T_e74bb_row1_col6\" class=\"data row1 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row1_col7\" class=\"data row1 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row1_col8\" class=\"data row1 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row1_col9\" class=\"data row1 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row1_col10\" class=\"data row1 col10\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row1_col11\" class=\"data row1 col11\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row1_col12\" class=\"data row1 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row1_col13\" class=\"data row1 col13\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row1_col14\" class=\"data row1 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row1_col15\" class=\"data row1 col15\" >-0.01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row2\" class=\"row_heading level0 row2\" >CostAmountDue1</th>\n",
              "      <td id=\"T_e74bb_row2_col0\" class=\"data row2 col0\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row2_col1\" class=\"data row2 col1\" >0.11</td>\n",
              "      <td id=\"T_e74bb_row2_col2\" class=\"data row2 col2\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row2_col3\" class=\"data row2 col3\" >0.06</td>\n",
              "      <td id=\"T_e74bb_row2_col4\" class=\"data row2 col4\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row2_col5\" class=\"data row2 col5\" >0.14</td>\n",
              "      <td id=\"T_e74bb_row2_col6\" class=\"data row2 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row2_col7\" class=\"data row2 col7\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row2_col8\" class=\"data row2 col8\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row2_col9\" class=\"data row2 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row2_col10\" class=\"data row2 col10\" >-0.07</td>\n",
              "      <td id=\"T_e74bb_row2_col11\" class=\"data row2 col11\" >-0.02</td>\n",
              "      <td id=\"T_e74bb_row2_col12\" class=\"data row2 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row2_col13\" class=\"data row2 col13\" >-0.05</td>\n",
              "      <td id=\"T_e74bb_row2_col14\" class=\"data row2 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row2_col15\" class=\"data row2 col15\" >-0.04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row3\" class=\"row_heading level0 row3\" >TaxAmountDue2</th>\n",
              "      <td id=\"T_e74bb_row3_col0\" class=\"data row3 col0\" >0.03</td>\n",
              "      <td id=\"T_e74bb_row3_col1\" class=\"data row3 col1\" >0.48</td>\n",
              "      <td id=\"T_e74bb_row3_col2\" class=\"data row3 col2\" >0.06</td>\n",
              "      <td id=\"T_e74bb_row3_col3\" class=\"data row3 col3\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row3_col4\" class=\"data row3 col4\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row3_col5\" class=\"data row3 col5\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row3_col6\" class=\"data row3 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row3_col7\" class=\"data row3 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row3_col8\" class=\"data row3 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row3_col9\" class=\"data row3 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row3_col10\" class=\"data row3 col10\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row3_col11\" class=\"data row3 col11\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row3_col12\" class=\"data row3 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row3_col13\" class=\"data row3 col13\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row3_col14\" class=\"data row3 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row3_col15\" class=\"data row3 col15\" >-0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row4\" class=\"row_heading level0 row4\" >CostAmountDue2</th>\n",
              "      <td id=\"T_e74bb_row4_col0\" class=\"data row4 col0\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row4_col1\" class=\"data row4 col1\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row4_col2\" class=\"data row4 col2\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row4_col3\" class=\"data row4 col3\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row4_col4\" class=\"data row4 col4\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row4_col5\" class=\"data row4 col5\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row4_col6\" class=\"data row4 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row4_col7\" class=\"data row4 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row4_col8\" class=\"data row4 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row4_col9\" class=\"data row4 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row4_col10\" class=\"data row4 col10\" >-0.02</td>\n",
              "      <td id=\"T_e74bb_row4_col11\" class=\"data row4 col11\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row4_col12\" class=\"data row4 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row4_col13\" class=\"data row4 col13\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row4_col14\" class=\"data row4 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row4_col15\" class=\"data row4 col15\" >-0.01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row5\" class=\"row_heading level0 row5\" >TaxRate</th>\n",
              "      <td id=\"T_e74bb_row5_col0\" class=\"data row5 col0\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row5_col1\" class=\"data row5 col1\" >0.04</td>\n",
              "      <td id=\"T_e74bb_row5_col2\" class=\"data row5 col2\" >0.14</td>\n",
              "      <td id=\"T_e74bb_row5_col3\" class=\"data row5 col3\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row5_col4\" class=\"data row5 col4\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row5_col5\" class=\"data row5 col5\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row5_col6\" class=\"data row5 col6\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row5_col7\" class=\"data row5 col7\" >0.09</td>\n",
              "      <td id=\"T_e74bb_row5_col8\" class=\"data row5 col8\" >0.03</td>\n",
              "      <td id=\"T_e74bb_row5_col9\" class=\"data row5 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row5_col10\" class=\"data row5 col10\" >0.08</td>\n",
              "      <td id=\"T_e74bb_row5_col11\" class=\"data row5 col11\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row5_col12\" class=\"data row5 col12\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row5_col13\" class=\"data row5 col13\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row5_col14\" class=\"data row5 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row5_col15\" class=\"data row5 col15\" >0.04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row6\" class=\"row_heading level0 row6\" >ReturningVetExemptionAmount</th>\n",
              "      <td id=\"T_e74bb_row6_col0\" class=\"data row6 col0\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col1\" class=\"data row6 col1\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col2\" class=\"data row6 col2\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col3\" class=\"data row6 col3\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col4\" class=\"data row6 col4\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col5\" class=\"data row6 col5\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col6\" class=\"data row6 col6\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row6_col7\" class=\"data row6 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col8\" class=\"data row6 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col9\" class=\"data row6 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col10\" class=\"data row6 col10\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col11\" class=\"data row6 col11\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col12\" class=\"data row6 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col13\" class=\"data row6 col13\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row6_col14\" class=\"data row6 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row6_col15\" class=\"data row6 col15\" >-0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row7\" class=\"row_heading level0 row7\" >DisabledPersonExemptionAmount</th>\n",
              "      <td id=\"T_e74bb_row7_col0\" class=\"data row7 col0\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row7_col1\" class=\"data row7 col1\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row7_col2\" class=\"data row7 col2\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row7_col3\" class=\"data row7 col3\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row7_col4\" class=\"data row7 col4\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row7_col5\" class=\"data row7 col5\" >0.09</td>\n",
              "      <td id=\"T_e74bb_row7_col6\" class=\"data row7 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row7_col7\" class=\"data row7 col7\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row7_col8\" class=\"data row7 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row7_col9\" class=\"data row7 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row7_col10\" class=\"data row7 col10\" >0.04</td>\n",
              "      <td id=\"T_e74bb_row7_col11\" class=\"data row7 col11\" >0.03</td>\n",
              "      <td id=\"T_e74bb_row7_col12\" class=\"data row7 col12\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row7_col13\" class=\"data row7 col13\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row7_col14\" class=\"data row7 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row7_col15\" class=\"data row7 col15\" >-0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row8\" class=\"row_heading level0 row8\" >DisabledVetExemptionAmount</th>\n",
              "      <td id=\"T_e74bb_row8_col0\" class=\"data row8 col0\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col1\" class=\"data row8 col1\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col2\" class=\"data row8 col2\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row8_col3\" class=\"data row8 col3\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col4\" class=\"data row8 col4\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col5\" class=\"data row8 col5\" >0.03</td>\n",
              "      <td id=\"T_e74bb_row8_col6\" class=\"data row8 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col7\" class=\"data row8 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col8\" class=\"data row8 col8\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row8_col9\" class=\"data row8 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col10\" class=\"data row8 col10\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row8_col11\" class=\"data row8 col11\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col12\" class=\"data row8 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row8_col13\" class=\"data row8 col13\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row8_col14\" class=\"data row8 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row8_col15\" class=\"data row8 col15\" >-0.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row9\" class=\"row_heading level0 row9\" >DisabledPersonVetExemptionAmount</th>\n",
              "      <td id=\"T_e74bb_row9_col0\" class=\"data row9 col0\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col1\" class=\"data row9 col1\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col2\" class=\"data row9 col2\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col3\" class=\"data row9 col3\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col4\" class=\"data row9 col4\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col5\" class=\"data row9 col5\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col6\" class=\"data row9 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col7\" class=\"data row9 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col8\" class=\"data row9 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col9\" class=\"data row9 col9\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row9_col10\" class=\"data row9 col10\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col11\" class=\"data row9 col11\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col12\" class=\"data row9 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col13\" class=\"data row9 col13\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row9_col14\" class=\"data row9 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row9_col15\" class=\"data row9 col15\" >-0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row10\" class=\"row_heading level0 row10\" >HomeownerExemptAmount</th>\n",
              "      <td id=\"T_e74bb_row10_col0\" class=\"data row10 col0\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row10_col1\" class=\"data row10 col1\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row10_col2\" class=\"data row10 col2\" >-0.07</td>\n",
              "      <td id=\"T_e74bb_row10_col3\" class=\"data row10 col3\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row10_col4\" class=\"data row10 col4\" >-0.02</td>\n",
              "      <td id=\"T_e74bb_row10_col5\" class=\"data row10 col5\" >0.08</td>\n",
              "      <td id=\"T_e74bb_row10_col6\" class=\"data row10 col6\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row10_col7\" class=\"data row10 col7\" >0.04</td>\n",
              "      <td id=\"T_e74bb_row10_col8\" class=\"data row10 col8\" >0.02</td>\n",
              "      <td id=\"T_e74bb_row10_col9\" class=\"data row10 col9\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row10_col10\" class=\"data row10 col10\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row10_col11\" class=\"data row10 col11\" >0.17</td>\n",
              "      <td id=\"T_e74bb_row10_col12\" class=\"data row10 col12\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row10_col13\" class=\"data row10 col13\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row10_col14\" class=\"data row10 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row10_col15\" class=\"data row10 col15\" >-0.03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row11\" class=\"row_heading level0 row11\" >SeniorFreezeExemptAmount</th>\n",
              "      <td id=\"T_e74bb_row11_col0\" class=\"data row11 col0\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row11_col1\" class=\"data row11 col1\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row11_col2\" class=\"data row11 col2\" >-0.02</td>\n",
              "      <td id=\"T_e74bb_row11_col3\" class=\"data row11 col3\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row11_col4\" class=\"data row11 col4\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row11_col5\" class=\"data row11 col5\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row11_col6\" class=\"data row11 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row11_col7\" class=\"data row11 col7\" >0.03</td>\n",
              "      <td id=\"T_e74bb_row11_col8\" class=\"data row11 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row11_col9\" class=\"data row11 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row11_col10\" class=\"data row11 col10\" >0.17</td>\n",
              "      <td id=\"T_e74bb_row11_col11\" class=\"data row11 col11\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row11_col12\" class=\"data row11 col12\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row11_col13\" class=\"data row11 col13\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row11_col14\" class=\"data row11 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row11_col15\" class=\"data row11 col15\" >-0.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row12\" class=\"row_heading level0 row12\" >LongtimeHomeownersExemptAmount</th>\n",
              "      <td id=\"T_e74bb_row12_col0\" class=\"data row12 col0\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col1\" class=\"data row12 col1\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col2\" class=\"data row12 col2\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col3\" class=\"data row12 col3\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col4\" class=\"data row12 col4\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col5\" class=\"data row12 col5\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row12_col6\" class=\"data row12 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col7\" class=\"data row12 col7\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row12_col8\" class=\"data row12 col8\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col9\" class=\"data row12 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col10\" class=\"data row12 col10\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row12_col11\" class=\"data row12 col11\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row12_col12\" class=\"data row12 col12\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row12_col13\" class=\"data row12 col13\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row12_col14\" class=\"data row12 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row12_col15\" class=\"data row12 col15\" >0.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row13\" class=\"row_heading level0 row13\" >Year</th>\n",
              "      <td id=\"T_e74bb_row13_col0\" class=\"data row13 col0\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row13_col1\" class=\"data row13 col1\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row13_col2\" class=\"data row13 col2\" >-0.05</td>\n",
              "      <td id=\"T_e74bb_row13_col3\" class=\"data row13 col3\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row13_col4\" class=\"data row13 col4\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row13_col5\" class=\"data row13 col5\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row13_col6\" class=\"data row13 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row13_col7\" class=\"data row13 col7\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row13_col8\" class=\"data row13 col8\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row13_col9\" class=\"data row13 col9\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row13_col10\" class=\"data row13 col10\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row13_col11\" class=\"data row13 col11\" >0.01</td>\n",
              "      <td id=\"T_e74bb_row13_col12\" class=\"data row13 col12\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row13_col13\" class=\"data row13 col13\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row13_col14\" class=\"data row13 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row13_col15\" class=\"data row13 col15\" >-0.05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row14\" class=\"row_heading level0 row14\" >LatePmt1</th>\n",
              "      <td id=\"T_e74bb_row14_col0\" class=\"data row14 col0\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col1\" class=\"data row14 col1\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col2\" class=\"data row14 col2\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col3\" class=\"data row14 col3\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col4\" class=\"data row14 col4\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col5\" class=\"data row14 col5\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col6\" class=\"data row14 col6\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col7\" class=\"data row14 col7\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col8\" class=\"data row14 col8\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col9\" class=\"data row14 col9\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col10\" class=\"data row14 col10\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col11\" class=\"data row14 col11\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col12\" class=\"data row14 col12\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col13\" class=\"data row14 col13\" >nan</td>\n",
              "      <td id=\"T_e74bb_row14_col14\" class=\"data row14 col14\" >1.00</td>\n",
              "      <td id=\"T_e74bb_row14_col15\" class=\"data row14 col15\" >nan</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_e74bb_level0_row15\" class=\"row_heading level0 row15\" >LatePmt2</th>\n",
              "      <td id=\"T_e74bb_row15_col0\" class=\"data row15 col0\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row15_col1\" class=\"data row15 col1\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row15_col2\" class=\"data row15 col2\" >-0.04</td>\n",
              "      <td id=\"T_e74bb_row15_col3\" class=\"data row15 col3\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row15_col4\" class=\"data row15 col4\" >-0.01</td>\n",
              "      <td id=\"T_e74bb_row15_col5\" class=\"data row15 col5\" >0.04</td>\n",
              "      <td id=\"T_e74bb_row15_col6\" class=\"data row15 col6\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row15_col7\" class=\"data row15 col7\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row15_col8\" class=\"data row15 col8\" >-0.02</td>\n",
              "      <td id=\"T_e74bb_row15_col9\" class=\"data row15 col9\" >-0.00</td>\n",
              "      <td id=\"T_e74bb_row15_col10\" class=\"data row15 col10\" >-0.03</td>\n",
              "      <td id=\"T_e74bb_row15_col11\" class=\"data row15 col11\" >-0.02</td>\n",
              "      <td id=\"T_e74bb_row15_col12\" class=\"data row15 col12\" >0.00</td>\n",
              "      <td id=\"T_e74bb_row15_col13\" class=\"data row15 col13\" >-0.05</td>\n",
              "      <td id=\"T_e74bb_row15_col14\" class=\"data row15 col14\" >nan</td>\n",
              "      <td id=\"T_e74bb_row15_col15\" class=\"data row15 col15\" >1.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {},
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Outlier treatment\n",
        "Outlier treatment of numerical variables capped by 1st and 99th percentile values, ie,\n",
        "- if value < 1st percentile value, then use 1st percentile value\n",
        "- if value > 99th percentile value, then use 99th percentile value\n",
        "\n",
        "Proceed to 'Drop columns' part if you want to run the whole model."
      ],
      "metadata": {
        "id": "Jad3lg7LtLMX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data1 = data1.withColumn(\"AdjustedAmountDue1\", when(col(\"AdjustedAmountDue1\")>3186751,3186751).otherwise(col(\"AdjustedAmountDue1\")))"
      ],
      "metadata": {
        "id": "fQZVsI05YQB4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Drop columns\n",
        "\n",
        "- drop_cols_inst2 - Drop metrics related to installment2. This is needed for model 1 where we predict Late Payment for 2nd installment at Feb/Mar where we dont have this information yet\n",
        "- drop_cols_univariate (Univariate analysis) - Group by categorical variables and get the count & %count of each category. If one of the categories in the feature is >95%, then remove this feature as it will not be very useful in the model. Some exceptions are present where the feature makes good business sense to be included\n",
        "- drop_cols_numric_corr - Drop numerical columns that are correlated with Pearsons correlation >0.5\n"
      ],
      "metadata": {
        "id": "w_FSVDuRtPHb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "drop_cols_inst2 = data1.columns[42:53] + [\"LastPaymentSource2\"]\n",
        "drop_cols_unrelated = [\"SegmentCode\", \"TaxpayerName\", \"TaxpayerMailingAddress\", \"TaxpayerMailingZip\",\n",
        "                       \"TaxpayerPropertyHouse\",\"TaxpayerPropertyDirection\", \"TaxpayerPropertyStreet\",\n",
        "                       \"TaxpayerPropertyZip\", \"RecordCount\"\n",
        "                       ]\n",
        "drop_cols_univariate = [\"LongtimeHomeownersExempt\", \"TaxInfoType\", \"TaxType\", \"TaxpayerPropertyState\", \"BillYear\", \"SegmentCode2\",\n",
        "                        \"InstallmentNumber1\", \"RefundTaxAmountDueIndicator1\", \"RefundInterestDueIndicator1\",\n",
        "                        \"RefundTotalDueIndicator1\", \"RefundCostDueIndicator1\", \"EndMarker\", \"TaxpayerMailingCity\"\n",
        "                        ]\n",
        "drop_cols_numeric_corr = [\"OriginalTaxDue2\", \"AdjustedTaxDue2\", \"AssessedValuation\", \"EqualizedEvaluation\",\n",
        "                          \"LastPaymentReceivedAmount2\", \"TaxDueEstimated1\", \"LastPaymentReceivedAmount1\",\n",
        "                          \"AdjustedAmountDue1_Org\", \"InterestAmountDue1\", \"TotalAmountDue1\", \"InterestAmountDue2\",\n",
        "                          \"TotalAmountDue2\", \"SeniorExemptAmount\", \"VeteranExempt\"]\n",
        "#drop_cols_unverfied = [\"Volume\", \"Classification\", \"TaxpayerPropertyTown\", \"LastPaymentDate1\", \"LastPaymentSource1\"] # just dropping temporarily"
      ],
      "metadata": {
        "id": "8ToTkGb3Xcgt"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data1 = data1.drop(*drop_cols_inst2)\n",
        "data1 = data1.drop(*drop_cols_unrelated)\n",
        "data1 = data1.drop(*drop_cols_univariate)\n",
        "data1 = data1.drop(*drop_cols_numeric_corr)\n",
        "#data1 = data1.drop(*drop_cols_unverfied)"
      ],
      "metadata": {
        "id": "QREyti09j7mt"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Transformation"
      ],
      "metadata": {
        "id": "nTBJi-x-u5eZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data1 = data1.withColumn(\"TaxpayerPropertyCity\", when(data1.TaxpayerPropertyCity==\"CHICAGO\", 1).otherwise(0))\n",
        "data1 = data1.withColumn(\"TaxpayerMailingState\", when(data1.TaxpayerMailingState==\"IL\", 1).otherwise(0))\n",
        "data1 = data1.withColumn(\"BillType\", when(data1.BillType==\"1\", 1).otherwise(0))\n",
        "data1 = data1.withColumn(\"TaxSale\", when(data1.LastPaymentDate2==\"00000000\", 1).otherwise(0))\n",
        "data1 = data1.withColumn(\"CofENumber\", when(data1.CofENumber==\"0000000\", 0).otherwise(1))\n",
        "data1 = data1.withColumn(\"DiffPmt1\", f.datediff(col(\"LastPaymentDate1\"), col(\"PmtDue1\")))\n",
        "data1 = data1.withColumn(\"DiffPmt1\", when(col(\"DiffPmt1\")<0,0).otherwise(col(\"DiffPmt1\")))\n",
        "data1 = data1.withColumn(\"Volume\", col(\"Volume\").cast(\"int\"))\n",
        "data1 = data1.withColumn(\"TaxStatus\", col(\"TaxStatus\").cast(\"int\"))\n",
        "data1 = data1.withColumn(\"PastTaxSaleStatus\", when(col(\"PastTaxSaleStatus\")==\"Y\", 1).otherwise(0))"
      ],
      "metadata": {
        "id": "kidacUDxjpVZ"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Persist data in disk/memory\n",
        "# Until this point Spark does not actually execute any of the above statements (Lazy loading). It only creates a plan\n",
        "# Persist does not get executed until an 'action' command is called on the dataframe (action is count() here)\n",
        "data2 = data1.persist()\n",
        "print([data2.count(), len(data2.columns)])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ExtVMZuaOppw",
        "outputId": "165c7eae-3da5-43c7-94ca-63c73de4a532"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[9325500, 44]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### WOE Transformation"
      ],
      "metadata": {
        "id": "o9feOrSBiumP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Reference: https://www.listendata.com/2015/03/weight-of-evidence-woe-and-information.html\n",
        "def WOE_transform(df, var, y=\"LatePmt2\"):\n",
        "  woe_df = df.groupBy(var) \\\n",
        "    .agg(count(var).alias(\"cnt\"), \\\n",
        "        sum(y).alias(\"cnt_1\")) \\\n",
        "    .withColumn(\"cnt_0\", col(\"cnt\") - col(\"cnt_1\")) \\\n",
        "    .withColumn(\"perc_1\", col(\"cnt_1\")/col(\"cnt\")) \\\n",
        "    .withColumn(\"perc_0\", col(\"cnt_0\")/col(\"cnt\")) \\\n",
        "    .withColumn(\"WOE_\"+var, log(col(\"perc_0\")/col(\"perc_1\")))\n",
        "  woe_df = woe_df.withColumn(\"WOE_\"+var, f.round(col(\"WOE_\"+var), 2))\n",
        "  df = df.join(woe_df.select(*[var, \"WOE_\"+var]), [var])#.drop(woe_df[var])\n",
        "  return(df)"
      ],
      "metadata": {
        "id": "FFDxQG9LisFb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "woe_transform_cols = [\"TaxpayerPropertySuffix\", \"Classification\", \"TaxpayerPropertyTown\",\n",
        "                       \"TaxStatus\", \"AccountStatus\", \"LastPaymentSource1\"\n",
        "                      #\"Volume\", \"TaxCode\"\n",
        "                      ]"
      ],
      "metadata": {
        "id": "43dzLpc3uF_i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for var in woe_transform_cols:\n",
        "  data2 = WOE_transform(data2, var)\n",
        "data2 = data2.drop(*woe_transform_cols)"
      ],
      "metadata": {
        "id": "GUSH4RHoycEb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data3 = data2.persist()\n",
        "print([data3.count(), len(data3.columns)])"
      ],
      "metadata": {
        "id": "eNF5Kud304Re",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2c144952-6dd3-47fb-f088-958cc089bde1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[9325500, 44]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Window Transformations"
      ],
      "metadata": {
        "id": "4zFUxG_4rXvZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "windowval1 = (Window.partitionBy('PIN').orderBy('TaxYear')\n",
        "             .rangeBetween(Window.unboundedPreceding, 0))\n",
        "data3 = data3.withColumn('PastTaxSaleStatusSum', F.sum('PastTaxSaleStatus').over(windowval1))\n",
        "windowval2 = (Window.partitionBy('PIN').orderBy('TaxYear')\n",
        "            .rowsBetween(Window.unboundedPreceding, Window.currentRow-1))\n",
        "data3 = data3.withColumn('LatePmt2Sum', F.sum('LatePmt2').over(windowval2))\n",
        "data3 = data3.na.fill(value=0,subset=[\"LatePmt2Sum\"])\n",
        "\n",
        "windowval3 = (Window.partitionBy('PIN').orderBy('TaxYear')\n",
        "            .rowsBetween(Window.unboundedPreceding, 0))\n",
        "data3 = data3.withColumn('LatePmt1Sum', F.sum('LatePmt1').over(windowval3))"
      ],
      "metadata": {
        "id": "wQbgkKnLlirN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data4 = data3.persist()\n",
        "print([data4.count(), len(data4.columns)])"
      ],
      "metadata": {
        "id": "ObHcfjyemDdw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6c3a0a80-91de-44f2-e94b-a80e5e5d5800"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[9325500, 47]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data4.show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zvgfrRKerVJQ",
        "outputId": "c9a5ddeb-de8d-43c8-8d9a-f0c2be03a587"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+\n",
            "|Volume|TaxpayerMailingState|TaxpayerPropertyCity|TaxCode|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|BillType|AdjustedAmountDue1|TaxAmountDue1|CostAmountDue1|CofENumber|PastTaxSaleStatus|TaxRate|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|LatePmt1|LatePmt2|DiffPmt1|WOE_TaxpayerPropertySuffix|WOE_Classification|WOE_TaxpayerPropertyTown|WOE_TaxStatus|WOE_AccountStatus|WOE_LastPaymentSource1|PastTaxSaleStatusSum|LatePmt2Sum|LatePmt1Sum|\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+\n",
            "|     1|                   0|                   0|  10102|              0|           0|                 0|       0|               0.0|          0.0|           0.0|         0|                0|    0.0|                 0|                         0|                0|           1|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|       0|       0|    null|                      2.08|              9.21|                    2.34|         null|             2.15|                  3.35|                   0|          0|          0|\n",
            "|     1|                   0|                   0|  10102|              0|           0|                 0|       0|               0.0|          0.0|           0.0|         0|                0|    0.0|                 0|                         0|                0|           1|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|       0|       0|    null|                      2.08|              9.21|                    2.34|         null|             2.15|                  3.35|                   0|          0|          0|\n",
            "|     1|                   0|                   0|  10100|              0|           0|                 0|       0|               0.0|          0.0|           0.0|         0|                0|    0.0|                 0|                         0|                0|           1|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|       0|       0|    null|                      2.08|              9.21|                    2.34|         null|             2.15|                  3.35|                   0|          0|          0|\n",
            "|     1|                   0|                   0|  10100|              0|           0|                 0|       0|               0.0|          0.0|           0.0|         0|                0|    0.0|                 0|                         0|                0|           1|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|       0|       0|    null|                      2.08|              9.21|                    2.34|         null|             2.15|                  3.35|                   0|          0|          0|\n",
            "|     1|                   0|                   0|  10100|              0|           0|                 0|       0|               0.0|          0.0|           0.0|         0|                0|    0.0|                 0|                         0|                0|           1|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|       0|       0|    null|                      2.08|              9.21|                    2.34|         null|             2.15|                  3.35|                   0|          0|          0|\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Drop other columns"
      ],
      "metadata": {
        "id": "YAmQF1O514m9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "other_cols = [\"LastPaymentDate1\", \"LastPaymentDate2\", \"TaxYear\", \"PmtDue1\", \"PmtDue2\", \"PIN\",\n",
        "              \"TaxSale\", \"Year\"]"
      ],
      "metadata": {
        "id": "xvX9PiPiSrw4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data4 = data4.drop(*woe_transform_cols)\n",
        "data4 = data4.drop(*other_cols)"
      ],
      "metadata": {
        "id": "shckPBQOQB88"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "string_cols = []\n",
        "for c in data4.schema.fields:\n",
        "  if isinstance(c.dataType, StringType):\n",
        "    string_cols.append(c.name)"
      ],
      "metadata": {
        "id": "5o_vmgEzvmfs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for c in string_cols:\n",
        "  data4 = data4.withColumn(c, data4[c].cast('int'))"
      ],
      "metadata": {
        "id": "RWS0ddPOwTYp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Other temp fixes"
      ],
      "metadata": {
        "id": "8-HNPWo6sJz-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#from pyspark.sql.functions import col,isnan, when, count\n",
        "#data4.select([f.count(when(col(c).isNull(), c)).alias(c) for c in data4.columns]).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s1g6PDhIs2zr",
        "outputId": "b45a03bf-aef7-416d-e3ac-f070e20de174"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+\n",
            "|Volume|TaxpayerMailingState|TaxpayerPropertyCity|TaxCode|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|BillType|AdjustedAmountDue1|TaxAmountDue1|CostAmountDue1|CofENumber|PastTaxSaleStatus|TaxRate|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|LatePmt1|LatePmt2|DiffPmt1|WOE_TaxpayerPropertySuffix|WOE_Classification|WOE_TaxpayerPropertyTown|WOE_TaxStatus|WOE_AccountStatus|WOE_LastPaymentSource1|PastTaxSaleStatusSum|LatePmt2Sum|LatePmt1Sum|\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+\n",
            "|     0|                   0|                   0|      0|              0|           0|                 0|       0|                 0|            0|             0|         0|                0|      0|                 0|                         0|                0|           0|             0|           0|                          0|                            0|                         0|                               0|                    0|                       0|                             0|       0|       0|       0|                         0|                 0|                       0|            0|                0|                     0|                   0|          0|          0|\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Temporarily set null as 0. Needs rework\n",
        "null_cols = [\"WOE_Classification\", \"WOE_TaxStatus\", \"WOE_LastPaymentSource1\", \"WOE_TaxpayerPropertySuffix\", \"DiffPmt1\"]\n",
        "for c in null_cols:\n",
        "  data4 = data4.withColumn(c, when(data4[c].isNull(), 0).otherwise(data4[c]))"
      ],
      "metadata": {
        "id": "00T1kr-csz6W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### EDA\n",
        "Proceed to 'Model building' in order to continue running the codes for the model. This part is not required"
      ],
      "metadata": {
        "id": "HY5i7tsA_Wsv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "fig, ax = plt.subplots()\n",
        "hist(ax, data1.select('AdjustedAmountDue1'), bins = 50, color=['red'])\n",
        "plt.title(\"Histogram of AdjustedAmountDue1\")\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 470
        },
        "id": "Prx85RBbTB4t",
        "outputId": "38c8217e-e185-4e87-f6c6-ab48b844ed78"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAHFCAYAAADyj/PrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2IElEQVR4nO3de1xVVeL///cBBLwB4g1JRLyRV0xSwzSlNCS6WJppaoBZ0wzWqN10mkmtJmrGsvlOlvlwEi3NUlPLTGG8MSqN461MHVPzNgqamaCoqLB+f/TjfDwCIspFWK/n47EfetZee++119ls3mfvtTkOY4wRAACABdwqugEAAADlheADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4IMqq2nTpoqLi6voZlR5f/3rX9WsWTO5u7urY8eOZbqtuLg4NW3a1KXM4XBowoQJZbrdsjBhwgQ5HI6KbgZgHYIPKoWkpCQ5HA5t3Lix0Pm9evVSu3btrns7S5curZS/RCtKcnKyXnjhBd1+++2aMWOGXn/99atabuDAgXI4HHrxxRfLuIXX58iRI5owYYK2bt1artvNzc1VYGCgHA6Hvv7663Lddnm4Ur/GxcXJ4XA4p1q1aqlZs2YaMGCAFixYoLy8vHJp46effqqhQ4eqZcuWcjgc6tWrV7lsF2XPo6IbAJSVXbt2yc2tZNl+6dKlmjJlCuHnKq1cuVJubm76xz/+IU9Pz6taJisrS19++aWaNm2qTz75RG+88cZ1Xfk4e/asPDzK5lR25MgRTZw4UU2bNi3zq1mXWrlypdLT09W0aVPNnj1b0dHR5bbt8lBcv3p5eWn69OmSfn1/Dxw4oC+//FIDBgxQr169tHjxYvn4+JRpG99//31t2rRJnTt31s8//1ym20L54ooPqiwvLy9Vq1atoptRItnZ2RXdhBI5duyYqlevftWhR5IWLFig3Nxcffjhhzp06JBSU1Ovqw3e3t5lFnwqyscff6xOnTpp9OjRWrRoUaU7Lq6Xh4eHhg4dqqFDh+qJJ57Qa6+9pm+//VaJiYlavXq1nnjiiTJvw0cffaTMzEytXLlSgYGBZb49lB+CD6qsy8f4XLhwQRMnTlTLli3l7e2tunXrqnv37kpJSZH06yX2KVOmSJLLpfZ82dnZevbZZxUUFCQvLy+FhoZq0qRJMsa4bPfs2bN65plnVK9ePdWuXVv333+/Dh8+XGAsSv4Yjx07dujRRx9VnTp11L17d0nSd999p7i4ODVr1kze3t4KCAjQ8OHDC3zyzF/HDz/8oKFDh8rX11f169fXn/70JxljdOjQIT3wwAPy8fFRQECA3nrrravqu4sXL+rVV19V8+bN5eXlpaZNm+oPf/iDcnJynHUcDodmzJih7OxsZ18lJSUVu+7Zs2erT58+ioyMVOvWrTV79uxC6y1atEjt2rWTt7e32rVrp4ULFxZa7/J+LWwckFT4mJqUlBR1795dfn5+qlWrlkJDQ/WHP/xBkrR69Wp17txZkhQfH1/oPv773/9W37595evrqxo1aqhnz55at25dgW2vXbtWnTt3lre3t5o3b64PPvigyP45e/asFi5cqEGDBmngwIE6e/asFi9eXKBeXFycatWqpYMHD+ree+9VrVq1dNNNNzmP4W3btunOO+9UzZo1FRwcrDlz5hRYx48//qiHH35Y/v7+qlGjhm677TZ99dVXLnXybzPv37/fpXz16tVyOBxavXq1syz/lvOOHTsUGRmpGjVq6KabbtJf/vIXl+WK69eijB07VnfffbfmzZunH374wVle1Divwsb5nTx5UqNGjXL+HLdo0UJvvvlmgVtoQUFBJb5ijMqhan1MQpWXmZmp48ePFyi/cOFCsctOmDBBiYmJGjFihLp06aKsrCxt3LhRmzdvVp8+ffSb3/xGR44cUUpKij766COXZY0xuv/++7Vq1So9/vjj6tixo5YvX67nn39ehw8f1uTJk5114+Li9Nlnn2nYsGG67bbbtGbNGsXExBTZrocfflgtW7bU66+/7gxRKSkp+vHHHxUfH6+AgABt375d06ZN0/bt2/XNN98U+AX+yCOPqHXr1nrjjTf01Vdf6bXXXpO/v78++OAD3XnnnXrzzTc1e/ZsPffcc+rcubPuuOOOK/bViBEjNHPmTA0YMEDPPvus/v3vfysxMVE7d+50BpCPPvpI06ZN04YNG5y3Jbp163bF9R45ckSrVq3SzJkzJUmDBw/W5MmT9e6777pcNUpOTlb//v3Vpk0bJSYm6ueff1Z8fLwaN258xfWXxPbt23XvvfeqQ4cOeuWVV+Tl5aU9e/Y4g0vr1q31yiuv6OWXX9aTTz6pHj16uOzjypUrFR0drfDwcI0fP15ubm6aMWOG7rzzTv3rX/9Sly5dJP0aQO6++27Vr19fEyZM0MWLFzV+/Hg1bNiw0HZ98cUXOn36tAYNGqSAgAD16tVLs2fP1qOPPlqgbm5urqKjo3XHHXfoL3/5i2bPnq2RI0eqZs2aeumllzRkyBA99NBDmjp1qh577DFFREQoJCREknT06FF169ZNZ86c0TPPPKO6detq5syZuv/++zV//nw9+OCD19Svv/zyi/r27auHHnpIAwcO1Pz58/Xiiy+qffv2io6OLrZfizNs2DAlJycrJSVFrVq1KlHbzpw5o549e+rw4cP6zW9+oyZNmmj9+vUaN26c0tPT9c4775R0d1EZGaASmDFjhpF0xalt27YuywQHB5vY2Fjn67CwMBMTE3PF7SQkJJjCfiwWLVpkJJnXXnvNpXzAgAHG4XCYPXv2GGOM2bRpk5FkRo0a5VIvLi7OSDLjx493lo0fP95IMoMHDy6wvTNnzhQo++STT4wkk5qaWmAdTz75pLPs4sWLpnHjxsbhcJg33njDWf7LL7+Y6tWru/RJYbZu3WokmREjRriUP/fcc0aSWblypbMsNjbW1KxZ84rru9SkSZNM9erVTVZWljHGmB9++MFIMgsXLnSp17FjR9OoUSNz8uRJZ1lycrKRZIKDg13qXt6vsbGxBeoY8399lW/y5MlGkvnpp5+KbO9//vMfI8nMmDHDpTwvL8+0bNnSREVFmby8PGf5mTNnTEhIiOnTp4+zrF+/fsbb29scOHDAWbZjxw7j7u5e6LF27733mttvv935etq0acbDw8McO3bMpV5sbKyRZF5//XVnWf577HA4zNy5c53l//3vfwv006hRo4wk869//ctZdurUKRMSEmKaNm1qcnNzjTH/97O3b98+l+2vWrXKSDKrVq1ylvXs2dNIMrNmzXKW5eTkmICAANO/f39nWVH9mr9fVzqmtmzZYiSZ0aNHO8su37d8l58DXn31VVOzZk3zww8/uNQbO3ascXd3NwcPHix0m23btjU9e/Yssk2oXLiOh0plypQpSklJKTB16NCh2GX9/Py0fft27d69u8TbXbp0qdzd3fXMM8+4lD/77LMyxjifvFm2bJkk6Xe/+51LvaeffrrIdT/11FMFyqpXr+78/7lz53T8+HHddtttkqTNmzcXqD9ixAjn/93d3XXrrbfKGKPHH3/cWe7n56fQ0FD9+OOPRbZF+nVfJWnMmDEu5c8++6wkFbgVUhKzZ89WTEyMateuLUlq2bKlwsPDXW53paena+vWrYqNjZWvr6+zvE+fPmrTps01b/tyfn5+kqTFixeX+EmhrVu3avfu3Xr00Uf1888/6/jx4zp+/Liys7N11113KTU1VXl5ecrNzdXy5cvVr18/NWnSxLl869atFRUVVWC9P//8s5YvX67Bgwc7y/r37y+Hw6HPPvus0LZc+t7nv8c1a9bUwIEDneWhoaHy8/Nzee+XLl2qLl26OG+vSlKtWrX05JNPav/+/dqxY0eJ+uTSdQwdOtT52tPTU126dCn2uCvJ+iXp1KlTJV523rx56tGjh+rUqeN8z44fP67evXsrNzf3useboXLgVhcqlS5duujWW28tUJ5/IruSV155RQ888IBatWqldu3aqW/fvho2bNhVhaYDBw4oMDDQ+Qs7X+vWrZ3z8/91c3Nz3k7I16JFiyLXfXldSTpx4oQmTpyouXPn6tixYy7zMjMzC9S/9JeqJPn6+srb21v16tUrUF7cEyr5+3B5mwMCAuTn5+fc15LauXOntmzZoscee0x79uxxlvfq1UtTpkxRVlaWfHx8nOtv2bJlgXWEhoYWGvyuxSOPPKLp06drxIgRGjt2rO666y499NBDGjBgQLFjO/LDc2xsbJF1MjMzlZOTo7Nnzxa5L/khM9+nn36qCxcu6JZbbnHpo65du2r27NlKSEhwqe/t7a369eu7lPn6+qpx48YFbof6+vrql19+cb4+cOCAunbtWqBdlx7T1/InIgrbdp06dfTdd9+VeF2FOX36tCQV+Fm8Grt379Z3331XoM/yXf6zhqqJ4ANr3HHHHdq7d68WL16s5ORkTZ8+XZMnT9bUqVNdPjWXt0uv7uQbOHCg1q9fr+eff14dO3ZUrVq1lJeXp759+xZ6dcLd3f2qyiQVGIxdlNL+43off/yxJGn06NEaPXp0gfkLFixQfHz8dW+nqHbn5ua6vK5evbpSU1O1atUqffXVV1q2bJk+/fRT3XnnnUpOTi6y/yQ534O//vWvRT7mXqtWLZfB4Fcj/8rX7bffXuj8H3/8Uc2aNXO+LqqN1/veX+pq+7Mstl2Y77//XtKVP0zku7yNeXl56tOnj1544YVC65d0zBAqJ4IPrOLv76/4+HjFx8fr9OnTuuOOOzRhwgRn8CnqJB8cHKx//vOfOnXqlMsnzf/+97/O+fn/5uXlad++fS6f8i/99F6cX375RStWrNDEiRP18ssvO8uv5Rbdtcjfh927dzs//Uu/DoY9efKkc19LwhijOXPmKDIyssBtQEl69dVXNXv2bMXHxzvXX9j+7tq1q9ht1alTRydPnixQXtiVKjc3N911112666679Pbbb+v111/XSy+9pFWrVql3795FHg/NmzeXJPn4+Kh3795FtqV+/fqqXr36Ve3Lvn37tH79eo0cOVI9e/Z0mZeXl6dhw4Zpzpw5+uMf/1jk9koiODi40P68/JiuU6eOJBXo02u98iddX6j+6KOP5HA41KdPH2dZYe/5+fPnlZ6e7lLWvHlznT59+orvGao+xvjAGpff4qlVq5ZatGjh8qm8Zs2akgqe5O+55x7l5ubq3XffdSmfPHmyHA6H8w/M5Y/beO+991zq/f3vf7/qduZ/Yr78E3J5PXFyzz33FLq9t99+W5Ku+IRaUdatW6f9+/crPj5eAwYMKDA98sgjWrVqlY4cOaJGjRqpY8eOmjlzpsttvZSUlKsad9K8eXNlZma63FpJT08v8Dj8iRMnCiybf/Um/5go6ngIDw9X8+bNNWnSJOetl0v99NNPkn59L6OiorRo0SIdPHjQOX/nzp1avny5yzL5V3teeOGFAv0zcOBA9ezZs8hH/6/FPffcow0bNigtLc1Zlp2drWnTpqlp06bO8VT5Ie/S8S+5ubmaNm3aNW+7qH4tzhtvvKHk5GQ98sgjLh8smjdvXmB8zrRp0wpc8Rk4cKDS0tIK9H1+Wy5evFii9qBy4ooPrNGmTRv16tVL4eHh8vf318aNGzV//nyNHDnSWSc8PFyS9MwzzygqKkru7u4aNGiQ7rvvPkVGRuqll17S/v37FRYWpuTkZC1evFijRo1y/nIIDw9X//799c477+jnn392Ps6e/zdHruaTro+Pj/Px5AsXLuimm25ScnKy9u3bVwa9UlBYWJhiY2M1bdo0nTx5Uj179tSGDRs0c+ZM9evXT5GRkSVe5+zZs+Xu7l5kaLr//vv10ksvae7cuRozZowSExMVExOj7t27a/jw4Tpx4oT+/ve/q23btoUGjUsNGjRIL774oh588EE988wzOnPmjN5//321atXKZXzQK6+8otTUVMXExCg4OFjHjh3Te++9p8aNGzsH/DZv3lx+fn6aOnWqateurZo1a6pr164KCQnR9OnTFR0drbZt2yo+Pl433XSTDh8+rFWrVsnHx0dffvmlJGnixIlatmyZevTood/97ne6ePGic18uDWezZ89Wx44dFRQUVGQfPf3009q8ebM6depUov4vzNixY/XJJ58oOjpazzzzjPz9/TVz5kzt27dPCxYscI5zatu2rW677TaNGzdOJ06ckL+/v+bOnXtdIeFK/Sr9+nek8m+Nnjt3TgcOHNAXX3yh7777TpGRkQVC14gRI/TUU0+pf//+6tOnj7799lstX768wBi3559/Xl988YXuvfdexcXFKTw8XNnZ2dq2bZvmz5+v/fv3O5dJTU11hqmffvpJ2dnZeu211yT9etu8uD8JgRtYBT5RBly1/Edq//Of/xQ6v2fPnsU+zv7aa6+ZLl26GD8/P1O9enVz8803mz//+c/m/PnzzjoXL140Tz/9tKlfv75xOBwujxufOnXKjB492gQGBppq1aqZli1bmr/+9a8ujzMbY0x2drZJSEgw/v7+platWqZfv35m165dRpLL4+X5j1cX9jj1//73P/Pggw8aPz8/4+vrax5++GFz5MiRIh+Jv3wdRT0SXFg/FebChQtm4sSJJiQkxFSrVs0EBQWZcePGmXPnzl3Vdi51/vx5U7duXdOjR48r1gsJCTG33HKL8/WCBQtM69atjZeXl2nTpo35/PPPC31U/fI+MebXR9/btWtnPD09TWhoqPn4448LPM6+YsUK88ADD5jAwEDj6elpAgMDzeDBgws86rx48WLTpk0b4+HhUeAR7C1btpiHHnrI1K1b13h5eZng4GAzcOBAs2LFCpd1rFmzxoSHhxtPT0/TrFkzM3XqVJf25P8ZhD/96U9F9s/+/ftdHuMu6XscHBxc4M857N271wwYMMD4+fkZb29v06VLF7NkyZICy+7du9f07t3beHl5mYYNG5o//OEPJiUlpdDH2QvbdmHvW1H9mv+Yfv5Uo0YN07RpU9O/f38zf/5852P2l8rNzTUvvviiqVevnqlRo4aJiooye/bsKXAOMObXn+Nx48aZFi1aGE9PT1OvXj3TrVs3M2nSJJdzQf77U9hU2KPzqDwcxpTSiDMARdq6datuueUWffzxxxoyZEhFN6fKyM3NlYeHh1599dVSG/sCoGpjjA9Qys6ePVug7J133pGbmxuXx0tZ/uDVy29pAEBRGOMDlLK//OUv2rRpkyIjI+Xh4aGvv/5aX3/9tZ588skix2+g5ObPn69Zs2bJ4XBc07gjAHbiVhdQylJSUjRx4kTt2LFDp0+fVpMmTTRs2DC99NJLVe5bxCtSs2bN5HA49Mc//rFU/v4PADsQfAAAgDUY4wMAAKxB8AEAANZgwMEl8vLydOTIEdWuXbvUv6cIAACUDWOMTp06pcDAwGK/ZJjgc4kjR47w1A0AAJXUoUOH1Lhx4yvWIfhcIv/LJw8dOiQfH58Kbg0AALgaWVlZCgoKcvkS6aIQfC6Rf3vLx8eH4AMAQCVzNcNUGNwMAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2Pim6AVRyO4usYU/btAADAUlzxAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArFFuwScxMVGdO3dW7dq11aBBA/Xr10+7du0qdrl58+bp5ptvlre3t9q3b6+lS5e6zO/Vq5ccDofeeOONAsvGxMTI4XBowoQJpbUbAACgEiu34LNmzRolJCTom2++UUpKii5cuKC7775b2dnZRS6zfv16DR48WI8//ri2bNmifv36qV+/fvr+++9d6gUFBSkpKcml7PDhw1qxYoUaNWpUFrsDAAAqoXILPsuWLVNcXJzatm2rsLAwJSUl6eDBg9q0aVORy/ztb39T37599fzzz6t169Z69dVX1alTJ7377rsu9e69914dP35c69atc5bNnDlTd999txo0aFBm+wQAACqXChvjk5mZKUny9/cvsk5aWpp69+7tUhYVFaW0tDSXMk9PTw0ZMkQzZsxwliUlJWn48OFXbENOTo6ysrJcJgAAUHVVSPDJy8vTqFGjdPvtt6tdu3ZF1svIyFDDhg1dyho2bKiMjIwCdYcPH67PPvtM2dnZSk1NVWZmpu69994rtiMxMVG+vr7OKSgo6Np2CAAAVAoVEnwSEhL0/fffa+7cuaW2zrCwMLVs2VLz58/Xhx9+qGHDhsnDw+OKy4wbN06ZmZnO6dChQ6XWHgAAcOO5cjIoAyNHjtSSJUuUmpqqxo0bX7FuQECAjh496lJ29OhRBQQEFFp/+PDhmjJlinbs2KENGzYU2xYvLy95eXldfeMBAEClVm5XfIwxGjlypBYuXKiVK1cqJCSk2GUiIiK0YsUKl7KUlBRFREQUWv/RRx/Vtm3b1K5dO7Vp06ZU2g0AAKqOcrvik5CQoDlz5mjx4sWqXbu2c5yOr6+vqlevLkl67LHHdNNNNykxMVGS9Pvf/149e/bUW2+9pZiYGM2dO1cbN27UtGnTCt1GnTp1lJ6ermrVqpXPTgEAgEql3K74vP/++8rMzFSvXr3UqFEj5/Tpp5866xw8eFDp6enO1926ddOcOXM0bdo0hYWFaf78+Vq0aNEVB0T7+fmpZs2aZbovAACgcnIYY0xFN+JGkZWVJV9fX2VmZsrHx6f0N+BwFF+HtwMAgBIpye9vvqsLAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsEa5BZ/U1FTdd999CgwMlMPh0KJFi65Yf/Xq1XI4HAWmjIwMZ524uDg5HA499dRTBZZPSEiQw+FQXFxcKe8JAACorMot+GRnZyssLExTpkwp0XK7du1Senq6c2rQoIHL/KCgIM2dO1dnz551lp07d05z5sxRkyZNSqXtAACgavAorw1FR0crOjq6xMs1aNBAfn5+Rc7v1KmT9u7dq88//1xDhgyRJH3++edq0qSJQkJCrrW5AACgCrrhx/h07NhRjRo1Up8+fbRu3bpC6wwfPlwzZsxwvv7www8VHx9f7LpzcnKUlZXlMgEAgKrrhg0+jRo10tSpU7VgwQItWLBAQUFB6tWrlzZv3lyg7tChQ7V27VodOHBABw4c0Lp16zR06NBit5GYmChfX1/nFBQUVBa7AgAAbhDldqurpEJDQxUaGup83a1bN+3du1eTJ0/WRx995FK3fv36iomJUVJSkowxiomJUb169Yrdxrhx4zRmzBjn66ysLMIPAABV2A0bfArTpUsXrV27ttB5w4cP18iRIyXpqgdQe3l5ycvLq9TaBwAAbmyVKvhs3bpVjRo1KnRe3759df78eTkcDkVFRZVzywAAQGVQbsHn9OnT2rNnj/P1vn37tHXrVvn7+6tJkyYaN26cDh8+rFmzZkmS3nnnHYWEhKht27Y6d+6cpk+frpUrVyo5ObnQ9bu7u2vnzp3O/wMAAFyu3ILPxo0bFRkZ6XydP7YmNjZWSUlJSk9P18GDB53zz58/r2effVaHDx9WjRo11KFDB/3zn/90WcflfHx8ym4HAABApecwxpiKbsSNIisrS76+vsrMzCybEOVwFF+HtwMAgBIpye/vG/ZxdgAAgNJG8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYw6OiG4Br4HAUX8eYsm8HAACVDFd8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYg6e6bjRX88QWAAC4JlzxAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDZ7qqqr4Pi8AAArgig8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYw6OiG4AK5HAUX8eYsm8HAADlhCs+AADAGgQfAABgDYIPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDXKLfikpqbqvvvuU2BgoBwOhxYtWlTsMqtXr1anTp3k5eWlFi1aKCkpyWV+XFycHA6HnnrqqQLLJiQkyOFwKC4urnR2AAAAVHrlFnyys7MVFhamKVOmXFX9ffv2KSYmRpGRkdq6datGjRqlESNGaPny5S71goKCNHfuXJ09e9ZZdu7cOc2ZM0dNmjQp1X0AAACVm0d5bSg6OlrR0dFXXX/q1KkKCQnRW2+9JUlq3bq11q5dq8mTJysqKspZr1OnTtq7d68+//xzDRkyRJL0+eefq0mTJgoJCSndnQAAAJXaDTvGJy0tTb1793Ypi4qKUlpaWoG6w4cP14wZM5yvP/zwQ8XHxxe7jZycHGVlZblMAACg6rphg09GRoYaNmzoUtawYUNlZWW53NaSpKFDh2rt2rU6cOCADhw4oHXr1mno0KHFbiMxMVG+vr7OKSgoqFT3AQAA3FjK7VZXWapfv75iYmKUlJQkY4xiYmJUr169YpcbN26cxowZ43ydlZVF+AEAoAq7YYNPQECAjh496lJ29OhR+fj4qHr16gXqDx8+XCNHjpSkqx5A7eXlJS8vr+tvLAAAqBRu2FtdERERWrFihUtZSkqKIiIiCq3ft29fnT9/XhcuXHAZ/AwAAJCv3K74nD59Wnv27HG+3rdvn7Zu3Sp/f381adJE48aN0+HDhzVr1ixJ0lNPPaV3331XL7zwgoYPH66VK1fqs88+01dffVXo+t3d3bVz507n/wEAAC5XbsFn48aNioyMdL7OH1sTGxurpKQkpaen6+DBg875ISEh+uqrrzR69Gj97W9/U+PGjTV9+vQrXs3x8fEpux0AAACVnsMYYyq6ETeKrKws+fr6KjMzs2xClMNR+ussaxweAIAbXEl+f9+wY3wAAABKG8EHAABYg+ADAACsQfABAADWuGH/gCFuEFczIJsB0ACASoIrPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABreFR0A1AFOBzF1zGm7NsBAEAxuOIDAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAa5Qo+EyYMEEOh8Nluvnmm53zz507p4SEBNWtW1e1atVS//79dfToUZd1HDx4UDExMapRo4YaNGig559/XhcvXrzidk+cOKEhQ4bIx8dHfn5+evzxx3X69Gnn/NWrV8vhcOjkyZPOsiNHjqh9+/a64447lJmZWZLdBAAAVVSJr/i0bdtW6enpzmnt2rXOeaNHj9aXX36pefPmac2aNTpy5Igeeugh5/zc3FzFxMTo/PnzWr9+vWbOnKmkpCS9/PLLV9zmkCFDtH37dqWkpGjJkiVKTU3Vk08+WWT9vXv3qnv37goODtby5cvl6+tb0t0EAABVkSmB8ePHm7CwsELnnTx50lSrVs3MmzfPWbZz504jyaSlpRljjFm6dKlxc3MzGRkZzjrvv/++8fHxMTk5OYWud8eOHUaS+c9//uMs+/rrr43D4TCHDx82xhizatUqI8n88ssv5ttvvzUBAQHm0UcfNRcuXLji/pw7d85kZmY6p0OHDhlJJjMz86r6o8QkeycAAMpIZmbmVf/+LvEVn927dyswMFDNmjXTkCFDdPDgQUnSpk2bdOHCBfXu3dtZ9+abb1aTJk2UlpYmSUpLS1P79u3VsGFDZ52oqChlZWVp+/bthW4vLS1Nfn5+uvXWW51lvXv3lpubm/7973+71F2/fr169uyp/v376+OPP5aHh8cV9yUxMVG+vr7OKSgoqGSdAQAAKpUSBZ+uXbsqKSlJy5Yt0/vvv699+/apR48eOnXqlDIyMuTp6Sk/Pz+XZRo2bKiMjAxJUkZGhkvoyZ+fP68wGRkZatCggUuZh4eH/P39Cyzz4IMP6r777tO7774rh8NR7P6MGzdOmZmZzunQoUPFLgMAACqvK18SuUx0dLTz/x06dFDXrl0VHByszz77TNWrVy/1xpXUAw88oIULF+pf//qXevToUWx9Ly8veXl5lUPLAADAjeC6Hmf38/NTq1attGfPHgUEBOj8+fMuT1ZJ0tGjRxUQECBJCggIKPCUV/7r/DqXCwgI0LFjx1zKLl68qBMnThRY5oMPPtCgQYMUHR2t1NTU69k1AABQBV1X8Dl9+rT27t2rRo0aKTw8XNWqVdOKFSuc83ft2qWDBw8qIiJCkhQREaFt27a5BJmUlBT5+PioTZs2hW4jIiJCJ0+e1KZNm5xlK1euVF5enrp27epS1+FwaNq0aRoyZIjuuecerVmz5np2DwAAVDElCj7PPfec1qxZo/3792v9+vV68MEH5e7ursGDB8vX11ePP/64xowZo1WrVmnTpk2Kj49XRESEbrvtNknS3XffrTZt2mjYsGH69ttvtXz5cv3xj39UQkKC85bThg0bdPPNN+vw4cOSpNatW6tv37564okntGHDBq1bt04jR47UoEGDFBgYWKCNDodDU6dO1WOPPaZ77rlHq1evvs4uAgAAVUWJxvj873//0+DBg/Xzzz+rfv366t69u7755hvVr19fkjR58mS5ubmpf//+ysnJUVRUlN577z3n8u7u7lqyZIl++9vfKiIiQjVr1lRsbKxeeeUVZ50zZ85o165dunDhgrNs9uzZGjlypO666y7n+v/f//t/RbbT4XBoypQpcnNzU0xMjJYsWaLIyMiS7CoAAKiCHMYYU9GNuFFkZWXJ19dXmZmZ8vHxKf0NXMWTZlbjUAQAXIOS/P7mu7oAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKxB8AEAANYg+AAAAGsQfAAAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDY+KbgDg5HAUX8eYsm8HAKDK4ooPAACwBsEHAABYg+ADAACsQfABAADWIPgAAABrEHwAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKzBd3WhcuH7vAAA14ErPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1uBxdlQ9PPIOACgCV3wAAIA1CD4AAMAaBB8AAGANgg8AALAGwQcAAFiD4AMAAKzB4+ywE4+8A4CVuOIDAACsQfABAADWIPgAAABrEHwAAIA1GNwMFIUB0ABQMpXgvMkVHwAAYA2CDwAAsAa3uoDrUQku6wIA/g9XfAAAgDUIPgAAwBrc6gLKGrfDAOCGwRUfAABgDa74ADcCrgoBQLngig8AALAGV3yAyoKrQgBw3Qg+QFVCOAKAKyL4ALYhHAGwWJUc4zNlyhQ1bdpU3t7e6tq1qzZs2FDRTQIqF4ejdCYAuMFUueDz6aefasyYMRo/frw2b96ssLAwRUVF6dixYxXdNMA+BCgAN5gqF3zefvttPfHEE4qPj1ebNm00depU1ahRQx9++GFFNw3AtSqtAEUQA6xXpcb4nD9/Xps2bdK4ceOcZW5uburdu7fS0tIK1M/JyVFOTo7zdWZmpiQpKyur7BsLoPIi/FQ9///5H+WgDH7H5v/eNlcxPrFKBZ/jx48rNzdXDRs2dClv2LCh/vvf/xaon5iYqIkTJxYoDwoKKrM2AgBuQL6+Fd0Ce5RhX586dUq+xay/SgWfkho3bpzGjBnjfJ2Xl6cTJ06obt26cpTyJ7qsrCwFBQXp0KFD8vHxKdV1V2b0S9Hom6LRN0WjbwpHvxStKvSNMUanTp1SYGBgsXWrVPCpV6+e3N3ddfToUZfyo0ePKiAgoEB9Ly8veXl5uZT5+fmVZRPl4+NTaQ+sskS/FI2+KRp9UzT6pnD0S9Eqe98Ud6UnX5Ua3Ozp6anw8HCtWLHCWZaXl6cVK1YoIiKiAlsGAABuBFXqio8kjRkzRrGxsbr11lvVpUsXvfPOO8rOzlZ8fHxFNw0AAFSwKhd8HnnkEf300096+eWXlZGRoY4dO2rZsmUFBjyXNy8vL40fP77ArTXb0S9Fo2+KRt8Ujb4pHP1SNNv6xmGu5tkvAACAKqBKjfEBAAC4EoIPAACwBsEHAABYg+ADAACsQfABAADWIPiUoilTpqhp06by9vZW165dtWHDhivWnzdvnm6++WZ5e3urffv2Wrp0aTm1tHyVpF+SkpLkcDhcJm9v73JsbflJTU3Vfffdp8DAQDkcDi1atKjYZVavXq1OnTrJy8tLLVq0UFJSUpm3s7yVtF9Wr15d4JhxOBzKyMgonwaXo8TERHXu3Fm1a9dWgwYN1K9fP+3atavY5ar6ueZa+sWWc83777+vDh06OP8qc0REhL7++usrLlPVjxeCTyn59NNPNWbMGI0fP16bN29WWFiYoqKidOzYsULrr1+/XoMHD9bjjz+uLVu2qF+/furXr5++//77cm552Sppv0i//tn09PR053TgwIFybHH5yc7OVlhYmKZMmXJV9fft26eYmBhFRkZq69atGjVqlEaMGKHly5eXcUvLV0n7Jd+uXbtcjpsGDRqUUQsrzpo1a5SQkKBvvvlGKSkpunDhgu6++25lZ2cXuYwN55pr6RfJjnNN48aN9cYbb2jTpk3auHGj7rzzTj3wwAPavn17ofVtOF5kUCq6dOliEhISnK9zc3NNYGCgSUxMLLT+wIEDTUxMjEtZ165dzW9+85sybWd5K2m/zJgxw/j6+pZT624ckszChQuvWOeFF14wbdu2dSl75JFHTFRUVBm2rGJdTb+sWrXKSDK//PJLubTpRnLs2DEjyaxZs6bIOracay51Nf1i67nGGGPq1Kljpk+fXug8G44XrviUgvPnz2vTpk3q3bu3s8zNzU29e/dWWlpaocukpaW51JekqKioIutXRtfSL5J0+vRpBQcHKygo6IqfTGxjwzFzPTp27KhGjRqpT58+WrduXUU3p1xkZmZKkvz9/YusY+NxczX9Itl3rsnNzdXcuXOVnZ1d5PdX2nC8EHxKwfHjx5Wbm1vgazEaNmxY5DiDjIyMEtWvjK6lX0JDQ/Xhhx9q8eLF+vjjj5WXl6du3brpf//7X3k0+YZW1DGTlZWls2fPVlCrKl6jRo00depULViwQAsWLFBQUJB69eqlzZs3V3TTylReXp5GjRql22+/Xe3atSuyng3nmktdbb/YdK7Ztm2batWqJS8vLz311FNauHCh2rRpU2hdG46XKvddXajcIiIiXD6JdOvWTa1bt9YHH3ygV199tQJbhhtVaGioQkNDna+7deumvXv3avLkyfroo48qsGVlKyEhQd9//73Wrl1b0U25oVxtv9h0rgkNDdXWrVuVmZmp+fPnKzY2VmvWrCky/FR1XPEpBfXq1ZO7u7uOHj3qUn706FEFBAQUukxAQECJ6ldG19Ivl6tWrZpuueUW7dmzpyyaWKkUdcz4+PioevXqFdSqG1OXLl2q9DEzcuRILVmyRKtWrVLjxo2vWNeGc02+kvTL5aryucbT01MtWrRQeHi4EhMTFRYWpr/97W+F1rXheCH4lAJPT0+Fh4drxYoVzrK8vDytWLGiyPuoERERLvUlKSUlpcj6ldG19MvlcnNztW3bNjVq1Kismllp2HDMlJatW7dWyWPGGKORI0dq4cKFWrlypUJCQopdxobj5lr65XI2nWvy8vKUk5NT6Dwbjhee6iolc+fONV5eXiYpKcns2LHDPPnkk8bPz89kZGQYY4wZNmyYGTt2rLP+unXrjIeHh5k0aZLZuXOnGT9+vKlWrZrZtm1bRe1CmShpv0ycONEsX77c7N2712zatMkMGjTIeHt7m+3bt1fULpSZU6dOmS1btpgtW7YYSebtt982W7ZsMQcOHDDGGDN27FgzbNgwZ/0ff/zR1KhRwzz//PNm586dZsqUKcbd3d0sW7asonahTJS0XyZPnmwWLVpkdu/ebbZt22Z+//vfGzc3N/PPf/6zonahzPz2t781vr6+ZvXq1SY9Pd05nTlzxlnHxnPNtfSLLeeasWPHmjVr1ph9+/aZ7777zowdO9Y4HA6TnJxsjLHzeCH4lKK///3vpkmTJsbT09N06dLFfPPNN855PXv2NLGxsS71P/vsM9OqVSvj6elp2rZta7766qtybnH5KEm/jBo1ylm3YcOG5p577jGbN2+ugFaXvfzHsC+f8vsjNjbW9OzZs8AyHTt2NJ6enqZZs2ZmxowZ5d7uslbSfnnzzTdN8+bNjbe3t/H39ze9evUyK1eurJjGl7HC+kWSy3Fg47nmWvrFlnPN8OHDTXBwsPH09DT169c3d911lzP0GGPn8eIwxpjyu74EAABQcRjjAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgDYIPAACwBsEHAACUqdTUVN13330KDAyUw+HQokWLSrwOY4wmTZqkVq1aycvLSzfddJP+/Oc/l3g9fDs7AAAoU9nZ2QoLC9Pw4cP10EMPXdM6fv/73ys5OVmTJk1S+/btdeLECZ04caLE6+EvNwMAgHLjcDi0cOFC9evXz1mWk5Ojl156SZ988olOnjypdu3a6c0331SvXr0kSTt37lSHDh30/fffKzQ09Lq2z60uAABQoUaOHKm0tDTNnTtX3333nR5++GH17dtXu3fvliR9+eWXatasmZYsWaKQkBA1bdpUI0aMuKYrPgQfAABQYQ4ePKgZM2Zo3rx56tGjh5o3b67nnntO3bt314wZMyRJP/74ow4cOKB58+Zp1qxZSkpK0qZNmzRgwIASb48xPgAAoMJs27ZNubm5atWqlUt5Tk6O6tatK0nKy8tTTk6OZs2a5az3j3/8Q+Hh4dq1a1eJbn8RfAAAQIU5ffq03N3dtWnTJrm7u7vMq1WrliSpUaNG8vDwcAlHrVu3lvTrFSOCDwAAqBRuueUW5ebm6tixY+rRo0ehdW6//XZdvHhRe/fuVfPmzSVJP/zwgyQpODi4RNvjqS4AAFCmTp8+rT179kj6Nei8/fbbioyMlL+/v5o0aaKhQ4dq3bp1euutt3TLLbfop59+0ooVK9ShQwfFxMQoLy9PnTt3Vq1atfTOO+8oLy9PCQkJ8vHxUXJyconaQvABAABlavXq1YqMjCxQHhsbq6SkJF24cEGvvfaaZs2apcOHD6tevXq67bbbNHHiRLVv316SdOTIET399NNKTk5WzZo1FR0drbfeekv+/v4lagvBBwAAWIPH2QEAgDUIPgAAwBoEHwAAYA2CDwAAsAbBBwAAWIPgAwAArEHwAQAA1iD4AAAAaxB8AACANQg+AADAGgQfAABgjf8P9PUsO7c9xPkAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model Building"
      ],
      "metadata": {
        "id": "7R1I5YAO_ttn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "feature_cols = data4.columns\n",
        "feature_cols.remove(\"LatePmt2\")"
      ],
      "metadata": {
        "id": "uuA9NbDywooA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.ml.feature import VectorAssembler\n",
        "\n",
        "# Convert all features into a vector with name 'features'\n",
        "assembler = VectorAssembler(inputCols=feature_cols, outputCol=\"features\")\n",
        "data4 = assembler.transform(data4)"
      ],
      "metadata": {
        "id": "VZSG5lcjvIu2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Test train split\n",
        "train, test = data4.randomSplit([0.8,0.2], seed=100)"
      ],
      "metadata": {
        "id": "1qlfXV3--dZ8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train = train.persist()\n",
        "train.count()"
      ],
      "metadata": {
        "id": "yZrqOGeBCBil",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "126f94b0-7446-4822-bfe4-8ae9bddc0e21"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7460923"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#from pyspark.ml.classification import LogisticRegression\n",
        "#logit = LogisticRegression(labelCol=\"LatePmt2\")\n",
        "#logit = logit.fit(train)"
      ],
      "metadata": {
        "id": "S0SL41Ye9pDW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rf = RandomForestClassifier(featuresCol = 'features', labelCol = 'LatePmt2')\n",
        "rfModel = rf.fit(train)\n",
        "predictions = rfModel.transform(test)"
      ],
      "metadata": {
        "id": "z_u8hp6buvzG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "actual = test.select(test.LatePmt2).toPandas()['LatePmt2']"
      ],
      "metadata": {
        "id": "ipXeoNTG4wxc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predictions.where(col(\"prediction\")==1).show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AAb0NaswOHDM",
        "outputId": "3908d1e9-aed2-426e-c949-ab78c1bb29c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+--------------------+--------------------+--------------------+----------+\n",
            "|Volume|TaxpayerMailingState|TaxpayerPropertyCity|TaxCode|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|BillType|AdjustedAmountDue1|TaxAmountDue1|CostAmountDue1|CofENumber|PastTaxSaleStatus|TaxRate|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|LatePmt1|LatePmt2|DiffPmt1|WOE_TaxpayerPropertySuffix|WOE_Classification|WOE_TaxpayerPropertyTown|WOE_TaxStatus|WOE_AccountStatus|WOE_LastPaymentSource1|PastTaxSaleStatusSum|LatePmt2Sum|LatePmt1Sum|            features|       rawPrediction|         probability|prediction|\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+--------------------+--------------------+--------------------+----------+\n",
            "|    11|                   1|                   0|  12075|              1|           0|                 0|       1|          223179.0|          0.0|           0.0|         0|                0|21906.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|                     0.0|                           0.0|       1|       1|      19|                      2.44|               2.2|                    1.98|         2.03|             2.15|                 -2.04|                   1|          1|          2|(38,[0,1,3,4,7,8,...|[8.47701438447072...|[0.42385071922353...|       1.0|\n",
            "|    14|                   1|                   0|  12028|              1|           0|                 0|       1|          175002.0|          0.0|           0.0|         0|                0|20213.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|                     0.0|                           0.0|       1|       0|     279|                       2.1|              2.34|                    1.98|         2.03|             2.15|                  1.88|                   0|          3|          3|(38,[0,1,3,4,7,8,...|[6.73141719443978...|[0.33657085972198...|       1.0|\n",
            "|    15|                   1|                   0|  12080|              0|           0|                 0|       1|           24307.0|          0.0|           0.0|         0|                0|23631.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|       1|       0|     184|                      2.14|              1.75|                    1.98|         2.03|             2.81|                  1.37|                   0|          1|          4|(38,[0,1,3,7,8,13...|[7.71729133865536...|[0.38586456693276...|       1.0|\n",
            "|    18|                   1|                   0|  12028|              0|           0|                 0|       1|          127046.0|          0.0|           0.0|         0|                0|19750.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|                  0.0|                     0.0|                           0.0|       1|       1|     162|                      2.08|               2.2|                    1.98|         2.03|             2.15|                  1.26|                   0|          2|          3|(38,[0,1,3,7,8,13...|[6.24469767891394...|[0.31223488394569...|       1.0|\n",
            "|    22|                   1|                   0|  12030|              1|           0|                 0|       1|           44960.0|          0.0|           0.0|         0|                0|17730.0|                 0|                         0|                0|           0|             0|           0|                        0.0|                          0.0|                       0.0|                             0.0|              10000.0|                     0.0|                           0.0|       1|       1|     426|                      2.31|              2.22|                    1.98|         2.03|             2.15|                  0.64|                   0|          2|          3|(38,[0,1,3,4,7,8,...|[5.93423133005414...|[0.29671156650270...|       1.0|\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+--------------------+--------------------+--------------------+----------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "predictions2.where(col(\"prediction\")==0).show(5, truncate=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PZG5wDlgQklp",
        "outputId": "d3210fb0-82d1-4b93-d7ca-69af5b864fe3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+-------------------------------------------------------------------------------------------------------------------------+---------------------------------------+----------------------------------------+----------+---------------+\n",
            "|Volume|TaxpayerMailingState|TaxpayerPropertyCity|TaxCode|HomeownerExempt|SeniorExempt|SeniorFreezeExempt|BillType|AdjustedAmountDue1|TaxAmountDue1|CostAmountDue1|CofENumber|PastTaxSaleStatus|TaxRate|CondemnationStatus|MunicipalAcquisitionStatus|AcquisitionStatus|ExemptStatus|BankruptStatus|RefundStatus|ReturningVetExemptionAmount|DisabledPersonExemptionAmount|DisabledVetExemptionAmount|DisabledPersonVetExemptionAmount|HomeownerExemptAmount|SeniorFreezeExemptAmount|LongtimeHomeownersExemptAmount|LatePmt1|LatePmt2|DiffPmt1|WOE_TaxpayerPropertySuffix|WOE_Classification|WOE_TaxpayerPropertyTown|WOE_TaxStatus|WOE_AccountStatus|WOE_LastPaymentSource1|PastTaxSaleStatusSum|LatePmt2Sum|LatePmt1Sum|features                                                                                                                 |rawPrediction                          |probability                             |prediction|prob_prediction|\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+-------------------------------------------------------------------------------------------------------------------------+---------------------------------------+----------------------------------------+----------+---------------+\n",
            "|1     |0                   |0                   |10032  |0              |0           |0                 |0       |0.0               |0.0          |0.0           |0         |0                |0.0    |0                 |0                         |0                |1           |0             |0           |0.0                        |0.0                          |0.0                       |0.0                             |0.0                  |0.0                     |0.0                           |0       |0       |0       |2.19                      |9.21              |2.34                    |0.0          |2.15             |3.35                  |0                   |0          |0          |(38,[0,3,17,29,30,31,33,34],[1.0,10032.0,1.0,2.19,9.21,2.34,2.15,3.35])                                                  |[19.124134577964092,0.8758654220359062]|[0.9562067288982046,0.04379327110179531]|0.0       |0              |\n",
            "|1     |0                   |0                   |10100  |0              |0           |0                 |0       |0.0               |0.0          |0.0           |0         |0                |0.0    |0                 |0                         |0                |1           |0             |0           |0.0                        |0.0                          |0.0                       |0.0                             |0.0                  |0.0                     |0.0                           |0       |0       |0       |2.08                      |9.21              |2.34                    |0.0          |2.15             |3.35                  |0                   |0          |0          |(38,[0,3,17,29,30,31,33,34],[1.0,10100.0,1.0,2.08,9.21,2.34,2.15,3.35])                                                  |[19.124134577964092,0.8758654220359062]|[0.9562067288982046,0.04379327110179531]|0.0       |0              |\n",
            "|1     |0                   |0                   |10102  |0              |0           |0                 |0       |0.0               |0.0          |0.0           |0         |0                |0.0    |0                 |0                         |0                |1           |0             |0           |0.0                        |0.0                          |0.0                       |0.0                             |0.0                  |0.0                     |0.0                           |0       |0       |0       |2.08                      |9.21              |2.34                    |0.0          |2.15             |3.35                  |0                   |0          |0          |(38,[0,3,17,29,30,31,33,34],[1.0,10102.0,1.0,2.08,9.21,2.34,2.15,3.35])                                                  |[19.124134577964092,0.8758654220359062]|[0.9562067288982046,0.04379327110179531]|0.0       |0              |\n",
            "|1     |1                   |0                   |10012  |1              |0           |0                 |1       |899543.0          |0.0          |0.0           |0         |0                |6933.0 |0                 |0                         |0                |0           |0             |0           |0.0                        |0.0                          |0.0                       |0.0                             |10000.0              |0.0                     |0.0                           |0       |0       |0       |2.4                       |2.07              |2.34                    |2.03         |2.15             |1.37                  |0                   |0          |0          |(38,[0,1,3,4,7,8,13,24,29,30,31,32,33,34],[1.0,1.0,10012.0,1.0,1.0,899543.0,6933.0,10000.0,2.4,2.07,2.34,2.03,2.15,1.37])|[18.60108145780277,1.3989185421972252] |[0.9300540728901386,0.06994592710986128]|0.0       |0              |\n",
            "|1     |1                   |0                   |10021  |0              |0           |0                 |1       |364102.0          |0.0          |0.0           |1         |0                |7240.0 |0                 |0                         |0                |0           |0             |0           |0.0                        |0.0                          |0.0                       |0.0                             |0.0                  |0.0                     |0.0                           |0       |1       |0       |2.08                      |2.2               |2.34                    |2.03         |2.15             |1.37                  |0                   |0          |0          |(38,[0,1,3,7,8,11,13,29,30,31,32,33,34],[1.0,1.0,10021.0,1.0,364102.0,1.0,7240.0,2.08,2.2,2.34,2.03,2.15,1.37])          |[18.431889525747714,1.5681104742522822]|[0.9215944762873859,0.07840552371261413]|0.0       |0              |\n",
            "+------+--------------------+--------------------+-------+---------------+------------+------------------+--------+------------------+-------------+--------------+----------+-----------------+-------+------------------+--------------------------+-----------------+------------+--------------+------------+---------------------------+-----------------------------+--------------------------+--------------------------------+---------------------+------------------------+------------------------------+--------+--------+--------+--------------------------+------------------+------------------------+-------------+-----------------+----------------------+--------------------+-----------+-----------+-------------------------------------------------------------------------------------------------------------------------+---------------------------------------+----------------------------------------+----------+---------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "element=f.udf(lambda v:float(v[1]),FloatType())\n",
        "nums = [0.2, 0.25, 0.3, 0.35, 0.4, 0.5]\n",
        "acc = []\n",
        "f1 = []\n",
        "rec = []\n",
        "prec = []\n",
        "for num in nums:\n",
        "  print(num)\n",
        "  predictions2 = predictions.withColumn(\"prob_prediction\", when(element(\"probability\")>num, 1).otherwise(0))\n",
        "  pred = predictions2.select(predictions2.prob_prediction).toPandas()[\"prob_prediction\"]\n",
        "  pred = pred.tolist()\n",
        "  acc.append(accuracy_score(actual, pred))\n",
        "  f1.append(f1_score(actual, pred))\n",
        "  rec.append(recall_score(actual, pred))\n",
        "  prec.append(precision_score(actual, pred))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_KSWAKldNgXk",
        "outputId": "a0b757db-812d-4c3f-e054-f1c2444431f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.2\n",
            "0.25\n",
            "0.3\n",
            "0.35\n",
            "0.4\n",
            "0.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "res_df = pd.DataFrame(nums, columns=[\"Threshold\"])\n",
        "res_df[\"Acc\"] = acc\n",
        "res_df[\"F1\"] = f1\n",
        "res_df[\"Rec\"] = rec\n",
        "res_df[\"Prec\"] = prec\n",
        "res_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "-Fqy64pE4OvG",
        "outputId": "f02a32e2-a9ed-4f24-e565-11643e82e2b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Threshold       Acc        F1       Rec      Prec\n",
              "0       0.20  0.898545  0.508064  0.507650  0.508479\n",
              "1       0.25  0.903438  0.509589  0.486125  0.535434\n",
              "2       0.30  0.914462  0.508735  0.429158  0.624542\n",
              "3       0.35  0.917957  0.476846  0.362302  0.697302\n",
              "4       0.40  0.918561  0.455075  0.329505  0.735278\n",
              "5       0.50  0.918379  0.407131  0.271556  0.813042"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-95f4e42d-f4f2-4618-a8ef-25124ec033fe\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Threshold</th>\n",
              "      <th>Acc</th>\n",
              "      <th>F1</th>\n",
              "      <th>Rec</th>\n",
              "      <th>Prec</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.20</td>\n",
              "      <td>0.898545</td>\n",
              "      <td>0.508064</td>\n",
              "      <td>0.507650</td>\n",
              "      <td>0.508479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.25</td>\n",
              "      <td>0.903438</td>\n",
              "      <td>0.509589</td>\n",
              "      <td>0.486125</td>\n",
              "      <td>0.535434</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.30</td>\n",
              "      <td>0.914462</td>\n",
              "      <td>0.508735</td>\n",
              "      <td>0.429158</td>\n",
              "      <td>0.624542</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.35</td>\n",
              "      <td>0.917957</td>\n",
              "      <td>0.476846</td>\n",
              "      <td>0.362302</td>\n",
              "      <td>0.697302</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.40</td>\n",
              "      <td>0.918561</td>\n",
              "      <td>0.455075</td>\n",
              "      <td>0.329505</td>\n",
              "      <td>0.735278</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0.50</td>\n",
              "      <td>0.918379</td>\n",
              "      <td>0.407131</td>\n",
              "      <td>0.271556</td>\n",
              "      <td>0.813042</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-95f4e42d-f4f2-4618-a8ef-25124ec033fe')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-95f4e42d-f4f2-4618-a8ef-25124ec033fe button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-95f4e42d-f4f2-4618-a8ef-25124ec033fe');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cm = confusion_matrix(actual, pred)\n",
        "tn, fp, fn, tp = cm.ravel()"
      ],
      "metadata": {
        "id": "AKmzRl_C1dmP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Accuracy: \", accuracy_score(actual, pred))\n",
        "print(\"F1 Score: \", f1_score(actual, pred))\n",
        "print(\"Precision: \", precision_score(actual, pred))\n",
        "print(\"Recall: \", recall_score(actual, pred))"
      ],
      "metadata": {
        "id": "6mxKIv8O5Dba"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"True Negatives: \", tn)\n",
        "print(\"False Positives: \", fp)\n",
        "print(\"False Negatives: \", fn)\n",
        "print(\"True Positives: \", tp)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nwNW5M9u2m-V",
        "outputId": "67877ba7-2ff5-418d-b07d-df7b331c2a01"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "True Negatives:  1661930\n",
            "False Positives:  10219\n",
            "False Negatives:  147196\n",
            "True Positives:  45232\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "disp = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
        "disp.plot()\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 465
        },
        "id": "Tlcperik4Gw0",
        "outputId": "2362e3ef-9c81-4376-d3d2-299064756921"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfYAAAHACAYAAABQyEoMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/3UlEQVR4nO3de5yN5f7/8feaGXNgDs4zZgxDzmGIzJYUNZL6ie3brm+UidjfQslsFbucUlRKUsquMGkrOlEiZU8JUbZhOjLF0IzDDBrMgTmtdf/+mCytDNaatWbGrPv1fDzux6N1rfu678/S4rOuz3Xd920xDMMQAADwCj7VHQAAAPAcEjsAAF6ExA4AgBchsQMA4EVI7AAAeBESOwAAXoTEDgCAFyGxAwDgRUjsAAB4ERI7AABehMQOAPAKGzdu1MCBAxUZGSmLxaJVq1a5fAzDMPTss8+qTZs2CggIUFRUlJ588knPB1uJ/Ko7AAAAPKGgoECxsbEaOXKkhgwZUqFjjB8/Xp999pmeffZZderUSTk5OcrJyfFwpJXLwkNgAADexmKxaOXKlRo8eLC9raioSI8++qjefvttnThxQh07dtTTTz+tPn36SJJ27dqlzp0764cfflDbtm2rJ3APoBQPADCFcePGaevWrVq+fLm+++47/e1vf9ONN96oX375RZK0evVqtWzZUh9//LFatGihmJgYjRo1qsaN2EnsAACvl5GRoSVLlujdd99V7969ddlll2nixIm6+uqrtWTJEklSenq6fv31V7377rtaunSpkpKSlJKSoltvvbWao3cNc+wAAK/3/fffy2q1qk2bNg7tRUVFatCggSTJZrOpqKhIS5cute+3aNEidevWTWlpaTWmPE9iBwB4vfz8fPn6+iolJUW+vr4O7wUHB0uSmjRpIj8/P4fk3759e0llI34SOwAAl4iuXbvKarXqyJEj6t27d7n79OrVS6Wlpdq7d68uu+wySdLPP/8sSWrevHmVxeouVsUDALxCfn6+9uzZI6kskc+dO1d9+/ZV/fr11axZM91555366quv9Nxzz6lr1646evSokpOT1blzZ918882y2Wy68sorFRwcrHnz5slms2ns2LEKDQ3VZ599Vs2fznkkdgCAV9iwYYP69u17TntCQoKSkpJUUlKiJ554QkuXLtXBgwfVsGFD/eUvf9GMGTPUqVMnSdKhQ4d0//3367PPPlOdOnU0YMAAPffcc6pfv35Vf5wKI7EDAOBFuNwNAAAvQmIHAMCL1OhV8TabTYcOHVJISIgsFkt1hwMAcJFhGMrLy1NkZKR8fCpvrFlYWKji4mK3j+Pv76/AwEAPRFR5anRiP3TokKKjo6s7DACAmzIzM9W0adNKOXZhYaFaNA9W1hGr28eKiIjQvn37LunkXqMTe0hIiCTp1x0xCg1mVgHe6a9tOlV3CEClKVWJNmut/d/zylBcXKysI1b9mhKj0JCK54rcPJuad9uv4uJiEntlOVN+Dw32cet/FnAp87PUqu4QgMrz+3VZVTGdGhxiUXBIxc9jU82Y8q3RiR0AAGdZDZusblzgbTVsngumEpHYAQCmYJMhmyqe2d3pW5WoXwMA4EUYsQMATMEmm9wpprvXu+qQ2AEApmA1DFnduIu6O32rEqV4AAC8CCN2AIApsHgOAAAvYpMhqxubq4l948aNGjhwoCIjI2WxWLRq1aqL9ikqKtKjjz6q5s2bKyAgQDExMVq8eLFL52XEDgBAJSgoKFBsbKxGjhypIUOGONXntttuU3Z2thYtWqRWrVrp8OHDstlcW7RHYgcAmEJVl+IHDBigAQMGOL3/unXr9OWXXyo9PV3169eXJMXExLh0TolSPADAJM6sindnk6Tc3FyHraioyCPxffTRR+revbueeeYZRUVFqU2bNpo4caJOnz7t0nEYsQMA4II/P1V02rRpmj59utvHTU9P1+bNmxUYGKiVK1fq2LFjGjNmjH777TctWbLE6eOQ2AEApmD7fXOnv1T2iNnQ0FB7e0BAgDthnT2+zSaLxaJly5YpLCxMkjR37lzdeuutevnllxUUFOTUcUjsAABTOLO63Z3+khQaGuqQ2D2lSZMmioqKsid1SWrfvr0Mw9CBAwfUunVrp47DHDsAwBSshvtbZerVq5cOHTqk/Px8e9vPP/8sHx8fNW3a1OnjkNgBAKgE+fn5Sk1NVWpqqiRp3759Sk1NVUZGhiRp8uTJGj58uH3/oUOHqkGDBhoxYoR++uknbdy4UQ899JBGjhzpdBleIrEDAEzC5oHNFdu3b1fXrl3VtWtXSVJiYqK6du2qqVOnSpIOHz5sT/KSFBwcrPXr1+vEiRPq3r27hg0bpoEDB2r+/PkunZc5dgCAKdhkkVUWt/q7ok+fPjIu8OCYpKSkc9ratWun9evXuxqaA0bsAAB4EUbsAABTsBllmzv9awISOwDAFKxuluLd6VuVKMUDAOBFGLEDAEzBLCN2EjsAwBRshkU2w41V8W70rUqU4gEA8CKM2AEApkApHgAAL2KVj6xuFKqtHoylMpHYAQCmYLg5x24wxw4AAKoaI3YAgCkwxw4AgBexGj6yGm7MsdeQW8pSigcAwIswYgcAmIJNFtncGM/aVDOG7CR2AIApmGWOnVI8AABehBE7AMAU3F88RykeAIBLRtkcuxsPgaEUDwAAqhojdgCAKdjcvFc8q+IBALiEMMcOAIAXscnHFNexM8cOAIAXYcQOADAFq2GR1Y1Hr7rTtyqR2AEApmB1c/GclVI8AACoaozYAQCmYDN8ZHNjVbyNVfEAAFw6KMUDAIAahxE7AMAUbHJvZbvNc6FUKhI7AMAU3L9BTc0octeMKAEAgFNI7AAAUzhzr3h3Nlds3LhRAwcOVGRkpCwWi1atWuV036+++kp+fn7q0qWLax9SJHYAgEmceR67O5srCgoKFBsbqwULFrjU78SJExo+fLiuv/56l/qdwRw7AMAU3H+6m2t9BwwYoAEDBrh8nnvvvVdDhw6Vr6+vS6P8MxixAwBwiViyZInS09M1bdq0Ch+DETsAwBTcv0FNWd/c3FyH9oCAAAUEBLgVmyT98ssvmjRpkjZt2iQ/v4qnZ0bsAABTsBkWtzdJio6OVlhYmH2bPXu227FZrVYNHTpUM2bMUJs2bdw6FiN2AABckJmZqdDQUPtrT4zW8/LytH37du3cuVPjxo2TJNlsNhmGIT8/P3322We67rrrnDoWiR0AYAo2N0vxZ25QExoa6pDYPSE0NFTff/+9Q9vLL7+szz//XO+9955atGjh9LFI7AAAU3D/6W6u9c3Pz9eePXvsr/ft26fU1FTVr19fzZo10+TJk3Xw4EEtXbpUPj4+6tixo0P/xo0bKzAw8Jz2iyGxAwBQCbZv366+ffvaXycmJkqSEhISlJSUpMOHDysjI8Pj5yWxAwBMwSqLrC7eZObP/V3Rp08fGRd4hntSUtIF+0+fPl3Tp0936ZwSiR0AYBJVXYqvLjUjSgAA4BRG7AAAU7DK9XL6n/vXBCR2AIApmKUUT2IHAJhCVT8EprrUjCgBAIBTGLEDAEzBqMAz1f/cvyYgsQMATIFSPAAAqHEYsQMATOGPj16taP+agMQOADAFq5tPd3Onb1WqGVECAACnMGIHAJgCpXgAALyITT6yuVGodqdvVaoZUQIAAKcwYgcAmILVsMjqRjndnb5VicQOADAF5tgBAPAihptPdzO48xwAAKhqjNgBAKZglUVWNx7k4k7fqkRiBwCYgs1wb57cZngwmEpEKR4AAC/CiL2G+/7rOnr35cb65fvaysmupWmL9umqASfPu/+zDzbT+nfqn9PerM1pvbYhrdLi3Lg6TG8800TZB/wV1aJI9zx6SD2uz3PYJ+OXAC16IlLffR0sa6nUvE2Rpry2T42bllRaXKjZOsbl629jjqp1p1NqEFGq6SNjtHVdWKWec+Ddx3TrfUdUv1Gp0n8K0suPRSkttXY5exp64t/7dOV1eVUSFy7O5ubiOXf6VqVLIsoFCxYoJiZGgYGBiouL07Zt26o7pBqj8JSPWl5+WuNmHXBq//seP6C3U3+wb//e/qNC6pXqmv93/h8DF/PtlmAN79HhvO//+N/amj0mRjfe8Zte/ixNV914UjNGttD+3YH2fQ7t91fi4NaKblWoOe/t0cLkNA19MEv+gTWk9oVqEVjbpvQfA/XSP5t65Hj9bsvRM+/tOe/7195yXH+fdkjL5kZobP82Sv8pUE++la6wBuf++Pzr6GMy+PpeUmyyuL3VBNU+Yl+xYoUSExO1cOFCxcXFad68eerfv7/S0tLUuHHj6g7vknfldXm68rq8i+/4uzqhNtUJtdlfb/kkTPknfHXD//5mb7PZpHcWNNbafzfQ8aO1FNWySMMezFLvCib/Va83Uve+ufrbmKOSpISHs7RjY4g+XNJQ458u+0GS9FQT9bguV6OmHLb3i4wprtD5YB7bvwjV9i9Cz/t+LX+b7p6UpT6Djis4zKb9uwO16Mkm+m5rcIXON+Tvx7Turfr6bEVZ1Wv+I03V4/pc9b8jR++8FG7fr+Xlp/U//3dU9w9oreXf/lShcwEVVe0j9rlz52r06NEaMWKEOnTooIULF6p27dpavHhxdYdmCuverq+uvfMU/ody9/IXw/Wfd+vrgacP6NUvdmvI6KN6+v7m+m5rnQqdY1dKHXXtne/Q1u3aPO1KKTuezSZtSw5VVMsi/fOOlrqt0+V64ObW2vIJpUu4Z+yTB9W+W4Fm39dc917fRps+DtOTy9IV2aLI5WP51bKpdedT2rEpxN5mGBbt3BSiDt1O2dsCgmyatOBXLXg0SseP1vLI54BnnLnznDtbTVCtib24uFgpKSmKj4+3t/n4+Cg+Pl5bt26txsjM4bcsP/33i1DdODTH3lZcZNHy+Y2VODdD3fvkqUnzYt1we46uH3Jca95sWKHzHD/qp3oNHUuV9RqV6PiRsoLRiWN+Ol3gqxUvNVb3vnma/Xa6et14Uo+PiqnwjwmgUVTZd/eJv8foh23BOvxrgN5b2Fg//reO+t+ec/ED/Elofat8/aQTRx0LnceP+aleo1L76/+bflA/ba+jrZ/yw/RSc2aO3Z2tJqjWUvyxY8dktVoVHh7u0B4eHq7du3efs39RUZGKis7+0s7Nza30GL3Z+nfrKzjUqqtuPFtiP7Q/QEWnfTX5fy9z2Le0xKLLOp62vx7UqpP9v202i0qKLA5t1/3PcXuZ/WKM32cGevbP1ZC/l5XrL+t4Wj9tr6M1Sxuqc88Clz8b0KJdoXz9pMWbHf8tqeVvU+5xX0llyf+Pi0Z9fQ351jK06pfv7W3L5zfW8hcd/406n7/ccFJdeuVrzA1tPPAJgIqp9jl2V8yePVszZsyo7jC8gmFIny5voOtvzVEt/7MrfAoLyn6RznwzXQ0iHEfZf9zv5fVn/zFM21lHi55s4rDoqE7I2Xn8eo1KdfyYY0ny+NFaqte4bJRTNhIy1LxNocM+0a0L9eM2RuyomKA6VllLpXE3tpbN6lhCPf379/y3rFoa0+9sEu5100ldfdNJPT2umb0t70TZj4DcHF9ZS6W6fxidS1K9hqU6/vsovkuvfDWJKdYHu39w2GfKa/v1wzd19PCtrTz3AeEym9y8VzyL5y6uYcOG8vX1VXZ2tkN7dna2IiIiztl/8uTJSkxMtL/Ozc1VdHR0pcfpjb7bGqxD+wJ04x2OJclmbQpVK8CmIwf9LzhSjmpxdmHbscP+8vV1bPuj9t0KlLopWENGH7W37dgYovbdyo5fy99Qm9hTOrA3wKHfwfQALnVDhe35obZ8/aS6DUr1w7byF8vZrBYd2n/2e3fimJ+KCh3bzigt8dEv39VW16vz7JeuWSyGulydr4+SGkiSVrzUWJ+85Xg56atf/Kx/TY/U15+df5Efqobh5sp2g8R+cf7+/urWrZuSk5M1ePBgSZLNZlNycrLGjRt3zv4BAQEKCDj3L5yZnS7w0aF9Z/9MsjL9tfeHIIXULVXjpiVaPKuJjmXV0sPzMxz6ffp2fbW7okAx7RxHybWDbbr13iP617QoGTbp8h4FKsj11U//raPaIVb1u+24yzEOHnVUD/1Pa723sJF6XJ+rLz+sp1++C9KDczLt+/xtzBHNure5Ov4lX7FX5Wv7F6H6en2Y5lzg0iMgsLZVkX/4QRkRXayWl59W3glfHUwPUPL7dfXQ/Ey9OqOJ9vwQpLoNrOpydZ727QrStmTXE+0HrzbUxHmZ+vnb2krbWVt/HX1UgbVt+mx5WTI/frRWuQvmjhz0V3Ym/3ZVN57uVkUSExOVkJCg7t27q0ePHpo3b54KCgo0YsSI6g6tRvj529oO5b1/TY+SVHY97sR5Gco5UktHD/o79CnI9dHmNXV178zy58ATHs5SWINSLX8xXFkZ/qoTalWrTqd1xwPZ5e5/MZdfeUqTFuzXG083UdJTTRTZokjTFu9z+FHRa8BJPfDUAS1/KVyvTGmqpi3Lbk7TMY75dZxfm9jTmvP+Xvvre2cckiR9tqKenpvQTM9NaKahD2br79MOq0FEiXJzfLVrRx1985+KjZ6//KiewhpYNfyhLNVrVKr0H4P06LAWOnGM1e+4dFgMo/pvofDSSy9pzpw5ysrKUpcuXTR//nzFxcVdtF9ubq7CwsJ0/OeWCg2pGasVAVf1j+xS3SEAlabUKNEGfaiTJ08qNLRypivO5Iq/rh+hWnX8L97hPEoKirWy35JKjdUTqn3ELknjxo0rt/QOAICnmKUUzzAXAIBKsHHjRg0cOFCRkZGyWCxatWrVBff/4IMP1K9fPzVq1EihoaHq2bOnPv30U5fPS2IHAJhCVd8rvqCgQLGxsVqwYIFT+2/cuFH9+vXT2rVrlZKSor59+2rgwIHauXOnS+e9JErxAABUtqouxQ8YMEADBgxwev958+Y5vJ41a5Y+/PBDrV69Wl27dnX6OCR2AABc8Oe7nlbWpdg2m015eXmqX//cR21fCKV4AIApnBmxu7NJUnR0tMLCwuzb7NmzKyXeZ599Vvn5+brttttc6seIHQBgCp4qxWdmZjpc7lYZo/W33npLM2bM0IcffujyI8xJ7AAAuCA0NLRSr2Nfvny5Ro0apXfffdfh6afOIrEDAEyhJlzH/vbbb2vkyJFavny5br755godg8QOADAFQ+49oc3V27Tm5+drz56zz7vYt2+fUlNTVb9+fTVr1kyTJ0/WwYMHtXTpUkll5feEhAS98MILiouLU1ZWliQpKChIYWFhTp+XxXMAAFPw1OI5Z23fvl1du3a1X6qWmJiorl27aurUqZKkw4cPKyPj7AO6Xn31VZWWlmrs2LFq0qSJfRs/frxL52XEDgBAJejTp48u9DiWpKQkh9cbNmzwyHlJ7AAAU6gJc+yeQGIHAJiCWRI7c+wAAHgRRuwAAFMwy4idxA4AMAXDsMhwIzm707cqUYoHAMCLMGIHAJhCRZ6p/uf+NQGJHQBgCmaZY6cUDwCAF2HEDgAwBbMsniOxAwBMwSyleBI7AMAUzDJiZ44dAAAvwogdAGAKhpul+JoyYiexAwBMwZB0gaeoOtW/JqAUDwCAF2HEDgAwBZsssnDnOQAAvAOr4gEAQI3DiB0AYAo2wyILN6gBAMA7GIabq+JryLJ4SvEAAHgRRuwAAFMwy+I5EjsAwBRI7AAAeBGzLJ5jjh0AAC/CiB0AYApmWRVPYgcAmEJZYndnjt2DwVQiSvEAAHgRRuwAAFNgVTwAAF7EkHvPVK8hlXhK8QAAeBNG7AAAU6AUDwCANzFJLZ5SPADAHH4fsVd0k4sj9o0bN2rgwIGKjIyUxWLRqlWrLtpnw4YNuuKKKxQQEKBWrVopKSnJ5Y9JYgcAoBIUFBQoNjZWCxYscGr/ffv26eabb1bfvn2VmpqqBx98UKNGjdKnn37q0nkpxQMATKGq7zw3YMAADRgwwOn9Fy5cqBYtWui5556TJLVv316bN2/W888/r/79+zt9HEbsAABTcKcM/8eFd7m5uQ5bUVGRR+LbunWr4uPjHdr69++vrVu3unQcEjsAAC6Ijo5WWFiYfZs9e7ZHjpuVlaXw8HCHtvDwcOXm5ur06dNOH4dSPADAHCqwAO6c/pIyMzMVGhpqbw4ICHA3Mo8isQMATMFTc+yhoaEOid1TIiIilJ2d7dCWnZ2t0NBQBQUFOX0cSvEAAFwCevbsqeTkZIe29evXq2fPni4dh8QOADAHwwObC/Lz85WamqrU1FRJZZezpaamKiMjQ5I0efJkDR8+3L7/vffeq/T0dD388MPavXu3Xn75Zb3zzjuaMGGCS+d1qhT/0UcfOX3AW265xaUAAACoClV9S9nt27erb9++9teJiYmSpISEBCUlJenw4cP2JC9JLVq00Jo1azRhwgS98MILatq0qV5//XWXLnWTnEzsgwcPdupgFotFVqvVpQAAAPBGffr0kXGBSf3y7irXp08f7dy5063zOpXYbTabWycBAOCSUEPu9+4Ot1bFFxYWKjAw0FOxAABQaczydDeXF89ZrVbNnDlTUVFRCg4OVnp6uiRpypQpWrRokccDBADAI6p48Vx1cTmxP/nkk0pKStIzzzwjf39/e3vHjh31+uuvezQ4AADgGpcT+9KlS/Xqq69q2LBh8vX1tbfHxsZq9+7dHg0OAADPsXhgu/S5PMd+8OBBtWrV6px2m82mkpISjwQFAIDHuVtO99ZSfIcOHbRp06Zz2t977z117drVI0EBAICKcXnEPnXqVCUkJOjgwYOy2Wz64IMPlJaWpqVLl+rjjz+ujBgBAHAfI/byDRo0SKtXr9Z//vMf1alTR1OnTtWuXbu0evVq9evXrzJiBADAfWee7ubOVgNU6Dr23r17a/369Z6OBQAAuKnCN6jZvn27du3aJals3r1bt24eCwoAAE/z1GNbL3UuJ/YDBw7ojjvu0FdffaW6detKkk6cOKGrrrpKy5cvV9OmTT0dIwAA7mOOvXyjRo1SSUmJdu3apZycHOXk5GjXrl2y2WwaNWpUZcQIAACc5PKI/csvv9SWLVvUtm1be1vbtm314osvqnfv3h4NDgAAj3F3AZy3Lp6Ljo4u90Y0VqtVkZGRHgkKAABPsxhlmzv9awKXS/Fz5szR/fffr+3bt9vbtm/frvHjx+vZZ5/1aHAAAHiMSR4C49SIvV69erJYzpYgCgoKFBcXJz+/su6lpaXy8/PTyJEjNXjw4EoJFAAAXJxTiX3evHmVHAYAAJWMOfazEhISKjsOAAAql0kud6vwDWokqbCwUMXFxQ5toaGhbgUEAAAqzuXFcwUFBRo3bpwaN26sOnXqqF69eg4bAACXJJMsnnM5sT/88MP6/PPP9corryggIECvv/66ZsyYocjISC1durQyYgQAwH0mSewul+JXr16tpUuXqk+fPhoxYoR69+6tVq1aqXnz5lq2bJmGDRtWGXECAAAnuDxiz8nJUcuWLSWVzafn5ORIkq6++mpt3LjRs9EBAOApJnlsq8uJvWXLltq3b58kqV27dnrnnXcklY3kzzwUBgCAS82ZO8+5s9UELif2ESNG6Ntvv5UkTZo0SQsWLFBgYKAmTJighx56yOMBAgAA57k8xz5hwgT7f8fHx2v37t1KSUlRq1at1LlzZ48GBwCAx3Adu3OaN2+u5s2beyIWAADgJqcS+/z5850+4AMPPFDhYAAAqCwWufl0N49FUrmcSuzPP/+8UwezWCwkdgAAqpFTif3MKvhL1d+u6y8/n4DqDgOoFJaAo9UdAlBpLIaPVFRFJ+MhMAAAeBGTLJ5z+XI3AABw6WLEDgAwB0bsAAB4j+q689yCBQsUExOjwMBAxcXFadu2bRfcf968eWrbtq2CgoIUHR2tCRMmqLCw0OnzkdgBAKgkK1asUGJioqZNm6YdO3YoNjZW/fv315EjR8rd/6233tKkSZM0bdo07dq1S4sWLdKKFSv0z3/+0+lzViixb9q0SXfeead69uypgwcPSpLefPNNbd68uSKHAwCg8lXDY1vnzp2r0aNHa8SIEerQoYMWLlyo2rVra/HixeXuv2XLFvXq1UtDhw5VTEyMbrjhBt1xxx0XHeX/kcuJ/f3331f//v0VFBSknTt3qqio7DqFkydPatasWa4eDgCAquGhxJ6bm+uwncmDf1ZcXKyUlBTFx8fb23x8fBQfH6+tW7eW2+eqq65SSkqKPZGnp6dr7dq1uummm5z+mC4n9ieeeEILFy7Ua6+9plq1atnbe/XqpR07drh6OAAAapTo6GiFhYXZt9mzZ5e737Fjx2S1WhUeHu7QHh4erqysrHL7DB06VI8//riuvvpq1apVS5dddpn69OnjUine5VXxaWlpuuaaa85pDwsL04kTJ1w9HAAAVcLdR6+e6ZuZmanQ0FB7e0CA526QtmHDBs2aNUsvv/yy4uLitGfPHo0fP14zZ87UlClTnDqGy4k9IiJCe/bsUUxMjEP75s2b1bJlS1cPBwBA1fDQnedCQ0MdEvv5NGzYUL6+vsrOznZoz87OVkRERLl9pkyZorvuukujRo2SJHXq1EkFBQX6+9//rkcffVQ+PhcvtLtcih89erTGjx+vb775RhaLRYcOHdKyZcs0ceJE3Xfffa4eDgCAqlHFi+f8/f3VrVs3JScn29tsNpuSk5PVs2fPcvucOnXqnOTt6+tbFr7hXAAuj9gnTZokm82m66+/XqdOndI111yjgIAATZw4Uffff7+rhwMAwGslJiYqISFB3bt3V48ePTRv3jwVFBRoxIgRkqThw4crKirKPk8/cOBAzZ07V127drWX4qdMmaKBAwfaE/zFuJzYLRaLHn30UT300EPas2eP8vPz1aFDBwUHB7t6KAAAqoyn5thdcfvtt+vo0aOaOnWqsrKy1KVLF61bt86+oC4jI8NhhP7YY4/JYrHoscce08GDB9WoUSMNHDhQTz75pAtxOju2vwTl5uYqLCxM8U3v4+lu8FrWIzzdDd6r1CjRF0Xv6OTJk07NW1fEmVzRcuos+QQGVvg4tsJCpT/+z0qN1RNcHrH37dtXFsv5Fx98/vnnbgUEAAAqzuXE3qVLF4fXJSUlSk1N1Q8//KCEhARPxQUAgGe5WYqvKQ+BcTmxP//88+W2T58+Xfn5+W4HBABApeDpbq658847z3vvWwAAUDU89jz2rVu3KtCNRQkAAFQqk4zYXU7sQ4YMcXhtGIYOHz6s7du3O327OwAAqlp1XO5WHVxO7GFhYQ6vfXx81LZtWz3++OO64YYbPBYYAABwnUuJ3Wq1asSIEerUqZPq1atXWTEBAIAKcmnxnK+vr2644Qae4gYAqHmq+F7x1cXlVfEdO3ZUenp6ZcQCAEClOTPH7s5WE7ic2J944glNnDhRH3/8sQ4fPqzc3FyHDQAAVB+n59gff/xx/eMf/9BNN90kSbrlllscbi1rGIYsFousVqvnowQAwBNqyKjbHU4n9hkzZujee+/VF198UZnxAABQObiO3dGZh8Bde+21lRYMAABwj0uXu13oqW4AAFzKuEFNOdq0aXPR5J6Tk+NWQAAAVApK8eeaMWPGOXeeAwAAlw6XEvv//u//qnHjxpUVCwAAlYZS/J8wvw4AqNFMUop3+gY1Z1bFAwCAS5fTI3abzVaZcQAAULlMMmJ3+bGtAADURMyxAwDgTUwyYnf5ITAAAODSxYgdAGAOJhmxk9gBAKZgljl2SvEAAHgRRuwAAHOgFA8AgPegFA8AAGocRuwAAHOgFA8AgBcxSWKnFA8AgBchsQMATMHiga0iFixYoJiYGAUGBiouLk7btm274P4nTpzQ2LFj1aRJEwUEBKhNmzZau3at0+ejFA8AMIdqKMWvWLFCiYmJWrhwoeLi4jRv3jz1799faWlpaty48Tn7FxcXq1+/fmrcuLHee+89RUVF6ddff1XdunWdPieJHQBgCtVxudvcuXM1evRojRgxQpK0cOFCrVmzRosXL9akSZPO2X/x4sXKycnRli1bVKtWLUlSTEyMS+ekFA8AQCUoLi5WSkqK4uPj7W0+Pj6Kj4/X1q1by+3z0UcfqWfPnho7dqzCw8PVsWNHzZo1S1ar1enzMmIHAJiDh0rxubm5Ds0BAQEKCAg4Z/djx47JarUqPDzcoT08PFy7d+8u9xTp6en6/PPPNWzYMK1du1Z79uzRmDFjVFJSomnTpjkVJiN2AIB5GG5sv4uOjlZYWJh9mz17tsfCs9lsaty4sV599VV169ZNt99+ux599FEtXLjQ6WMwYgcAwAWZmZkKDQ21vy5vtC5JDRs2lK+vr7Kzsx3as7OzFRERUW6fJk2aqFatWvL19bW3tW/fXllZWSouLpa/v/9F42PEDgAwhTOL59zZJCk0NNRhO19i9/f3V7du3ZScnGxvs9lsSk5OVs+ePcvt06tXL+3Zs0c2m83e9vPPP6tJkyZOJXWJxA4AMAt3yvAVnJ9PTEzUa6+9pjfeeEO7du3Sfffdp4KCAvsq+eHDh2vy5Mn2/e+77z7l5ORo/Pjx+vnnn7VmzRrNmjVLY8eOdfqclOIBAKgkt99+u44ePaqpU6cqKytLXbp00bp16+wL6jIyMuTjc3aMHR0drU8//VQTJkxQ586dFRUVpfHjx+uRRx5x+pwkdgCAKVTXY1vHjRuncePGlfvehg0bzmnr2bOnvv7664qdTCR2AIBZ8BAYAABQ0zBiBwCYQnWV4qsaiR0AYA4mKcWT2AEA5mCSxM4cOwAAXoQROwDAFJhjBwDAm1CKBwAANQ0jdgCAKVgMQxaj4sNud/pWJRI7AMAcKMUDAICahhE7AMAUWBUPAIA3oRQPAABqGkbsAABToBQPAIA3MUkpnsQOADAFs4zYmWMHAMCLMGIHAJgDpXgAALxLTSmnu4NSPAAAXoQROwDAHAyjbHOnfw1AYgcAmAKr4gEAQI3DiB0AYA6sigcAwHtYbGWbO/1rAhK7l7m8y2/6nzvT1ardSTVoVKSZD3XT1xsjyt137CPf66YhGXr1+Q76cHkLSVKnK37TU698Xe7+D97dS7/sqqta/laNe+QHtWp3UtEx+dr2VWM98XD3c/a/+db9GnjrfjVuclpHs4O0Ykkrff5JU4d96gSXaPh9abqqT5ZCQkt0JCtIrz7fQdu3NHbzTwJmdNu9hzTykQNauThc/5rZXJL0zNu71PkveQ77rVnWSC8+Vvadb9H+lG6/95Au756v0Polyj4QoDXLGuvDpLN/by7vnqeRj2Qq+rLTCgiy6cjBAK19q7FWLj67z+33HVKv/sfV9LLTKi700U87grX46WgdSA+qgk8OnEVi9zKBQVbt+yVU61dH67FnUs67X89rs9Su4wkdOxLg0L7ru3q6c8D1Dm13/t/P6nLlMf2yK0yS5ONjqKjIRx+9E6NefbPKPf5NQ37V3WPSNH9WJ/3yU121ufyE7p/8nfLzamnb5nBJkp+fTU+8+I1OHvfXrMlX6LejgWoccVoF+bXc+SOASbXpnK+bhh5R+q5zE+natxvpzblR9tdFhb72/27dsUAnfqulZxJb6ughf3Xolq8HZu2XzWbR6qVl39XCUz5avTRc+3YHqfCUry6/Mk8PPLlfhad99MnbZT9CO8XlafWbjfXzd3Xk4yeNmJipJ5em6e/9OqnotK9wCaAUX/k2btyoOXPmKCUlRYcPH9bKlSs1ePDg6gypxkvZ2lgpWy882m3QqFD3TvxRUx7ooelz/+vwXmmpj47nBNpf+/ra9JdrsrX63RhJFklSUaGfXn6mkySpQ+fjqhNScs45rhtwQJ+sbKZN/4mUJGUdqq3W7U/o1uF77Ym938BMhYSWaOKoq2S1lq3jPHK4doU+N8wtsLZVD8/bqxcmt9Ad4w6d837RaR8dP+Zfbt/P3m3k8DorM1Dtr8hXr/459sS+96c62vtTHfs+2QcD1Kv/cXW8Ms+e2B+7u63DcZ57qKVWpOxU604F+mFbqFufD57BqvgqUFBQoNjYWC1YsKA6wzAVi8XQP6an6v1/t1TGvpCL7h93TbZCwoq1/uOmF933j2r521Rc7Pj1Ki7yVZsOJ+Tra7Mfe/f3dTXm4R/070/Wa8FbX+q2hD3y8akhf3twyRj7+H5t+7yudn4VVu77fQf9phUpO7Rw3fca8VCmAgKtFzxenRCr8k6ef9xzWYcCte+Wr++/Of/fodohZefIO0Fh9JJx5jp2d7YaoFq/cQMGDNCAAQOqMwTTuXX4XlmtFn20Isap/W+4JVM7vmmk3464Nk+Y8nUj9b8lU19/GaE9u0PVqt1J9R+UqVq1DIXWLdbx3wIVEXlK4d1Oa8OnkZo+oYeaNC3QmId/kK+fTW8valOBTwczuvb//aZWl5/SA4MuL/f9Lz5qoCMH/fVbtr9atDulkY9kqmnLQs28r3W5+7e/Ik/X3Jyjqfec+x18c8tOhdUvla+foWUvRGndivKrYxaLoXun/Kof/xusX3+mCoWqVaN+ShYVFamoqMj+Ojc3txqjqXlatTupQbfv1wPDr9aZsvqFNGh8WlfEHdVTj17h8rmWL26teg2K9Nyir2SRdDzHX8lronTr8HQZRtm5fXykE8f99eLszrLZLNqzO0wNGhXqf+5MJ7HDKQ2bFOneab/qn3e1U0lx+QXIM6VySdqfVls5R/z19Fu71aRZoQ5nBDrs27zNKU179Rctmx+pHZvOHf1PvK2DgupY1a5rvkY+nKlD+wO1YXWDc/Yb+/iviml7Wv/4Wwc3PyE8ySyl+BqV2GfPnq0ZM2ZUdxg11uVdchRWr0hJH35ub/P1M3TPAz9p0O37NPKv1zns3+//HVDeSX99szHc5XMVF/nqhSdi9dLsTqrboEjHjwXqxsEZOlXgp5PHy+Y6c44FyFpqkc129kdG5v5g1W9YJD8/m0pLuX8SLqx1x1Oq17BUL63+wd7m6yd17JGnW4Zna2DbKx2+X5K0O7VsrjwyxjGxN2t1Wk8t261PljfS2y9FqTzZB8oWm+5Pq616DUt054MHz0nsY2bsV9x1JzTx9vY6llX+vD6qCYvnLj2TJ09WYmKi/XVubq6io6OrMaKa5fO1UUrd1tCh7fEXvtEXnzQtZw7dUL//l6nPP4myL2yrCKvVx17Gv6bfIW3b3Ng+Yv/pu3rqc8MhWSyGvS2qWYF+OxpAUodTUreE6v/6d3Ro+8cz+5SZHqh3FjY5J6lL0mUdTkmSco6cTbrNW5/SU2/t1n/eb6g3nnXu3xSLpWwtyVmGxsz4VVfdcFwP39He/iMAWLBggebMmaOsrCzFxsbqxRdfVI8ePS7ab/ny5brjjjs0aNAgrVq1yunz1ajEHhAQoIAA/rJcSGBQqSKbFthfR0SeUsvWJ5WX66+j2UHKy3UcQVhLfXQ8J0AHM4Id2mO7/6aIqNP69MNm5Z4nukWeavnZFBJarKA6pWrZ+qQkKf2XsvJlZHS+2l5+Qmk/1lNwSIkGD01X88vyNPfxWPsx1r7fXAP/9qv+L/FHffROjKKaFei2u/dotZPz/8DpAt9z5rALT/so97iffv25tpo0K1TfQb9p2xd1lXfcTy3an9LfH8vQd9+EaN/usn7N25zS08t2K2VTmD54PUL1GhZLkmw2i07mlF16OfCubB055K/MvWU/Ujv1yNP/jD6sj944ex372Md/Vd9Bv2nG31vrdL6P/TgFeX4qLuKH6qWgOkrxK1asUGJiohYuXKi4uDjNmzdP/fv3V1pamho3Pv8VTPv379fEiRPVu3dvl89ZoxI7Lq51+5MON5gZPWGXJOk/HzfV8zNjz9ftHDfckqmfvq2nA78Gl/v+jLn/VXjkafvrF/+9WZJ0c9zNkiRfX0N/HbpPUc2/l7XUR9+lNNDEUVc5XM527EiQpjzQQ6Mn/KQFyzbpt6OB+mh5C7335mXOf2DgAkpKLOrSK1eDR2QpsLZNRw/566t19RxK7b0H5Khuw1Jd/9ffdP1ff7O3Zx/wV0LvLpIki4+hEQ8dUER0kaylFh3OCNDip6O19q2z/zAPvOuIJGnO8t0OMTw3sYXWv+94SR2qSTU83W3u3LkaPXq0RowYIUlauHCh1qxZo8WLF2vSpEnl9rFarRo2bJhmzJihTZs26cSJEy6d02IY1bd+Pz8/X3v27JEkde3aVXPnzlXfvn1Vv359NWtW/kjxj3JzcxUWFqb4pvfJz4eRPLyT9cjR6g4BqDSlRom+KHpHJ0+eVGho5VzvfyZX/OWmx+VXK/DiHc6jtKRQX6+d6nSsxcXFql27tt577z2He7QkJCToxIkT+vDDD8vtN23aNH333XdauXKl7r77bp04caLmlOK3b9+uvn372l+fmT9PSEhQUlJSNUUFAPBGnirF//mKrPNNEx87dkxWq1Xh4Y4LkMPDw7V79+5z9pekzZs3a9GiRUpNTa1wnNWa2Pv06aNqLBgAAMzEQ6vi/7xoe9q0aZo+fbobBy6Tl5enu+66S6+99poaNmx48Q7nwRw7AAAuyMzMdCjFn29Rd8OGDeXr66vs7GyH9uzsbEVEnPtwrr1792r//v0aOHCgvc1mK7vyws/PT2lpabrssouvQSKxAwBMwVOl+NDQUKfm2P39/dWtWzclJyfb59htNpuSk5M1bty4c/Zv166dvv/+e4e2xx57THl5eXrhhRecvrybxA4AMAebUba5099FiYmJSkhIUPfu3dWjRw/NmzdPBQUF9lXyw4cPV1RUlGbPnq3AwEB17Oh4X4a6detK0jntF0JiBwCYQzXcee7222/X0aNHNXXqVGVlZalLly5at26dfUFdRkaGfHw8e58DEjsAAJVo3Lhx5ZbeJWnDhg0X7FuRK8RI7AAAU7DIzTl2j0VSuUjsAABzqIY7z1UHbmAMAIAXYcQOADAFnscOAIA3Mcnz2CnFAwDgRRixAwBMwWIYsrixAM6dvlWJxA4AMAfb75s7/WsASvEAAHgRRuwAAFOgFA8AgDcxyap4EjsAwBy48xwAAKhpGLEDAEyBO88BAOBNKMUDAICahhE7AMAULLayzZ3+NQGJHQBgDpTiAQBATcOIHQBgDtygBgAA72GWW8pSigcAwIswYgcAmINJFs+R2AEA5mDIvWeq14y8TmIHAJgDc+wAAKDGYcQOADAHQ27OsXsskkpFYgcAmINJFs9RigcAwIswYgcAmINNksXN/jUAiR0AYAqsigcAADUOI3YAgDmYZPEciR0AYA4mSeyU4gEAqEQLFixQTEyMAgMDFRcXp23btp1339dee029e/dWvXr1VK9ePcXHx19w//KQ2AEA5nBmxO7O5qIVK1YoMTFR06ZN044dOxQbG6v+/fvryJEj5e6/YcMG3XHHHfriiy+0detWRUdH64YbbtDBgwedPieJHQBgDjYPbC6aO3euRo8erREjRqhDhw5auHChateurcWLF5e7/7JlyzRmzBh16dJF7dq10+uvvy6bzabk5GSnz0liBwCYwpnL3dzZJCk3N9dhKyoqKvd8xcXFSklJUXx8vL3Nx8dH8fHx2rp1q1Mxnzp1SiUlJapfv77Tn5PEDgCAC6KjoxUWFmbfZs+eXe5+x44dk9VqVXh4uEN7eHi4srKynDrXI488osjISIcfBxfDqngAgDl4aFV8ZmamQkND7c0BAQHuRlaup556SsuXL9eGDRsUGBjodD8SOwDAHGyGZHEjsdvK+oaGhjok9vNp2LChfH19lZ2d7dCenZ2tiIiIC/Z99tln9dRTT+k///mPOnfu7FKYlOIBAKgE/v7+6tatm8PCtzML4Xr27Hnefs8884xmzpypdevWqXv37i6flxE7AMAcquEGNYmJiUpISFD37t3Vo0cPzZs3TwUFBRoxYoQkafjw4YqKirLP0z/99NOaOnWq3nrrLcXExNjn4oODgxUcHOzUOUnsAACTcDOxy/W+t99+u44ePaqpU6cqKytLXbp00bp16+wL6jIyMuTjc7Z4/sorr6i4uFi33nqrw3GmTZum6dOnO3VOEjsAAJVo3LhxGjduXLnvbdiwweH1/v373T4fiR0AYA4muVc8iR0AYA42QxUppzv2v/SxKh4AAC/CiB0AYA6GrWxzp38NQGIHAJgDc+wAAHgR5tgBAEBNw4gdAGAOlOIBAPAihtxM7B6LpFJRigcAwIswYgcAmAOleAAAvIjNJsmNa9FtNeM6dkrxAAB4EUbsAABzoBQPAIAXMUlipxQPAIAXYcQOADAHk9xSlsQOADAFw7DJcOMJbe70rUokdgCAORiGe6Nu5tgBAEBVY8QOADAHw8059hoyYiexAwDMwWaTLG7Mk9eQOXZK8QAAeBFG7AAAc6AUDwCA9zBsNhlulOJryuVulOIBAPAijNgBAOZAKR4AAC9iMySL9yd2SvEAAHgRRuwAAHMwDEnuXMdeM0bsJHYAgCkYNkOGG6V4g8QOAMAlxLDJvRE7l7sBAIAqxogdAGAKlOIBAPAmJinF1+jEfubXU6mtuJojASqP1Sip7hCASlP6+/e7KkbDpSpx6/40paoZfxdrdGLPy8uTJG04tKiaIwEAuCMvL09hYWGVcmx/f39FRERoc9Zat48VEREhf39/D0RVeSxGTZk0KIfNZtOhQ4cUEhIii8VS3eGYQm5urqKjo5WZmanQ0NDqDgfwKL7fVc8wDOXl5SkyMlI+PpW3nruwsFDFxe5Xd/39/RUYGOiBiCpPjR6x+/j4qGnTptUdhimFhobyDx+8Ft/vqlVZI/U/CgwMvOQTsqdwuRsAAF6ExA4AgBchscMlAQEBmjZtmgICAqo7FMDj+H7DG9ToxXMAAMARI3YAALwIiR0AAC9CYgcAwIuQ2OG0BQsWKCYmRoGBgYqLi9O2bduqOyTAIzZu3KiBAwcqMjJSFotFq1atqu6QgAojscMpK1asUGJioqZNm6YdO3YoNjZW/fv315EjR6o7NMBtBQUFio2N1YIFC6o7FMBtrIqHU+Li4nTllVfqpZdeklR2O9/o6Gjdf//9mjRpUjVHB3iOxWLRypUrNXjw4OoOBagQRuy4qOLiYqWkpCg+Pt7e5uPjo/j4eG3durUaIwMA/BmJHRd17NgxWa1WhYeHO7SHh4crKyurmqICAJSHxA4AgBchseOiGjZsKF9fX2VnZzu0Z2dnKyIiopqiAgCUh8SOi/L391e3bt2UnJxsb7PZbEpOTlbPnj2rMTIAwJ/V6Oexo+okJiYqISFB3bt3V48ePTRv3jwVFBRoxIgR1R0a4Lb8/Hzt2bPH/nrfvn1KTU1V/fr11axZs2qMDHAdl7vBaS+99JLmzJmjrKwsdenSRfPnz1dcXFx1hwW4bcOGDerbt+857QkJCUpKSqr6gAA3kNgBAPAizLEDAOBFSOwAAHgREjsAAF6ExA4AgBchsQMA4EVI7AAAeBESOwAAXoTEDgCAFyGxA266++67NXjwYPvrPn366MEHH6zyODZs2CCLxaITJ06cdx+LxaJVq1Y5fczp06erS5cubsW1f/9+WSwWpaamunUcAM4hscMr3X333bJYLLJYLPL391erVq30+OOPq7S0tNLP/cEHH2jmzJlO7etMMgYAV/AQGHitG2+8UUuWLFFRUZHWrl2rsWPHqlatWpo8efI5+xYXF8vf398j561fv75HjgMAFcGIHV4rICBAERERat68ue677z7Fx8fro48+knS2fP7kk08qMjJSbdu2lSRlZmbqtttuU926dVW/fn0NGjRI+/fvtx/TarUqMTFRdevWVYMGDfTwww/rz49b+HMpvqioSI888oiio6MVEBCgVq1aadGiRdq/f7/9wSP16tWTxWLR3XffLanssbizZ89WixYtFBQUpNjYWL333nsO51m7dq3atGmjoKAg9e3b1yFOZz3yyCNq06aNateurZYtW2rKlCkqKSk5Z79//etfio6OVu3atXXbbbfp5MmTDu+//vrrat++vQIDA9WuXTu9/PLLLscCwDNI7DCNoKAgFRcX218nJycrLS1N69ev18cff6ySkhL1799fISEh2rRpk7766isFBwfrxhtvtPd77rnnlJSUpMWLF2vz5s3KycnRypUrL3je4cOH6+2339b8+fO1a9cu/etf/1JwcLCio6P1/vvvS5LS0tJ0+PBhvfDCC5Kk2bNna+nSpVq4cKF+/PFHTZgwQXfeeae+/PJLSWU/QIYMGaKBAwcqNTVVo0aN0qRJk1z+MwkJCVFSUpJ++uknvfDCC3rttdf0/PPPO+yzZ88evfPOO1q9erXWrVunnTt3asyYMfb3ly1bpqlTp+rJJ5/Url27NGvWLE2ZMkVvvPGGy/EA8AAD8EIJCQnGoEGDDMMwDJvNZqxfv94ICAgwJk6caH8/PDzcKCoqsvd58803jbZt2xo2m83eVlRUZAQFBRmffvqpYRiG0aRJE+OZZ56xv19SUmI0bdrUfi7DMIxrr73WGD9+vGEYhpGWlmZIMtavX19unF988YUhyTh+/Li9rbCw0Khdu7axZcsWh33vuece44477jAMwzAmT55sdOjQweH9Rx555Jxj/ZkkY+XKled9f86cOUa3bt3sr6dNm2b4+voaBw4csLd98sknho+Pj3H48GHDMAzjsssuM9566y2H48ycOdPo2bOnYRiGsW/fPkOSsXPnzvOeF4DnMMcOr/Xxxx8rODhYJSUlstlsGjp0qKZPn25/v1OnTg7z6t9++6327NmjkJAQh+MUFhZq7969OnnypA4fPuzwDHo/Pz917979nHL8GampqfL19dW1117rdNx79uzRqVOn1K9fP4f24uJide3aVZK0a9cuhzgkqWfPnk6f44wVK1Zo/vz52rt3r/Lz81VaWqrQ0FCHfZo1a6aoqCiH89hsNqWlpSkkJER79+7VPffco9GjR9v3KS0tVVhYmMvxAHAfiR1eq2/fvnrllVfk7++vyMhI+fk5ft3r1Knj8Do/P1/dunXTsmXLzjlWo0aNKhRDUFCQy33y8/MlSWvWrHFIqFLZugFP2bp1q4YNG6YZM2aof//+CgsL0/Lly/Xcc8+5HOtrr712zg8NX19fj8UKwHkkdnitOnXqqFWrVk7vf8UVV2jFihVq3LjxOaPWM5o0aaJvvvlG11xzjaSykWlKSoquuOKKcvfv1KmTbDabvvzyS8XHx5/z/pmKgdVqtbd16NBBAQEBysjIOO9Iv3379vaFgGd8/fXXF/+Qf7BlyxY1b95cjz76qL3t119/PWe/jIwMHTp0SJGRkfbz+Pj4qG3btgoPD1dkZKTS09M1bNgwl84PoHKweA743bBhw9SwYUMNGjRImzZt0r59+7RhwwY98MADOnDggCRp/Pjxeuqpp7Rq1Srt3r1bY8aMueA16DExMUpISNDIkSO1atUq+zHfeecdSVLz5s1lsVj08ccf6+jRo8rPz1dISIgmTpyoCRMm6I033tDevXu1Y8cOvfjii/YFaffee69++eUXPfTQQ0pLS9Nbb72lpKQklz5v69atlZGRoeXLl2vv3r2aP39+uQsBAwMDlZCQoG+//VabNm3SAw88oNtuu00RERGSpBkzZmj27NmaP3++fv75Z33//fdasmSJ5s6d61I8ADyDxA78rnbt2tq4caOaNWumIUOGqH379rrnnntUWFhoH8H/4x//0F133aWEhAT17NlTISEh+utf/3rB477yyiu69dZbNWbMGLVr106jR49WQUGBJCkqKkozZszQpEmTFB4ernHjxkmSZs6cqSlTpmj27Nlq3769brzxRq1Zs0YtWrSQVDbv/f7772vVqlWKjY3VwoULNWvWLJc+7y233KIJEyZo3Lhx6tKli7Zs2aIpU6acs1+rVq00ZMgQ3XTTTbrhhhvUuXNnh8vZRo0apddff11LlixRp06ddO211yopKckeK4CqZTHOt+oHAADUOIzYAQDwIiR2AAC8CIkdAAAvQmIHAMCLkNgBAPAiJHYAALwIiR0AAC9CYgcAwIuQ2AEA8CIkdgAAvAiJHQAAL0JiBwDAi/x/f/XKKB45UkoAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "feat_imp = rfModel.featureImportances"
      ],
      "metadata": {
        "id": "CM_GrHvQ6I-C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "feat_imp = [c for c in feat_imp]"
      ],
      "metadata": {
        "id": "yRbnKkjwNxEC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "feat_imp"
      ],
      "metadata": {
        "id": "DvKaLGuuSDWb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_cols = train.columns\n",
        "train_cols.remove(\"features\")\n",
        "train_cols.remove(\"LatePmt2\")\n",
        "feat_df = pd.DataFrame(train_cols, columns=[\"Features\"])\n",
        "feat_df[\"Importance\"] = feat_imp\n",
        "feat_df.sort_values(\"Importance\", ascending=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "lcFEw2oFOslK",
        "outputId": "acb15e4a-4b92-4413-9fb9-8788ea0c045d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                            Features    Importance\n",
              "28                          DiffPmt1  3.715082e-01\n",
              "27                          LatePmt1  1.963587e-01\n",
              "37                       LatePmt1Sum  1.818436e-01\n",
              "34            WOE_LastPaymentSource1  1.162163e-01\n",
              "36                       LatePmt2Sum  5.716733e-02\n",
              "35              PastTaxSaleStatusSum  2.044941e-02\n",
              "12                 PastTaxSaleStatus  1.226229e-02\n",
              "9                      TaxAmountDue1  1.030470e-02\n",
              "13                           TaxRate  9.716395e-03\n",
              "10                    CostAmountDue1  5.080882e-03\n",
              "31          WOE_TaxpayerPropertyTown  4.475199e-03\n",
              "32                     WOE_TaxStatus  3.730332e-03\n",
              "30                WOE_Classification  3.221506e-03\n",
              "33                 WOE_AccountStatus  2.068290e-03\n",
              "17                      ExemptStatus  1.653200e-03\n",
              "24             HomeownerExemptAmount  1.315625e-03\n",
              "4                    HomeownerExempt  9.241792e-04\n",
              "29        WOE_TaxpayerPropertySuffix  5.854154e-04\n",
              "25          SeniorFreezeExemptAmount  2.669256e-04\n",
              "7                           BillType  2.547254e-04\n",
              "3                            TaxCode  2.226437e-04\n",
              "5                       SeniorExempt  1.512995e-04\n",
              "8                 AdjustedAmountDue1  1.151299e-04\n",
              "11                        CofENumber  7.994022e-05\n",
              "1               TaxpayerMailingState  2.175690e-05\n",
              "6                 SeniorFreezeExempt  5.060187e-06\n",
              "15        MunicipalAcquisitionStatus  9.979760e-07\n",
              "0                             Volume  0.000000e+00\n",
              "26    LongtimeHomeownersExemptAmount  0.000000e+00\n",
              "23  DisabledPersonVetExemptionAmount  0.000000e+00\n",
              "22        DisabledVetExemptionAmount  0.000000e+00\n",
              "21     DisabledPersonExemptionAmount  0.000000e+00\n",
              "20       ReturningVetExemptionAmount  0.000000e+00\n",
              "18                    BankruptStatus  0.000000e+00\n",
              "16                 AcquisitionStatus  0.000000e+00\n",
              "14                CondemnationStatus  0.000000e+00\n",
              "2               TaxpayerPropertyCity  0.000000e+00\n",
              "19                      RefundStatus  0.000000e+00"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4c989bac-7f66-454c-8646-ecd466057a72\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Features</th>\n",
              "      <th>Importance</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>DiffPmt1</td>\n",
              "      <td>3.715082e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>LatePmt1</td>\n",
              "      <td>1.963587e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>LatePmt1Sum</td>\n",
              "      <td>1.818436e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>WOE_LastPaymentSource1</td>\n",
              "      <td>1.162163e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>LatePmt2Sum</td>\n",
              "      <td>5.716733e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>PastTaxSaleStatusSum</td>\n",
              "      <td>2.044941e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>PastTaxSaleStatus</td>\n",
              "      <td>1.226229e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>TaxAmountDue1</td>\n",
              "      <td>1.030470e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>TaxRate</td>\n",
              "      <td>9.716395e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>CostAmountDue1</td>\n",
              "      <td>5.080882e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>WOE_TaxpayerPropertyTown</td>\n",
              "      <td>4.475199e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>WOE_TaxStatus</td>\n",
              "      <td>3.730332e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>WOE_Classification</td>\n",
              "      <td>3.221506e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>WOE_AccountStatus</td>\n",
              "      <td>2.068290e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>ExemptStatus</td>\n",
              "      <td>1.653200e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>HomeownerExemptAmount</td>\n",
              "      <td>1.315625e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>HomeownerExempt</td>\n",
              "      <td>9.241792e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>WOE_TaxpayerPropertySuffix</td>\n",
              "      <td>5.854154e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>SeniorFreezeExemptAmount</td>\n",
              "      <td>2.669256e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>BillType</td>\n",
              "      <td>2.547254e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>TaxCode</td>\n",
              "      <td>2.226437e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>SeniorExempt</td>\n",
              "      <td>1.512995e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>AdjustedAmountDue1</td>\n",
              "      <td>1.151299e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>CofENumber</td>\n",
              "      <td>7.994022e-05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>TaxpayerMailingState</td>\n",
              "      <td>2.175690e-05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>SeniorFreezeExempt</td>\n",
              "      <td>5.060187e-06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>MunicipalAcquisitionStatus</td>\n",
              "      <td>9.979760e-07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Volume</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>LongtimeHomeownersExemptAmount</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>DisabledPersonVetExemptionAmount</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>DisabledVetExemptionAmount</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>DisabledPersonExemptionAmount</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>ReturningVetExemptionAmount</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>BankruptStatus</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>AcquisitionStatus</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>CondemnationStatus</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>TaxpayerPropertyCity</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>RefundStatus</td>\n",
              "      <td>0.000000e+00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4c989bac-7f66-454c-8646-ecd466057a72')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4c989bac-7f66-454c-8646-ecd466057a72 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4c989bac-7f66-454c-8646-ecd466057a72');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    }
  ]
}