RealStateTransactionsPerMunicipality/Real State Analysis.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "gather": {
          "logged": 1671712902514
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: matplotlib in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (3.6.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (21.3)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (9.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (2.8.2)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (4.38.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (0.11.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (1.4.4)\n",
            "Requirement already satisfied: pyparsing>=2.2.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (3.0.9)\n",
            "Requirement already satisfied: numpy>=1.19 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (1.23.1)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib) (1.0.6)\n",
            "Requirement already satisfied: six>=1.5 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
            "Note: you may need to restart the kernel to use updated packages.\n",
            "Requirement already satisfied: seaborn in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (0.12.1)\n",
            "Requirement already satisfied: pandas>=0.25 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from seaborn) (1.4.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from seaborn) (1.23.1)\n",
            "Requirement already satisfied: matplotlib!=3.6.1,>=3.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from seaborn) (3.6.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (2.8.2)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (9.2.0)\n",
            "Requirement already satisfied: pyparsing>=2.2.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (3.0.9)\n",
            "Requirement already satisfied: packaging>=20.0 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (21.3)\n",
            "Requirement already satisfied: cycler>=0.10 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (0.11.0)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (4.38.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (1.4.4)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from matplotlib!=3.6.1,>=3.1->seaborn) (1.0.6)\n",
            "Requirement already satisfied: pytz>=2020.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from pandas>=0.25->seaborn) (2022.1)\n",
            "Requirement already satisfied: six>=1.5 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib!=3.6.1,>=3.1->seaborn) (1.16.0)\n",
            "Note: you may need to restart the kernel to use updated packages.\n",
            "Requirement already satisfied: scikit-learn in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (1.2.0)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from scikit-learn) (1.8.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from scikit-learn) (1.2.0)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from scikit-learn) (1.23.1)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages (from scikit-learn) (3.1.0)\n",
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        }
      ],
      "source": [
        "#First lets install the missing packages\n",
        "\n",
        "%pip install matplotlib\n",
        "%pip install seaborn\n",
        "%pip install scikit-learn"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "gather": {
          "logged": 1671712903549
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Import Libraries Required\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import sklearn"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "# Real State Transactions per Municipality in Belgium\n",
        "\n",
        "We have been provided with a transactions dataset per municipality in Belgium (2010-2018), the dataset contains the Quarterly transactions per municipality but it also includes the aggregation by semester and year, at a high level only, basically it includes the number of transactions, the total amount and the total surface.\n",
        "\n",
        "Our tasks:\n",
        "\n",
        "1. Data Cleaning.\n",
        "\n",
        "2. Data Preparation.\n",
        "\n",
        "\n",
        "## Lets introduce CRISP-DM\n",
        "\n",
        "CRISP-DM (Cross-Industry Standard Process for Data Mining) is a widely-used framework for structuring data mining and machine learning projects. It is designed to provide a systematic and organized approach to solving business problems using data-driven techniques.\n",
        "\n",
        "The CRISP-DM process is divided into six main phases:\n",
        "\n",
        "### Business Understanding:\n",
        "In this phase, the team defines the problem or opportunity that the project will address, as well as the objectives and goals of the project.\n",
        "\n",
        "### Data Understanding: \n",
        "In this phase, the team investigates the available data to determine what data is available and how it can be used to address the business problem.\n",
        "\n",
        "### Data Preparation: \n",
        "In this phase, the team cleans and prepares the data for analysis. This can include tasks such as missing value imputation, outlier detection and removal, and data transformation.\n",
        "\n",
        "### Modeling: \n",
        "In this phase, the team builds and tests machine learning models to find the best solution to the business problem.\n",
        "\n",
        "### Evaluation: \n",
        "In this phase, the team assesses the performance of the chosen model and determines whether it meets the objectives and goals of the project.\n",
        "\n",
        "### Deployment: \n",
        "In this phase, the team implements the chosen solution and makes it available for use in the business.\n",
        "\n",
        "By following the CRISP-DM process, data scientists and business analysts can ensure that their projects are well-structured, well-documented, and aligned with the business objectives of the organization.\n",
        "\n",
        "For the sake of this Portfolio Project only Data Understanding and Data Preparation phases will be done."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Data Understanding\n",
        "\n",
        "The second stage of the CRISP-DM process requires you to acquire the data listed in the project resources. This initial collection includes data loading, if this is necessary for data understanding. For example, if you use a specific tool for data understanding, it makes perfect sense to load your data into this tool. If you acquire multiple data sources then you need to consider how and when you're going to integrate these.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "### Initial Data Report\n",
        "Initial data collection report - List the data sources acquired together with their locations, the methods used to acquire them and any problems encountered. Record problems you encountered and any resolutions achieved. This will help both with future replication of this project and with the execution of similar future projects.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "gather": {
          "logged": 1671712904004
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>CD_TYPE_NL</th>\n",
              "      <th>CD_TYPE_FR</th>\n",
              "      <th>CD_REFNIS</th>\n",
              "      <th>CD_REFNIS_NL</th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_PERIOD</th>\n",
              "      <th>CD_CLASS_SURFACE</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <th>MS_P10</th>\n",
              "      <th>MS_P25</th>\n",
              "      <th>MS_P50</th>\n",
              "      <th>MS_P75</th>\n",
              "      <th>MS_P90</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q1</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>25</td>\n",
              "      <td>6.207500e+06</td>\n",
              "      <td>7676.0</td>\n",
              "      <td>248300.00000</td>\n",
              "      <td>185000.0</td>\n",
              "      <td>225000.0</td>\n",
              "      <td>247000.0</td>\n",
              "      <td>265000.0</td>\n",
              "      <td>320000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q2</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>21</td>\n",
              "      <td>5.548500e+06</td>\n",
              "      <td>6868.0</td>\n",
              "      <td>264214.28571</td>\n",
              "      <td>200000.0</td>\n",
              "      <td>212000.0</td>\n",
              "      <td>252500.0</td>\n",
              "      <td>285000.0</td>\n",
              "      <td>300000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q3</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>20</td>\n",
              "      <td>5.317000e+06</td>\n",
              "      <td>5141.0</td>\n",
              "      <td>265850.00000</td>\n",
              "      <td>163750.0</td>\n",
              "      <td>232500.0</td>\n",
              "      <td>252500.0</td>\n",
              "      <td>282500.0</td>\n",
              "      <td>370000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q4</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>24</td>\n",
              "      <td>6.004201e+06</td>\n",
              "      <td>9593.0</td>\n",
              "      <td>250175.05000</td>\n",
              "      <td>175000.0</td>\n",
              "      <td>210000.0</td>\n",
              "      <td>245000.0</td>\n",
              "      <td>284500.0</td>\n",
              "      <td>334000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>S1</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>46</td>\n",
              "      <td>1.175600e+07</td>\n",
              "      <td>14544.0</td>\n",
              "      <td>255565.21739</td>\n",
              "      <td>197500.0</td>\n",
              "      <td>220000.0</td>\n",
              "      <td>251000.0</td>\n",
              "      <td>270000.0</td>\n",
              "      <td>320000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>S2</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>44</td>\n",
              "      <td>1.132120e+07</td>\n",
              "      <td>14734.0</td>\n",
              "      <td>257300.02727</td>\n",
              "      <td>165000.0</td>\n",
              "      <td>220850.6</td>\n",
              "      <td>250000.0</td>\n",
              "      <td>282500.0</td>\n",
              "      <td>340000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>90</td>\n",
              "      <td>2.307720e+07</td>\n",
              "      <td>29278.0</td>\n",
              "      <td>256413.34667</td>\n",
              "      <td>180000.0</td>\n",
              "      <td>220000.0</td>\n",
              "      <td>250000.0</td>\n",
              "      <td>280000.0</td>\n",
              "      <td>332000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11002</td>\n",
              "      <td>ANTWERPEN</td>\n",
              "      <td>ANVERS</td>\n",
              "      <td>Q1</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>746</td>\n",
              "      <td>1.590808e+08</td>\n",
              "      <td>125331.0</td>\n",
              "      <td>213245.10657</td>\n",
              "      <td>115000.0</td>\n",
              "      <td>150000.0</td>\n",
              "      <td>192500.0</td>\n",
              "      <td>250000.0</td>\n",
              "      <td>320000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11002</td>\n",
              "      <td>ANTWERPEN</td>\n",
              "      <td>ANVERS</td>\n",
              "      <td>Q2</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>797</td>\n",
              "      <td>1.695462e+08</td>\n",
              "      <td>141220.0</td>\n",
              "      <td>212730.54090</td>\n",
              "      <td>120000.0</td>\n",
              "      <td>153000.0</td>\n",
              "      <td>193000.0</td>\n",
              "      <td>247000.0</td>\n",
              "      <td>329000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11002</td>\n",
              "      <td>ANTWERPEN</td>\n",
              "      <td>ANVERS</td>\n",
              "      <td>Q3</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>865</td>\n",
              "      <td>1.910487e+08</td>\n",
              "      <td>161453.0</td>\n",
              "      <td>220865.50306</td>\n",
              "      <td>125000.0</td>\n",
              "      <td>157000.0</td>\n",
              "      <td>205000.0</td>\n",
              "      <td>264000.0</td>\n",
              "      <td>335000.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   CD_YEAR         CD_TYPE_NL            CD_TYPE_FR  CD_REFNIS CD_REFNIS_NL  \\\n",
              "0     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "1     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "2     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "3     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "4     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "5     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "6     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "7     2010  gewone woonhuizen  maisons d'habitation      11002    ANTWERPEN   \n",
              "8     2010  gewone woonhuizen  maisons d'habitation      11002    ANTWERPEN   \n",
              "9     2010  gewone woonhuizen  maisons d'habitation      11002    ANTWERPEN   \n",
              "\n",
              "  CD_REFNIS_FR CD_PERIOD CD_CLASS_SURFACE  MS_TOTAL_TRANSACTIONS  \\\n",
              "0   AARTSELAAR        Q1   totaal / total                     25   \n",
              "1   AARTSELAAR        Q2   totaal / total                     21   \n",
              "2   AARTSELAAR        Q3   totaal / total                     20   \n",
              "3   AARTSELAAR        Q4   totaal / total                     24   \n",
              "4   AARTSELAAR        S1   totaal / total                     46   \n",
              "5   AARTSELAAR        S2   totaal / total                     44   \n",
              "6   AARTSELAAR         Y   totaal / total                     90   \n",
              "7       ANVERS        Q1   totaal / total                    746   \n",
              "8       ANVERS        Q2   totaal / total                    797   \n",
              "9       ANVERS        Q3   totaal / total                    865   \n",
              "\n",
              "   MS_TOTAL_PRICE  MS_TOTAL_SURFACE  MS_MEAN_PRICE    MS_P10    MS_P25  \\\n",
              "0    6.207500e+06            7676.0   248300.00000  185000.0  225000.0   \n",
              "1    5.548500e+06            6868.0   264214.28571  200000.0  212000.0   \n",
              "2    5.317000e+06            5141.0   265850.00000  163750.0  232500.0   \n",
              "3    6.004201e+06            9593.0   250175.05000  175000.0  210000.0   \n",
              "4    1.175600e+07           14544.0   255565.21739  197500.0  220000.0   \n",
              "5    1.132120e+07           14734.0   257300.02727  165000.0  220850.6   \n",
              "6    2.307720e+07           29278.0   256413.34667  180000.0  220000.0   \n",
              "7    1.590808e+08          125331.0   213245.10657  115000.0  150000.0   \n",
              "8    1.695462e+08          141220.0   212730.54090  120000.0  153000.0   \n",
              "9    1.910487e+08          161453.0   220865.50306  125000.0  157000.0   \n",
              "\n",
              "     MS_P50    MS_P75    MS_P90  \n",
              "0  247000.0  265000.0  320000.0  \n",
              "1  252500.0  285000.0  300000.0  \n",
              "2  252500.0  282500.0  370000.0  \n",
              "3  245000.0  284500.0  334000.0  \n",
              "4  251000.0  270000.0  320000.0  \n",
              "5  250000.0  282500.0  340000.0  \n",
              "6  250000.0  280000.0  332000.0  \n",
              "7  192500.0  250000.0  320000.0  \n",
              "8  193000.0  247000.0  329000.0  \n",
              "9  205000.0  264000.0  335000.0  "
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Data source:  immo_by_municipality__2010-2019.txt\n",
        "#Source Query location:  Azure Blob Storage\n",
        "\n",
        "path =  'immo_by_municipality__2010-2019.txt'\n",
        "\n",
        "# reads the data from the file - denotes as CSV, it has no header, sets column headers\n",
        "df =  pd.read_csv(path, sep='|') \n",
        "df.head(10)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "### 2.2 Describe Data\n",
        "Data description report - Describe the data that has been acquired including its format, its quantity (for example, the number of records and fields in each table), the identities of the fields and any other surface features which have been discovered. Evaluate whether the data acquired satisfies your requirements."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "gather": {
          "logged": 1671712904260
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['CD_YEAR', 'CD_TYPE_NL', 'CD_TYPE_FR', 'CD_REFNIS', 'CD_REFNIS_NL',\n",
              "       'CD_REFNIS_FR', 'CD_PERIOD', 'CD_CLASS_SURFACE',\n",
              "       'MS_TOTAL_TRANSACTIONS', 'MS_TOTAL_PRICE', 'MS_TOTAL_SURFACE',\n",
              "       'MS_MEAN_PRICE', 'MS_P10', 'MS_P25', 'MS_P50', 'MS_P75', 'MS_P90'],\n",
              "      dtype='object')"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#List the columns\n",
        "df.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "gather": {
          "logged": 1671712904458
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(112160, 17)"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Lets review the number of rows and columns\n",
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "gather": {
          "logged": 1671712904854
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "CD_YEAR                    int64\n",
              "CD_TYPE_NL                object\n",
              "CD_TYPE_FR                object\n",
              "CD_REFNIS                  int64\n",
              "CD_REFNIS_NL              object\n",
              "CD_REFNIS_FR              object\n",
              "CD_PERIOD                 object\n",
              "CD_CLASS_SURFACE          object\n",
              "MS_TOTAL_TRANSACTIONS      int64\n",
              "MS_TOTAL_PRICE           float64\n",
              "MS_TOTAL_SURFACE         float64\n",
              "MS_MEAN_PRICE            float64\n",
              "MS_P10                   float64\n",
              "MS_P25                   float64\n",
              "MS_P50                   float64\n",
              "MS_P75                   float64\n",
              "MS_P90                   float64\n",
              "dtype: object"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#And the data types\n",
        "df.dtypes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "gather": {
          "logged": 1671712905055
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>CD_REFNIS</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <th>MS_P10</th>\n",
              "      <th>MS_P25</th>\n",
              "      <th>MS_P50</th>\n",
              "      <th>MS_P75</th>\n",
              "      <th>MS_P90</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>112160.000000</td>\n",
              "      <td>112160.000000</td>\n",
              "      <td>112160.000000</td>\n",
              "      <td>6.329800e+04</td>\n",
              "      <td>50297.000000</td>\n",
              "      <td>6.329800e+04</td>\n",
              "      <td>63298.000000</td>\n",
              "      <td>6.329800e+04</td>\n",
              "      <td>6.329800e+04</td>\n",
              "      <td>6.329800e+04</td>\n",
              "      <td>6.329800e+04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>2013.296719</td>\n",
              "      <td>46436.990594</td>\n",
              "      <td>29.383425</td>\n",
              "      <td>1.046981e+07</td>\n",
              "      <td>31985.658325</td>\n",
              "      <td>2.006188e+05</td>\n",
              "      <td>118794.353623</td>\n",
              "      <td>1.493028e+05</td>\n",
              "      <td>1.881678e+05</td>\n",
              "      <td>2.367883e+05</td>\n",
              "      <td>2.982554e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.291938</td>\n",
              "      <td>24120.236728</td>\n",
              "      <td>93.230464</td>\n",
              "      <td>2.564732e+07</td>\n",
              "      <td>33419.918658</td>\n",
              "      <td>1.248665e+05</td>\n",
              "      <td>75927.311385</td>\n",
              "      <td>9.022346e+04</td>\n",
              "      <td>1.122684e+05</td>\n",
              "      <td>1.494275e+05</td>\n",
              "      <td>2.089012e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>2010.000000</td>\n",
              "      <td>11001.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.280000e+05</td>\n",
              "      <td>139.000000</td>\n",
              "      <td>1.393958e+01</td>\n",
              "      <td>7.281553</td>\n",
              "      <td>9.756098e+00</td>\n",
              "      <td>1.208897e+01</td>\n",
              "      <td>1.795143e+01</td>\n",
              "      <td>1.900000e+01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>2011.000000</td>\n",
              "      <td>24066.000000</td>\n",
              "      <td>5.000000</td>\n",
              "      <td>2.893500e+06</td>\n",
              "      <td>13223.000000</td>\n",
              "      <td>1.480562e+05</td>\n",
              "      <td>75000.000000</td>\n",
              "      <td>1.037094e+05</td>\n",
              "      <td>1.400000e+05</td>\n",
              "      <td>1.763340e+05</td>\n",
              "      <td>2.150000e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>2013.000000</td>\n",
              "      <td>44049.000000</td>\n",
              "      <td>12.000000</td>\n",
              "      <td>4.995500e+06</td>\n",
              "      <td>21962.000000</td>\n",
              "      <td>1.982737e+05</td>\n",
              "      <td>117000.000000</td>\n",
              "      <td>1.500000e+05</td>\n",
              "      <td>1.875000e+05</td>\n",
              "      <td>2.350000e+05</td>\n",
              "      <td>2.850000e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>2015.000000</td>\n",
              "      <td>63058.000000</td>\n",
              "      <td>27.000000</td>\n",
              "      <td>9.751412e+06</td>\n",
              "      <td>38546.000000</td>\n",
              "      <td>2.571778e+05</td>\n",
              "      <td>160000.000000</td>\n",
              "      <td>1.975000e+05</td>\n",
              "      <td>2.450000e+05</td>\n",
              "      <td>3.000000e+05</td>\n",
              "      <td>3.750000e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2018.000000</td>\n",
              "      <td>93090.000000</td>\n",
              "      <td>5699.000000</td>\n",
              "      <td>1.073798e+09</td>\n",
              "      <td>755681.000000</td>\n",
              "      <td>1.580633e+06</td>\n",
              "      <td>820000.000000</td>\n",
              "      <td>1.005000e+06</td>\n",
              "      <td>1.337500e+06</td>\n",
              "      <td>2.146760e+06</td>\n",
              "      <td>4.391000e+06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             CD_YEAR      CD_REFNIS  MS_TOTAL_TRANSACTIONS  MS_TOTAL_PRICE  \\\n",
              "count  112160.000000  112160.000000          112160.000000    6.329800e+04   \n",
              "mean     2013.296719   46436.990594              29.383425    1.046981e+07   \n",
              "std         2.291938   24120.236728              93.230464    2.564732e+07   \n",
              "min      2010.000000   11001.000000               1.000000    1.280000e+05   \n",
              "25%      2011.000000   24066.000000               5.000000    2.893500e+06   \n",
              "50%      2013.000000   44049.000000              12.000000    4.995500e+06   \n",
              "75%      2015.000000   63058.000000              27.000000    9.751412e+06   \n",
              "max      2018.000000   93090.000000            5699.000000    1.073798e+09   \n",
              "\n",
              "       MS_TOTAL_SURFACE  MS_MEAN_PRICE         MS_P10        MS_P25  \\\n",
              "count      50297.000000   6.329800e+04   63298.000000  6.329800e+04   \n",
              "mean       31985.658325   2.006188e+05  118794.353623  1.493028e+05   \n",
              "std        33419.918658   1.248665e+05   75927.311385  9.022346e+04   \n",
              "min          139.000000   1.393958e+01       7.281553  9.756098e+00   \n",
              "25%        13223.000000   1.480562e+05   75000.000000  1.037094e+05   \n",
              "50%        21962.000000   1.982737e+05  117000.000000  1.500000e+05   \n",
              "75%        38546.000000   2.571778e+05  160000.000000  1.975000e+05   \n",
              "max       755681.000000   1.580633e+06  820000.000000  1.005000e+06   \n",
              "\n",
              "             MS_P50        MS_P75        MS_P90  \n",
              "count  6.329800e+04  6.329800e+04  6.329800e+04  \n",
              "mean   1.881678e+05  2.367883e+05  2.982554e+05  \n",
              "std    1.122684e+05  1.494275e+05  2.089012e+05  \n",
              "min    1.208897e+01  1.795143e+01  1.900000e+01  \n",
              "25%    1.400000e+05  1.763340e+05  2.150000e+05  \n",
              "50%    1.875000e+05  2.350000e+05  2.850000e+05  \n",
              "75%    2.450000e+05  3.000000e+05  3.750000e+05  \n",
              "max    1.337500e+06  2.146760e+06  4.391000e+06  "
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Let Pandas calculate some basic statistics\n",
        "df.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "gather": {
          "logged": 1671712905263
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 112160 entries, 0 to 112159\n",
            "Data columns (total 17 columns):\n",
            " #   Column                 Non-Null Count   Dtype  \n",
            "---  ------                 --------------   -----  \n",
            " 0   CD_YEAR                112160 non-null  int64  \n",
            " 1   CD_TYPE_NL             112160 non-null  object \n",
            " 2   CD_TYPE_FR             112160 non-null  object \n",
            " 3   CD_REFNIS              112160 non-null  int64  \n",
            " 4   CD_REFNIS_NL           112160 non-null  object \n",
            " 5   CD_REFNIS_FR           112160 non-null  object \n",
            " 6   CD_PERIOD              112160 non-null  object \n",
            " 7   CD_CLASS_SURFACE       112160 non-null  object \n",
            " 8   MS_TOTAL_TRANSACTIONS  112160 non-null  int64  \n",
            " 9   MS_TOTAL_PRICE         63298 non-null   float64\n",
            " 10  MS_TOTAL_SURFACE       50297 non-null   float64\n",
            " 11  MS_MEAN_PRICE          63298 non-null   float64\n",
            " 12  MS_P10                 63298 non-null   float64\n",
            " 13  MS_P25                 63298 non-null   float64\n",
            " 14  MS_P50                 63298 non-null   float64\n",
            " 15  MS_P75                 63298 non-null   float64\n",
            " 16  MS_P90                 63298 non-null   float64\n",
            "dtypes: float64(8), int64(3), object(6)\n",
            "memory usage: 14.5+ MB\n"
          ]
        }
      ],
      "source": [
        "# And lets review how many rows are not null for each column together with data types\n",
        "df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "gather": {
          "logged": 1671712905459
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>CD_TYPE_NL</th>\n",
              "      <th>CD_TYPE_FR</th>\n",
              "      <th>CD_REFNIS</th>\n",
              "      <th>CD_REFNIS_NL</th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_PERIOD</th>\n",
              "      <th>CD_CLASS_SURFACE</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <th>MS_P10</th>\n",
              "      <th>MS_P25</th>\n",
              "      <th>MS_P50</th>\n",
              "      <th>MS_P75</th>\n",
              "      <th>MS_P90</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q1</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>25</td>\n",
              "      <td>6207500.0</td>\n",
              "      <td>7676.0</td>\n",
              "      <td>248300.00000</td>\n",
              "      <td>185000.0</td>\n",
              "      <td>225000.0</td>\n",
              "      <td>247000.0</td>\n",
              "      <td>265000.0</td>\n",
              "      <td>320000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q2</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>21</td>\n",
              "      <td>5548500.0</td>\n",
              "      <td>6868.0</td>\n",
              "      <td>264214.28571</td>\n",
              "      <td>200000.0</td>\n",
              "      <td>212000.0</td>\n",
              "      <td>252500.0</td>\n",
              "      <td>285000.0</td>\n",
              "      <td>300000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q3</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>20</td>\n",
              "      <td>5317000.0</td>\n",
              "      <td>5141.0</td>\n",
              "      <td>265850.00000</td>\n",
              "      <td>163750.0</td>\n",
              "      <td>232500.0</td>\n",
              "      <td>252500.0</td>\n",
              "      <td>282500.0</td>\n",
              "      <td>370000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Q4</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>24</td>\n",
              "      <td>6004201.2</td>\n",
              "      <td>9593.0</td>\n",
              "      <td>250175.05000</td>\n",
              "      <td>175000.0</td>\n",
              "      <td>210000.0</td>\n",
              "      <td>245000.0</td>\n",
              "      <td>284500.0</td>\n",
              "      <td>334000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2010</td>\n",
              "      <td>gewone woonhuizen</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>11001</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>S1</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>46</td>\n",
              "      <td>11756000.0</td>\n",
              "      <td>14544.0</td>\n",
              "      <td>255565.21739</td>\n",
              "      <td>197500.0</td>\n",
              "      <td>220000.0</td>\n",
              "      <td>251000.0</td>\n",
              "      <td>270000.0</td>\n",
              "      <td>320000.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   CD_YEAR         CD_TYPE_NL            CD_TYPE_FR  CD_REFNIS CD_REFNIS_NL  \\\n",
              "0     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "1     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "2     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "3     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "4     2010  gewone woonhuizen  maisons d'habitation      11001   AARTSELAAR   \n",
              "\n",
              "  CD_REFNIS_FR CD_PERIOD CD_CLASS_SURFACE  MS_TOTAL_TRANSACTIONS  \\\n",
              "0   AARTSELAAR        Q1   totaal / total                     25   \n",
              "1   AARTSELAAR        Q2   totaal / total                     21   \n",
              "2   AARTSELAAR        Q3   totaal / total                     20   \n",
              "3   AARTSELAAR        Q4   totaal / total                     24   \n",
              "4   AARTSELAAR        S1   totaal / total                     46   \n",
              "\n",
              "   MS_TOTAL_PRICE  MS_TOTAL_SURFACE  MS_MEAN_PRICE    MS_P10    MS_P25  \\\n",
              "0       6207500.0            7676.0   248300.00000  185000.0  225000.0   \n",
              "1       5548500.0            6868.0   264214.28571  200000.0  212000.0   \n",
              "2       5317000.0            5141.0   265850.00000  163750.0  232500.0   \n",
              "3       6004201.2            9593.0   250175.05000  175000.0  210000.0   \n",
              "4      11756000.0           14544.0   255565.21739  197500.0  220000.0   \n",
              "\n",
              "     MS_P50    MS_P75    MS_P90  \n",
              "0  247000.0  265000.0  320000.0  \n",
              "1  252500.0  285000.0  300000.0  \n",
              "2  252500.0  282500.0  370000.0  \n",
              "3  245000.0  284500.0  334000.0  \n",
              "4  251000.0  270000.0  320000.0  "
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Finally lets check the first 5 rows to see how the dataframe looks lie\n",
        "df.head(5)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "gather": {
          "logged": 1671712905635
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[\"maisons d'habitation\" 'villas, bungalows, maisons de campagne'\n",
            " 'appartements, flats, studios' 'terrains � b�tir']\n"
          ]
        }
      ],
      "source": [
        "unique_types = df[\"CD_TYPE_FR\"].unique()\n",
        "print(unique_types)\n",
        "\n",
        "# We are only interested in houses and apparment Types, villas or terrains are not relevant for us"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Data Preparation\n",
        "\n",
        "Data Preparation & Cleaning is an extremely important\n",
        "part of the overall Machine Learning process, one that must\n",
        "be considered before ever looking to build or train a model.\n",
        "A common phrase in Machine Learning is...\n",
        "\"Garbage in...Garbage Out!\"\n",
        "\n",
        "If the data isn't clean or isn’t prepared in an appropriate way,\n",
        "even the fanciest of algorithms or models will struggle to\n",
        "learn.\n",
        "On top of this, ensuring the data is clean can actually be one\n",
        "of the biggest boosters of model performance and accuracy!\n",
        "\n",
        "### 8-Step Checklist\n",
        "\n",
        "Thanks to Data Science Infinity Ebook here: https://cdn.fs.teachablecdn.com/KTJythseReJyqGoIjyQg\n",
        "\n",
        "1. Missing values.\n",
        "2. Duplicate & Low Variation Data.\n",
        "3. Incorrect & Irrelevant Data.\n",
        "4. Categorical Data.\n",
        "5. Outliers. \n",
        "6. Feature Scaling.\n",
        "7. Feature Engineering/Selection.\n",
        "8. Validation Split.\n",
        "\n",
        "Note: Which steps are applicable can depend on the data you’re\n",
        "using, the problem you’re solving, and on the type of model you’re\n",
        "applying! However, in the vast majority of cases you can’t go\n",
        "wrong at least considering these 8 steps!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Missing Values\n",
        "\n",
        "In many cases, your chosen model or algorithm simply\n",
        "won't know how to process missing values - and you will\n",
        "be returned an error.\n",
        "\n",
        "Even if the missing values don't lead to an error you do\n",
        "always want to ensure that you are passing the model\n",
        "the most useful information to learn from - and you\n",
        "should consider whether missing values meet that criteria\n",
        "or not.\n",
        "\n",
        "The two most common approaches for dealing with\n",
        "missing values:\n",
        "\n",
        "**Removal:** Often you will simply remove any observations\n",
        "(rows) where one or missing values are present. You can\n",
        "also remove entire columns if no information is present.\n",
        "\n",
        "**Imputation:** This is where you input or \"impute\"\n",
        "replacement values where they were originally missing.\n",
        "This can be based upon the column mean, median, or\n",
        "most common value or more advanced approaches that\n",
        "take into account other present data points to give an\n",
        "estimation of what the missing value might be!\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "gather": {
          "logged": 1671712905827
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "CD_YEAR                      0\n",
              "CD_TYPE_NL                   0\n",
              "CD_TYPE_FR                   0\n",
              "CD_REFNIS                    0\n",
              "CD_REFNIS_NL                 0\n",
              "CD_REFNIS_FR                 0\n",
              "CD_PERIOD                    0\n",
              "CD_CLASS_SURFACE             0\n",
              "MS_TOTAL_TRANSACTIONS        0\n",
              "MS_TOTAL_PRICE           48862\n",
              "MS_TOTAL_SURFACE         61863\n",
              "MS_MEAN_PRICE            48862\n",
              "MS_P10                   48862\n",
              "MS_P25                   48862\n",
              "MS_P50                   48862\n",
              "MS_P75                   48862\n",
              "MS_P90                   48862\n",
              "dtype: int64"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Lets see how many observations have missing values per feature\n",
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "gather": {
          "logged": 1671712906020
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "#Lets define a reusable function to list the missing values per field and its percentage over all data points.\n",
        "def missing_values_table(df):\n",
        "        mis_val = df.isnull().sum()\n",
        "        mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
        "        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
        "        mis_val_table_ren_columns = mis_val_table.rename(\n",
        "        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n",
        "        mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
        "            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
        "        '% of Total Values', ascending=False).round(1)\n",
        "        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n",
        "            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n",
        "              \" columns that have missing values.\")\n",
        "        return mis_val_table_ren_columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "gather": {
          "logged": 1671712906251
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Your selected dataframe has 17 columns.\n",
            "There are 8 columns that have missing values.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Missing Values</th>\n",
              "      <th>% of Total Values</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <td>61863</td>\n",
              "      <td>55.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <td>48862</td>\n",
              "      <td>43.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <td>48862</td>\n",
              "      <td>43.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_P10</th>\n",
              "      <td>48862</td>\n",
              "      <td>43.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_P25</th>\n",
              "      <td>48862</td>\n",
              "      <td>43.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_P50</th>\n",
              "      <td>48862</td>\n",
              "      <td>43.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_P75</th>\n",
              "      <td>48862</td>\n",
              "      <td>43.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_P90</th>\n",
              "      <td>48862</td>\n",
              "      <td>43.6</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  Missing Values  % of Total Values\n",
              "MS_TOTAL_SURFACE           61863               55.2\n",
              "MS_TOTAL_PRICE             48862               43.6\n",
              "MS_MEAN_PRICE              48862               43.6\n",
              "MS_P10                     48862               43.6\n",
              "MS_P25                     48862               43.6\n",
              "MS_P50                     48862               43.6\n",
              "MS_P75                     48862               43.6\n",
              "MS_P90                     48862               43.6"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Lets execute that function\n",
        "missing_values_table(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "gather": {
          "logged": 1671712906484
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Get the columns with > 50% missing\n",
        "# We could for example remove entire columns depending on the percentage of missing values, but here the code is commented for your information\n",
        "# As another approach will be used, keep reading\n",
        "\n",
        "# missing_df = missing_values_table(df);\n",
        "# missing_columns = list(missing_df[missing_df['% of Total Values'] > 50].index)\n",
        "# print('We will remove %d columns.' % len(missing_columns))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "We will drop the percentile columns, MS_P10, MS_P25, MS_P50, MS_P75, MS_P90 as they are irrelevant for our analysis. We also remove text fields which are not needed, more specifically those ones with NL suffix which are in dutch, we will leave the city name and property type features in french.\n",
        "\n",
        "We have the total price the total surface, which will allow us to get the price per square meter.  However the total surface is null  on  55.2% of the rows.  By manual analysis we can see that apparently this is empty on rows where the period is Q1-Q4 or S1-S2. As we want to do a yearly analysis and not per quarter or per semester, lets drop these rows  to see if we have a cleaner dataset on the yearly basis."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "gather": {
          "logged": 1671712906671
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "#Drop unnecesary fields as explained above\n",
        "df = df.drop(columns=['MS_P10', 'MS_P25', 'MS_P50', 'MS_P75', 'MS_P90','CD_TYPE_NL','CD_REFNIS','CD_REFNIS_NL'])\n",
        "\n",
        "#Drop unnecesary rows as explained above, we are only interested on yearly data points.\n",
        "df = df.drop(df[df['CD_PERIOD'] != 'Y'].index)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "gather": {
          "logged": 1671712906865
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "CD_YEAR                     0\n",
              "CD_TYPE_FR                  0\n",
              "CD_REFNIS_FR                0\n",
              "CD_PERIOD                   0\n",
              "CD_CLASS_SURFACE            0\n",
              "MS_TOTAL_TRANSACTIONS       0\n",
              "MS_TOTAL_PRICE           2798\n",
              "MS_TOTAL_SURFACE         5669\n",
              "MS_MEAN_PRICE            2798\n",
              "dtype: int64"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#And then check again if we have null values.\n",
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "Without a total price and total surface we wont be able to calculate a square meter average price. However when analyzing the dataset manually, I found out that total surface feature  is only available for type House data points (maissons), for appartments this field is always empty. So lets split the dataframe in 2 for easier analysis.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "gather": {
          "logged": 1671712907100
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "#Split the dataframe in one for houses and another for appartements\n",
        "df_houses = df[df['CD_TYPE_FR'] == \"maisons d'habitation\"]\n",
        "#Lets save the dataset as a new file for easier loading and processing later.\n",
        "df_houses.to_csv('df_houses.csv', index=False)\n",
        "\n",
        "df_appartements = df[df['CD_TYPE_FR'] == 'appartements, flats, studios'] \n",
        "#Lets save the dataset as a new file for easier loading and processing later.\n",
        "df_appartements.to_csv('df_houses.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "gather": {
          "logged": 1671712907315
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Your selected dataframe has 9 columns.\n",
            "There are 3 columns that have missing values.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Missing Values</th>\n",
              "      <th>% of Total Values</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <td>61</td>\n",
              "      <td>1.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <td>61</td>\n",
              "      <td>1.3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <td>61</td>\n",
              "      <td>1.3</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  Missing Values  % of Total Values\n",
              "MS_TOTAL_PRICE                61                1.3\n",
              "MS_TOTAL_SURFACE              61                1.3\n",
              "MS_MEAN_PRICE                 61                1.3"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "missing_values_table(df_houses)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "gather": {
          "logged": 1671712907559
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Your selected dataframe has 9 columns.\n",
            "There are 3 columns that have missing values.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Missing Values</th>\n",
              "      <th>% of Total Values</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <td>4271</td>\n",
              "      <td>100.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <td>1400</td>\n",
              "      <td>32.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <td>1400</td>\n",
              "      <td>32.8</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  Missing Values  % of Total Values\n",
              "MS_TOTAL_SURFACE            4271              100.0\n",
              "MS_TOTAL_PRICE              1400               32.8\n",
              "MS_MEAN_PRICE               1400               32.8"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "missing_values_table(df_appartements)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "gather": {
          "logged": 1671712907772
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Your selected dataframe has 9 columns.\n",
            "There are 0 columns that have missing values.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Missing Values</th>\n",
              "      <th>% of Total Values</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [Missing Values, % of Total Values]\n",
              "Index: []"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#For the houses dataframe Lets drop the rows where both total price and total surface are null at the same time\n",
        "df_houses = df_houses.dropna(subset=['MS_TOTAL_PRICE', 'MS_TOTAL_SURFACE'], how='any')\n",
        "\n",
        "#And now lets check the percentage of missing values per feature.\n",
        "missing_values_table(df_houses)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "gather": {
          "logged": 1671712908006
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Your selected dataframe has 8 columns.\n",
            "There are 0 columns that have missing values.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Missing Values</th>\n",
              "      <th>% of Total Values</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [Missing Values, % of Total Values]\n",
              "Index: []"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#For the appartments dataframe Lets drop first the column total surface as all values are empty\n",
        "df_appartements = df_appartements.drop(columns=['MS_TOTAL_SURFACE'])\n",
        "\n",
        "#For the appartments dataframe also Lets drop the rows where both total price and mean price are empty\n",
        "df_appartements = df_appartements.dropna(subset=['MS_TOTAL_PRICE', 'MS_MEAN_PRICE'], how='any')\n",
        "missing_values_table(df_appartements)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "Lets describe again our dataframes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "gather": {
          "logged": 1671712908204
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['CD_YEAR', 'CD_TYPE_FR', 'CD_REFNIS_FR', 'CD_PERIOD',\n",
              "       'CD_CLASS_SURFACE', 'MS_TOTAL_TRANSACTIONS', 'MS_TOTAL_PRICE',\n",
              "       'MS_TOTAL_SURFACE', 'MS_MEAN_PRICE'],\n",
              "      dtype='object')"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_houses.columns\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "gather": {
          "logged": 1671712908402
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['CD_YEAR', 'CD_TYPE_FR', 'CD_REFNIS_FR', 'CD_PERIOD',\n",
              "       'CD_CLASS_SURFACE', 'MS_TOTAL_TRANSACTIONS', 'MS_TOTAL_PRICE',\n",
              "       'MS_MEAN_PRICE'],\n",
              "      dtype='object')"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_appartements.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "gather": {
          "logged": 1671712908603
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(4644, 9)"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_houses.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "gather": {
          "logged": 1671712908804
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2871, 8)"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_appartements.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "gather": {
          "logged": 1671712909002
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "CD_YEAR                    int64\n",
              "CD_TYPE_FR                object\n",
              "CD_REFNIS_FR              object\n",
              "CD_PERIOD                 object\n",
              "CD_CLASS_SURFACE          object\n",
              "MS_TOTAL_TRANSACTIONS      int64\n",
              "MS_TOTAL_PRICE           float64\n",
              "MS_TOTAL_SURFACE         float64\n",
              "MS_MEAN_PRICE            float64\n",
              "dtype: object"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_houses.dtypes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "gather": {
          "logged": 1671712909203
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "CD_YEAR                    int64\n",
              "CD_TYPE_FR                object\n",
              "CD_REFNIS_FR              object\n",
              "CD_PERIOD                 object\n",
              "CD_CLASS_SURFACE          object\n",
              "MS_TOTAL_TRANSACTIONS      int64\n",
              "MS_TOTAL_PRICE           float64\n",
              "MS_MEAN_PRICE            float64\n",
              "dtype: object"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_appartements.dtypes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "gather": {
          "logged": 1671712909404
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>4644.000000</td>\n",
              "      <td>4644.000000</td>\n",
              "      <td>4.644000e+03</td>\n",
              "      <td>4644.000000</td>\n",
              "      <td>4644.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>2013.500000</td>\n",
              "      <td>110.769165</td>\n",
              "      <td>2.206725e+07</td>\n",
              "      <td>61856.257289</td>\n",
              "      <td>197431.718164</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.292662</td>\n",
              "      <td>205.485428</td>\n",
              "      <td>4.452167e+07</td>\n",
              "      <td>57171.806895</td>\n",
              "      <td>61213.966342</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>2010.000000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>1.217500e+06</td>\n",
              "      <td>3206.000000</td>\n",
              "      <td>85874.026749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>2012.000000</td>\n",
              "      <td>36.000000</td>\n",
              "      <td>6.637819e+06</td>\n",
              "      <td>29942.750000</td>\n",
              "      <td>156607.604650</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>2013.500000</td>\n",
              "      <td>62.000000</td>\n",
              "      <td>1.196470e+07</td>\n",
              "      <td>47386.000000</td>\n",
              "      <td>188701.402215</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>2015.000000</td>\n",
              "      <td>119.000000</td>\n",
              "      <td>2.311122e+07</td>\n",
              "      <td>75668.250000</td>\n",
              "      <td>225734.454175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2017.000000</td>\n",
              "      <td>3539.000000</td>\n",
              "      <td>8.275591e+08</td>\n",
              "      <td>755681.000000</td>\n",
              "      <td>690386.601090</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           CD_YEAR  MS_TOTAL_TRANSACTIONS  MS_TOTAL_PRICE  MS_TOTAL_SURFACE  \\\n",
              "count  4644.000000            4644.000000    4.644000e+03       4644.000000   \n",
              "mean   2013.500000             110.769165    2.206725e+07      61856.257289   \n",
              "std       2.292662             205.485428    4.452167e+07      57171.806895   \n",
              "min    2010.000000              10.000000    1.217500e+06       3206.000000   \n",
              "25%    2012.000000              36.000000    6.637819e+06      29942.750000   \n",
              "50%    2013.500000              62.000000    1.196470e+07      47386.000000   \n",
              "75%    2015.000000             119.000000    2.311122e+07      75668.250000   \n",
              "max    2017.000000            3539.000000    8.275591e+08     755681.000000   \n",
              "\n",
              "       MS_MEAN_PRICE  \n",
              "count    4644.000000  \n",
              "mean   197431.718164  \n",
              "std     61213.966342  \n",
              "min     85874.026749  \n",
              "25%    156607.604650  \n",
              "50%    188701.402215  \n",
              "75%    225734.454175  \n",
              "max    690386.601090  "
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_houses.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "gather": {
          "logged": 1671712909603
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2871.000000</td>\n",
              "      <td>2871.000000</td>\n",
              "      <td>2.871000e+03</td>\n",
              "      <td>2871.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>2013.412052</td>\n",
              "      <td>115.660745</td>\n",
              "      <td>2.419658e+07</td>\n",
              "      <td>200165.388617</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.277258</td>\n",
              "      <td>323.841204</td>\n",
              "      <td>6.649911e+07</td>\n",
              "      <td>49274.600797</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>2010.000000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>8.530000e+05</td>\n",
              "      <td>81136.363636</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>2011.000000</td>\n",
              "      <td>19.000000</td>\n",
              "      <td>3.627898e+06</td>\n",
              "      <td>169479.803920</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>2013.000000</td>\n",
              "      <td>38.000000</td>\n",
              "      <td>7.523552e+06</td>\n",
              "      <td>195206.496640</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>2015.000000</td>\n",
              "      <td>90.000000</td>\n",
              "      <td>1.833507e+07</td>\n",
              "      <td>223877.693840</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>2017.000000</td>\n",
              "      <td>5699.000000</td>\n",
              "      <td>1.073798e+09</td>\n",
              "      <td>543859.933210</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           CD_YEAR  MS_TOTAL_TRANSACTIONS  MS_TOTAL_PRICE  MS_MEAN_PRICE\n",
              "count  2871.000000            2871.000000    2.871000e+03    2871.000000\n",
              "mean   2013.412052             115.660745    2.419658e+07  200165.388617\n",
              "std       2.277258             323.841204    6.649911e+07   49274.600797\n",
              "min    2010.000000              10.000000    8.530000e+05   81136.363636\n",
              "25%    2011.000000              19.000000    3.627898e+06  169479.803920\n",
              "50%    2013.000000              38.000000    7.523552e+06  195206.496640\n",
              "75%    2015.000000              90.000000    1.833507e+07  223877.693840\n",
              "max    2017.000000            5699.000000    1.073798e+09  543859.933210"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_appartements.describe()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "gather": {
          "logged": 1671712909788
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 16529 entries, 6 to 110570\n",
            "Data columns (total 9 columns):\n",
            " #   Column                 Non-Null Count  Dtype  \n",
            "---  ------                 --------------  -----  \n",
            " 0   CD_YEAR                16529 non-null  int64  \n",
            " 1   CD_TYPE_FR             16529 non-null  object \n",
            " 2   CD_REFNIS_FR           16529 non-null  object \n",
            " 3   CD_PERIOD              16529 non-null  object \n",
            " 4   CD_CLASS_SURFACE       16529 non-null  object \n",
            " 5   MS_TOTAL_TRANSACTIONS  16529 non-null  int64  \n",
            " 6   MS_TOTAL_PRICE         13731 non-null  float64\n",
            " 7   MS_TOTAL_SURFACE       10860 non-null  float64\n",
            " 8   MS_MEAN_PRICE          13731 non-null  float64\n",
            "dtypes: float64(3), int64(2), object(4)\n",
            "memory usage: 1.3+ MB\n"
          ]
        }
      ],
      "source": [
        "df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "gather": {
          "logged": 1671712909981
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>CD_TYPE_FR</th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_PERIOD</th>\n",
              "      <th>CD_CLASS_SURFACE</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>90</td>\n",
              "      <td>2.307720e+07</td>\n",
              "      <td>29278.0</td>\n",
              "      <td>256413.34667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>ANVERS</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>3179</td>\n",
              "      <td>6.842183e+08</td>\n",
              "      <td>597584.0</td>\n",
              "      <td>215230.68177</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BOECHOUT</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>68</td>\n",
              "      <td>1.986547e+07</td>\n",
              "      <td>113712.0</td>\n",
              "      <td>292139.26471</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BOOM</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>168</td>\n",
              "      <td>2.833533e+07</td>\n",
              "      <td>28676.0</td>\n",
              "      <td>168662.67857</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BORSBEEK</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>60</td>\n",
              "      <td>1.430926e+07</td>\n",
              "      <td>19680.0</td>\n",
              "      <td>238487.74783</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3469</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BILZEN</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>116</td>\n",
              "      <td>2.006851e+07</td>\n",
              "      <td>116844.0</td>\n",
              "      <td>173004.36638</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3476</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>LOOZ</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>78</td>\n",
              "      <td>1.269005e+07</td>\n",
              "      <td>72003.0</td>\n",
              "      <td>162692.94872</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3483</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>HEERS</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>56</td>\n",
              "      <td>8.288100e+06</td>\n",
              "      <td>49266.0</td>\n",
              "      <td>148001.78571</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3486</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>HERSTAPPE</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3493</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>HOESELT</td>\n",
              "      <td>Y</td>\n",
              "      <td>totaal / total</td>\n",
              "      <td>44</td>\n",
              "      <td>7.927599e+06</td>\n",
              "      <td>67687.0</td>\n",
              "      <td>180172.70114</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>500 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      CD_YEAR            CD_TYPE_FR CD_REFNIS_FR CD_PERIOD CD_CLASS_SURFACE  \\\n",
              "6        2010  maisons d'habitation   AARTSELAAR         Y   totaal / total   \n",
              "13       2010  maisons d'habitation       ANVERS         Y   totaal / total   \n",
              "20       2010  maisons d'habitation     BOECHOUT         Y   totaal / total   \n",
              "27       2010  maisons d'habitation         BOOM         Y   totaal / total   \n",
              "34       2010  maisons d'habitation     BORSBEEK         Y   totaal / total   \n",
              "...       ...                   ...          ...       ...              ...   \n",
              "3469     2010  maisons d'habitation       BILZEN         Y   totaal / total   \n",
              "3476     2010  maisons d'habitation         LOOZ         Y   totaal / total   \n",
              "3483     2010  maisons d'habitation        HEERS         Y   totaal / total   \n",
              "3486     2010  maisons d'habitation    HERSTAPPE         Y   totaal / total   \n",
              "3493     2010  maisons d'habitation      HOESELT         Y   totaal / total   \n",
              "\n",
              "      MS_TOTAL_TRANSACTIONS  MS_TOTAL_PRICE  MS_TOTAL_SURFACE  MS_MEAN_PRICE  \n",
              "6                        90    2.307720e+07           29278.0   256413.34667  \n",
              "13                     3179    6.842183e+08          597584.0   215230.68177  \n",
              "20                       68    1.986547e+07          113712.0   292139.26471  \n",
              "27                      168    2.833533e+07           28676.0   168662.67857  \n",
              "34                       60    1.430926e+07           19680.0   238487.74783  \n",
              "...                     ...             ...               ...            ...  \n",
              "3469                    116    2.006851e+07          116844.0   173004.36638  \n",
              "3476                     78    1.269005e+07           72003.0   162692.94872  \n",
              "3483                     56    8.288100e+06           49266.0   148001.78571  \n",
              "3486                      1             NaN               NaN            NaN  \n",
              "3493                     44    7.927599e+06           67687.0   180172.70114  \n",
              "\n",
              "[500 rows x 9 columns]"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.head(500)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Duplicate & Low Variation Data\n",
        "\n",
        "When looking through your data, don't just look for\n",
        "missing values - keep an eye out for duplicate data, or\n",
        "data that has low variation.\n",
        "Duplicate data is most commonly rows of data that are\n",
        "exactly the same across all columns. This duplicate rows\n",
        "do not add anything to the learning process of the model\n",
        "or algorithm, but do add storage & processing overhead.\n",
        "In the vast majority of cases you can remove duplicate\n",
        "rows prior to training your model.\n",
        "Low variation data is where a column in your dataset\n",
        "contain only one (or few) unique value(s).\n",
        "An example: There is a column in a house price dataset\n",
        "called \"property_type\". Every row in this column has the\n",
        "value \"house\". This column won’t add any value to the\n",
        "learning process, so can be removed.\n",
        "\n",
        "In our dataset CD_CLASS_SURFACE has always the same value through the entire dataset, so we remove the entire feature.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "gather": {
          "logged": 1671712910163
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "df_houses = df_houses.drop(columns=['CD_CLASS_SURFACE'])\n",
        "df_appartements = df_appartements.drop(columns=['CD_CLASS_SURFACE'])\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Incorrect & Irrelevant Data\n",
        "\n",
        "Irrelevant data is anything that isn’t related specifically\n",
        "to the problem you’re looking to solve. For example, if\n",
        "you're predicting house prices, but your dataset contains\n",
        "commercial properties as well - these would need to be\n",
        "removed.\n",
        "Incorrect data can be hard to spot! An example could\n",
        "be looking for values that shouldn’t be possible such a\n",
        "negative house price values.\n",
        "For categorical or text variables you should spend time\n",
        "analysing the unique values within a column.\n",
        "For example, you were predicting car prices and had a\n",
        "column in the dataset called \"car_colour\". Upon\n",
        "inspection you find values of \"Orange\", \"orange\", \"orang\"\n",
        "are all present. These need to be rectifed prior to training\n",
        "otherwise you will be limited the potential learning that an\n",
        "take place!\n",
        "Always explore your data thoroughly!\n",
        "\n",
        "Our CD_TYPE_FR has the following unique values:\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "gather": {
          "logged": 1671712910341
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[\"maisons d'habitation\" 'villas, bungalows, maisons de campagne'\n",
            " 'appartements, flats, studios' 'terrains � b�tir']\n"
          ]
        }
      ],
      "source": [
        "unique_types = df[\"CD_TYPE_FR\"].unique()\n",
        "print(unique_types)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "For our analysis, we are interested only in houses, not terrains or villas, etc. During our missing values phase we split the original dataframe in 2, one for houses and other for appartments, so the incorrect & irrelevant data step is already done.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Categorical Data\n",
        "\n",
        "Generally speaking, ML models like being fed numerical\n",
        "data. They are less fond of categorical data!\n",
        "Categorical data is anything that is listed as groups,\n",
        "classes, or text. A simple example would be a column for\n",
        "gender which contains values of either \"Male\" or \"Female\".\n",
        "Your model or algorithm won’t know how to assign some\n",
        "numerical importance to these values, so you often want\n",
        "to turn these groups or classes into numerical values.\n",
        "A common approach is called One Hot Encoding where\n",
        "you create new columns, one for each unique class in your\n",
        "categorical column. You fill these new columns with\n",
        "values of 1 or 0 depending on which is true for each\n",
        "observation.\n",
        "Other encoding techniques you can consider are; Label\n",
        "Encoding, Binary Encoding, Target Encoding, Ordinal\n",
        "Encoding, & Feature Hashing.\n",
        "\n",
        "The only categoricla column is the municipality name, as there are over 500 municipalities in Belgium, we will use Label encoding in this case.\n",
        "\n",
        "In label encoding in Python, we replace the categorical value with a numeric value between 0 and the number of classes minus 1. If the categorical variable value contains 5 distinct classes, we use (0, 1, 2, 3, and 4). We will keep for now the string field and encoded field.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "gather": {
          "logged": 1671712910534
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([  2,  17,  68,  70,  73,  83,  84, 142, 153, 225, 253, 270, 273,\n",
              "       284, 329, 383, 394, 440, 458, 471, 472, 473, 491, 559, 564, 566,\n",
              "       569, 576, 586, 349,  51,  69,  72, 139, 223, 324, 348, 397, 433,\n",
              "       434, 477, 479, 560,  20,  36,  37,  44, 126, 187, 204, 230, 231,\n",
              "       234, 246, 255, 275, 325, 356, 365, 373, 401, 414, 441, 446, 449,\n",
              "       519, 532, 533, 553, 299,  11,  31,  49,  90, 157, 159, 175, 184,\n",
              "       261, 266, 282, 374, 461, 463, 470, 520, 546, 562, 563,  23,  45,\n",
              "        61, 130, 182, 198, 203, 208, 232, 241, 271, 272, 320, 336, 346,\n",
              "       358, 363, 408, 418, 423, 483, 494, 501, 527, 570, 574, 452, 138,\n",
              "       289, 328, 447, 551, 556, 312,   3,   1,  46,  47,  55,  60,  71,\n",
              "        77, 129, 189, 205, 229, 240, 243, 254, 276, 286, 287, 302, 339,\n",
              "       341, 413, 453, 502, 511, 515, 306, 330, 379, 508, 197,  42,  78,\n",
              "        79, 103, 113, 191, 202, 258, 260, 267, 295, 378, 399, 426, 450,\n",
              "       518, 525, 545, 547, 100, 224, 304, 410, 412, 439, 442, 540,  43,\n",
              "        65,  88, 118, 263, 405, 513, 571, 280, 134, 251, 281, 285, 331,\n",
              "       567, 367, 431, 552, 579, 239, 303, 531,  18,  32, 122, 217, 114,\n",
              "       292, 311, 361, 542, 555, 585, 152,  85, 196, 257, 370, 411, 415,\n",
              "       120, 245, 259, 262, 309, 319, 381, 454, 492, 125, 369, 406, 428,\n",
              "       456, 507, 558, 561,  19,   7, 297, 283, 395, 181,   6, 124, 201,\n",
              "       206, 237, 308, 398, 481, 580, 149,  52,  91, 500, 212, 300, 307,\n",
              "       536, 554, 557, 572,  24, 143, 274, 347, 480, 573,   0, 123, 121,\n",
              "       127, 160, 185, 183, 279, 333, 340, 360, 364, 372, 389, 393, 404,\n",
              "       482, 535, 537, 577, 582, 291,  30, 444, 575,  82, 278, 565, 247,\n",
              "       322, 343, 584,  58, 290, 334, 478, 465, 495, 497,  26,  48,  54,\n",
              "        87, 104, 145, 170, 179,  98,  99, 101, 112, 162, 169, 174, 193,\n",
              "       351, 380, 430, 474,   4, 314,  76, 137, 178, 226, 269, 313, 376,\n",
              "       435, 437, 460, 109, 244, 436, 384, 111,  80, 146, 296, 315, 305,\n",
              "       476, 485, 141,  12,  40,  64, 105, 150, 180, 332, 362, 375, 506,\n",
              "       155, 211, 382, 484,  16,  96, 154, 421, 425, 457, 514,  89, 316,\n",
              "       377,   8,  93, 108, 165, 213, 233, 256, 354, 371, 387, 416, 522,\n",
              "       526, 541,  15, 147,  14,  33,  34,  38,  59, 102, 110, 117, 151,\n",
              "       168, 235, 268, 321, 417, 475, 488, 490, 530, 200,  66, 167, 392,\n",
              "       517,  29,  35,  92,  95, 133, 158, 236, 264, 294, 323, 326, 337,\n",
              "       350, 402, 424, 438, 467, 489, 493, 496, 504, 523, 538, 548, 516,\n",
              "        94, 429, 505,  53,  81, 116, 136, 166, 188, 216, 327, 409, 443,\n",
              "       459, 543, 544, 161,  22,  50, 128, 192, 195, 209, 218, 228,  75,\n",
              "       342, 396, 407, 466, 503, 578, 583, 210, 238,  67,  86, 277, 335,\n",
              "       344, 390, 419, 422, 215, 221, 250, 357, 131,   5,  63, 338, 222,\n",
              "       242, 288, 301, 448, 512, 549, 345, 177,  21,  27,  28, 355, 366,\n",
              "        39,  56, 249, 524, 521, 199, 468, 140, 148, 248, 298, 353, 388,\n",
              "       445, 352,  57,  74, 119, 310, 317, 391, 420, 462, 498, 550, 318,\n",
              "       106, 156, 173, 359, 385, 464, 510, 529, 207, 455,  13,  41,  62,\n",
              "       107, 132, 186, 214, 220, 252, 403, 451, 487, 568, 219, 534,  10,\n",
              "        25, 144, 171, 176, 194, 368, 386, 400, 432, 486, 469, 164, 265,\n",
              "       293, 190,  97, 115, 135, 172, 427, 539, 528, 581, 499, 227, 163,\n",
              "       509,   9])"
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "# Import label encoder\n",
        "from sklearn import preprocessing\n",
        "# # label_encoder object knows how to understand word labels.\n",
        "label_encoder = preprocessing.LabelEncoder()\n",
        "  \n",
        "# Encode labels in column 'species'.\n",
        "df_appartements['CD_REFNIS_FR_ENCODED']= label_encoder.fit_transform(df_appartements['CD_REFNIS_FR'])\n",
        "df_houses['CD_REFNIS_FR_ENCODED']= label_encoder.fit_transform(df_houses['CD_REFNIS_FR'])\n",
        "   \n",
        "df_appartements['CD_REFNIS_FR_ENCODED'].unique()\n",
        "df_houses['CD_REFNIS_FR_ENCODED'].unique()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Outliers\n",
        "\n",
        "There is no formal definition for an outlier. You can think\n",
        "of them as any data point that is very different to the\n",
        "majority.\n",
        "How you deal with outliers is dependent on the problem\n",
        "you are solving, and the model you are applying. For\n",
        "example, if your data contained one value that was 1000x\n",
        "any other, this could badly affect a Linear Regression\n",
        "model which tries to generalise a rule across all\n",
        "observations. A Decision Tree would be unaffected\n",
        "however, as it deals with each observation independently.\n",
        "In practice, outliers are commonly isolated using the\n",
        "number of standard deviations from the mean, or a rule\n",
        "based upon the interquartile range.\n",
        "In cases where you want to mitigate the effects of outliers,\n",
        "you may look to simply remove any observations (rows)\n",
        "that contain outlier values in one or more of the columns\n",
        "or you may look to replace their values to reduce their\n",
        "effect.\n",
        "Always remember - just because a value is very high, or\n",
        "very low, that does not mean it is wrong to be included."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "gather": {
          "logged": 1671712910718
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# We wont remove outliers because our dataset is already aggregated at the yearly level,\n",
        "# we dont have each real state transaction to be able to see Outliers."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Feature Scaling\n",
        "\n",
        "Feature Scaling is where you force all the values from a\n",
        "column in your data to exist on the same scale. In certain\n",
        "scenarios it will help the model assess the relationships\n",
        "between variables more fairly, and more accurately.\n",
        "The two most common scaling techniques are:\n",
        "Standardisation: rescales all values to have a mean of 0\n",
        "and standard deviation of 1. In other words, the majority\n",
        "of your values end up between -4 and +4\n",
        "Normalisation: rescales data so that it exists in a range\n",
        "between 0 and 1\n",
        "Feature Scaling is essential for distance-based models\n",
        "such as k-means or k-nearest-neighbours.\n",
        "Feature Scaling is recommended for any algorithms that\n",
        "utilise Gradient Descent such as Linear Regression,\n",
        "Logistic Regression, and Neural Networks.\n",
        "Feature Scaling is not necessary for tree-based\n",
        "algorithms such as Decision Trees & Random Forests."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "To scale the values in  a Pandas DataFrame, you can use the StandardScaler class from scikit-learn library. \n",
        "\n",
        "This will scale the values in the specified columns to have a mean of 0 and a standard deviation of 1. You can also use the MinMaxScaler class to scale the values to a specific range (e.g. 0 to 1).\n",
        "\n",
        "It's important to note that you should only scale the values in a dataset if it is necessary for the analysis or machine learning task you are performing. Scaling the values can be useful if the features in the dataset have different units or scales, as it can help to put them on a common scale and prevent features with larger scales from dominating the analysis. However, scaling the values can also change the interpretation of the features, so you should carefully consider whether it is appropriate for your specific case.\n",
        "\n",
        "The scaled features are saved in a copy of the dataframe to keep the original values, the reason why, its at this point we dont know if the Time Series algorightms we will use require the numeric features to be scaled or not."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "gather": {
          "logged": 1671712910912
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>CD_TYPE_FR</th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_PERIOD</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <th>CD_REFNIS_FR_ENCODED</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.101085</td>\n",
              "      <td>0.022687</td>\n",
              "      <td>-0.569892</td>\n",
              "      <td>0.963636</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>ANVERS</td>\n",
              "      <td>Y</td>\n",
              "      <td>14.933230</td>\n",
              "      <td>14.874160</td>\n",
              "      <td>9.371497</td>\n",
              "      <td>0.290798</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BOECHOUT</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.208160</td>\n",
              "      <td>-0.049459</td>\n",
              "      <td>0.907114</td>\n",
              "      <td>1.547322</td>\n",
              "      <td>68</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BOOM</td>\n",
              "      <td>Y</td>\n",
              "      <td>0.278545</td>\n",
              "      <td>0.140802</td>\n",
              "      <td>-0.580423</td>\n",
              "      <td>-0.470026</td>\n",
              "      <td>70</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BORSBEEK</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.247096</td>\n",
              "      <td>-0.174271</td>\n",
              "      <td>-0.737790</td>\n",
              "      <td>0.670769</td>\n",
              "      <td>73</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103202</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>DOISCHE</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.432044</td>\n",
              "      <td>-0.438145</td>\n",
              "      <td>-0.837133</td>\n",
              "      <td>-1.322700</td>\n",
              "      <td>135</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103209</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>FLORENNES</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.203293</td>\n",
              "      <td>-0.283145</td>\n",
              "      <td>-0.285648</td>\n",
              "      <td>-0.985078</td>\n",
              "      <td>172</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103216</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>PHILIPPEVILLE</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.334703</td>\n",
              "      <td>-0.344021</td>\n",
              "      <td>-0.392705</td>\n",
              "      <td>-0.598913</td>\n",
              "      <td>427</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103223</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>WALCOURT</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.110819</td>\n",
              "      <td>-0.197193</td>\n",
              "      <td>0.347127</td>\n",
              "      <td>-0.758441</td>\n",
              "      <td>539</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103230</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>VIROINVAL</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.276298</td>\n",
              "      <td>-0.382199</td>\n",
              "      <td>-0.532037</td>\n",
              "      <td>-1.696839</td>\n",
              "      <td>528</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4644 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        CD_YEAR            CD_TYPE_FR   CD_REFNIS_FR CD_PERIOD  \\\n",
              "6          2010  maisons d'habitation     AARTSELAAR         Y   \n",
              "13         2010  maisons d'habitation         ANVERS         Y   \n",
              "20         2010  maisons d'habitation       BOECHOUT         Y   \n",
              "27         2010  maisons d'habitation           BOOM         Y   \n",
              "34         2010  maisons d'habitation       BORSBEEK         Y   \n",
              "...         ...                   ...            ...       ...   \n",
              "103202     2017  maisons d'habitation        DOISCHE         Y   \n",
              "103209     2017  maisons d'habitation      FLORENNES         Y   \n",
              "103216     2017  maisons d'habitation  PHILIPPEVILLE         Y   \n",
              "103223     2017  maisons d'habitation       WALCOURT         Y   \n",
              "103230     2017  maisons d'habitation      VIROINVAL         Y   \n",
              "\n",
              "        MS_TOTAL_TRANSACTIONS  MS_TOTAL_PRICE  MS_TOTAL_SURFACE  \\\n",
              "6                   -0.101085        0.022687         -0.569892   \n",
              "13                  14.933230       14.874160          9.371497   \n",
              "20                  -0.208160       -0.049459          0.907114   \n",
              "27                   0.278545        0.140802         -0.580423   \n",
              "34                  -0.247096       -0.174271         -0.737790   \n",
              "...                       ...             ...               ...   \n",
              "103202              -0.432044       -0.438145         -0.837133   \n",
              "103209              -0.203293       -0.283145         -0.285648   \n",
              "103216              -0.334703       -0.344021         -0.392705   \n",
              "103223              -0.110819       -0.197193          0.347127   \n",
              "103230              -0.276298       -0.382199         -0.532037   \n",
              "\n",
              "        MS_MEAN_PRICE  CD_REFNIS_FR_ENCODED  \n",
              "6            0.963636                     2  \n",
              "13           0.290798                    17  \n",
              "20           1.547322                    68  \n",
              "27          -0.470026                    70  \n",
              "34           0.670769                    73  \n",
              "...               ...                   ...  \n",
              "103202      -1.322700                   135  \n",
              "103209      -0.985078                   172  \n",
              "103216      -0.598913                   427  \n",
              "103223      -0.758441                   539  \n",
              "103230      -1.696839                   528  \n",
              "\n",
              "[4644 rows x 9 columns]"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "# Select the columns you want to scale\n",
        "columns_to_scale = ['MS_TOTAL_TRANSACTIONS', 'MS_TOTAL_PRICE','MS_TOTAL_SURFACE','MS_MEAN_PRICE']\n",
        "\n",
        "# Create a scaler object\n",
        "scaler = StandardScaler()\n",
        "\n",
        "# Fit the scaler to the data\n",
        "scaler.fit(df_houses[columns_to_scale])\n",
        "\n",
        "df_houses_scaled = df_houses.copy()\n",
        "\n",
        "# Transform the data\n",
        "df_houses_scaled[columns_to_scale] = scaler.transform(df_houses_scaled[columns_to_scale])\n",
        "df_houses_scaled\n",
        "#df_appartements_scaled "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "gather": {
          "logged": 1671712911155
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>CD_TYPE_FR</th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_PERIOD</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <th>CD_REFNIS_FR_ENCODED</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8004</th>\n",
              "      <td>2010</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.097783</td>\n",
              "      <td>-0.093766</td>\n",
              "      <td>0.277511</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8011</th>\n",
              "      <td>2010</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>ANVERS</td>\n",
              "      <td>Y</td>\n",
              "      <td>14.155511</td>\n",
              "      <td>10.554175</td>\n",
              "      <td>-0.927251</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8018</th>\n",
              "      <td>2010</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>BOECHOUT</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.181172</td>\n",
              "      <td>-0.183208</td>\n",
              "      <td>0.215833</td>\n",
              "      <td>53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8025</th>\n",
              "      <td>2010</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>BOOM</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.057633</td>\n",
              "      <td>-0.086058</td>\n",
              "      <td>-0.196966</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8032</th>\n",
              "      <td>2010</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>BORSBEEK</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.144110</td>\n",
              "      <td>-0.208224</td>\n",
              "      <td>-1.017593</td>\n",
              "      <td>58</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110458</th>\n",
              "      <td>2017</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>EGHEZEE</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.323242</td>\n",
              "      <td>-0.329526</td>\n",
              "      <td>0.157550</td>\n",
              "      <td>108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110486</th>\n",
              "      <td>2017</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>NAMUR</td>\n",
              "      <td>Y</td>\n",
              "      <td>0.649627</td>\n",
              "      <td>0.565073</td>\n",
              "      <td>-0.217114</td>\n",
              "      <td>284</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110498</th>\n",
              "      <td>2017</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>PROFONDEVILLE</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.301622</td>\n",
              "      <td>-0.320685</td>\n",
              "      <td>-0.820914</td>\n",
              "      <td>320</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110512</th>\n",
              "      <td>2017</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>SAMBREVILLE</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.267649</td>\n",
              "      <td>-0.302449</td>\n",
              "      <td>-1.201987</td>\n",
              "      <td>349</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110537</th>\n",
              "      <td>2017</td>\n",
              "      <td>appartements, flats, studios</td>\n",
              "      <td>GEMBLOUX</td>\n",
              "      <td>Y</td>\n",
              "      <td>-0.184260</td>\n",
              "      <td>-0.204043</td>\n",
              "      <td>-0.209875</td>\n",
              "      <td>131</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2871 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        CD_YEAR                    CD_TYPE_FR   CD_REFNIS_FR CD_PERIOD  \\\n",
              "8004       2010  appartements, flats, studios     AARTSELAAR         Y   \n",
              "8011       2010  appartements, flats, studios         ANVERS         Y   \n",
              "8018       2010  appartements, flats, studios       BOECHOUT         Y   \n",
              "8025       2010  appartements, flats, studios           BOOM         Y   \n",
              "8032       2010  appartements, flats, studios       BORSBEEK         Y   \n",
              "...         ...                           ...            ...       ...   \n",
              "110458     2017  appartements, flats, studios        EGHEZEE         Y   \n",
              "110486     2017  appartements, flats, studios          NAMUR         Y   \n",
              "110498     2017  appartements, flats, studios  PROFONDEVILLE         Y   \n",
              "110512     2017  appartements, flats, studios    SAMBREVILLE         Y   \n",
              "110537     2017  appartements, flats, studios       GEMBLOUX         Y   \n",
              "\n",
              "        MS_TOTAL_TRANSACTIONS  MS_TOTAL_PRICE  MS_MEAN_PRICE  \\\n",
              "8004                -0.097783       -0.093766       0.277511   \n",
              "8011                14.155511       10.554175      -0.927251   \n",
              "8018                -0.181172       -0.183208       0.215833   \n",
              "8025                -0.057633       -0.086058      -0.196966   \n",
              "8032                -0.144110       -0.208224      -1.017593   \n",
              "...                       ...             ...            ...   \n",
              "110458              -0.323242       -0.329526       0.157550   \n",
              "110486               0.649627        0.565073      -0.217114   \n",
              "110498              -0.301622       -0.320685      -0.820914   \n",
              "110512              -0.267649       -0.302449      -1.201987   \n",
              "110537              -0.184260       -0.204043      -0.209875   \n",
              "\n",
              "        CD_REFNIS_FR_ENCODED  \n",
              "8004                       2  \n",
              "8011                      11  \n",
              "8018                      53  \n",
              "8025                      55  \n",
              "8032                      58  \n",
              "...                      ...  \n",
              "110458                   108  \n",
              "110486                   284  \n",
              "110498                   320  \n",
              "110512                   349  \n",
              "110537                   131  \n",
              "\n",
              "[2871 rows x 8 columns]"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "# Select the columns you want to scale\n",
        "columns_to_scale = ['MS_TOTAL_TRANSACTIONS', 'MS_TOTAL_PRICE','MS_MEAN_PRICE']\n",
        "\n",
        "# Create a scaler object\n",
        "scaler = StandardScaler()\n",
        "\n",
        "# Fit the scaler to the data\n",
        "scaler.fit(df_appartements[columns_to_scale])\n",
        "\n",
        "df_appartements_scaled = df_appartements.copy()\n",
        "\n",
        "# Transform the data\n",
        "df_appartements_scaled[columns_to_scale] = scaler.transform(df_appartements_scaled[columns_to_scale])\n",
        "df_appartements_scaled\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Feature Engineering & Selection\n",
        "\n",
        "Feature Engineering is the process of using further\n",
        "knowledge to supplement or transform the original\n",
        "feature set.\n",
        "The key to good Feature Engineering is to create or refine\n",
        "features that the algorithm or model can understand\n",
        "better or that it will find more useful than the raw features\n",
        "for solving the particular problem at hand.\n",
        "Feature Selection is where you only keep a subset of the\n",
        "most informative variables. This can be done using\n",
        "human intuition, or dynamically based upon statistical\n",
        "analysis.\n",
        "A smaller feature set can lead to improved model\n",
        "accuracy through reduced noise. It can mean a lower\n",
        "computational cost, and improved processing speed. It\n",
        "can also make your models easier to understand &\n",
        "explain to stakeholders & customers!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "For this problem lets add the average price per square meter to the houses dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "gather": {
          "logged": 1671712911443
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>CD_TYPE_FR</th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_PERIOD</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>MS_TOTAL_PRICE</th>\n",
              "      <th>MS_TOTAL_SURFACE</th>\n",
              "      <th>MS_MEAN_PRICE</th>\n",
              "      <th>CD_REFNIS_FR_ENCODED</th>\n",
              "      <th>price_per_surface</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>Y</td>\n",
              "      <td>90</td>\n",
              "      <td>2.307720e+07</td>\n",
              "      <td>29278.00</td>\n",
              "      <td>256413.346670</td>\n",
              "      <td>2</td>\n",
              "      <td>788.209618</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>ANVERS</td>\n",
              "      <td>Y</td>\n",
              "      <td>3179</td>\n",
              "      <td>6.842183e+08</td>\n",
              "      <td>597584.00</td>\n",
              "      <td>215230.681770</td>\n",
              "      <td>17</td>\n",
              "      <td>1144.974326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BOECHOUT</td>\n",
              "      <td>Y</td>\n",
              "      <td>68</td>\n",
              "      <td>1.986547e+07</td>\n",
              "      <td>113712.00</td>\n",
              "      <td>292139.264710</td>\n",
              "      <td>68</td>\n",
              "      <td>174.699856</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BOOM</td>\n",
              "      <td>Y</td>\n",
              "      <td>168</td>\n",
              "      <td>2.833533e+07</td>\n",
              "      <td>28676.00</td>\n",
              "      <td>168662.678570</td>\n",
              "      <td>70</td>\n",
              "      <td>988.120031</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>2010</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>BORSBEEK</td>\n",
              "      <td>Y</td>\n",
              "      <td>60</td>\n",
              "      <td>1.430926e+07</td>\n",
              "      <td>19680.00</td>\n",
              "      <td>238487.747830</td>\n",
              "      <td>73</td>\n",
              "      <td>727.096792</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103202</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>DOISCHE</td>\n",
              "      <td>Y</td>\n",
              "      <td>22</td>\n",
              "      <td>2.562400e+06</td>\n",
              "      <td>14001.00</td>\n",
              "      <td>116472.727270</td>\n",
              "      <td>135</td>\n",
              "      <td>183.015499</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103209</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>FLORENNES</td>\n",
              "      <td>Y</td>\n",
              "      <td>69</td>\n",
              "      <td>9.462500e+06</td>\n",
              "      <td>45527.00</td>\n",
              "      <td>137137.681160</td>\n",
              "      <td>172</td>\n",
              "      <td>207.843697</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103216</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>PHILIPPEVILLE</td>\n",
              "      <td>Y</td>\n",
              "      <td>42</td>\n",
              "      <td>6.752500e+06</td>\n",
              "      <td>39407.04</td>\n",
              "      <td>160773.809520</td>\n",
              "      <td>427</td>\n",
              "      <td>171.352631</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103223</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>WALCOURT</td>\n",
              "      <td>Y</td>\n",
              "      <td>88</td>\n",
              "      <td>1.328884e+07</td>\n",
              "      <td>81700.00</td>\n",
              "      <td>151009.545450</td>\n",
              "      <td>539</td>\n",
              "      <td>162.654100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>103230</th>\n",
              "      <td>2017</td>\n",
              "      <td>maisons d'habitation</td>\n",
              "      <td>VIROINVAL</td>\n",
              "      <td>Y</td>\n",
              "      <td>54</td>\n",
              "      <td>5.052925e+06</td>\n",
              "      <td>31442.00</td>\n",
              "      <td>93572.685185</td>\n",
              "      <td>528</td>\n",
              "      <td>160.706221</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4644 rows × 10 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "        CD_YEAR            CD_TYPE_FR   CD_REFNIS_FR CD_PERIOD  \\\n",
              "6          2010  maisons d'habitation     AARTSELAAR         Y   \n",
              "13         2010  maisons d'habitation         ANVERS         Y   \n",
              "20         2010  maisons d'habitation       BOECHOUT         Y   \n",
              "27         2010  maisons d'habitation           BOOM         Y   \n",
              "34         2010  maisons d'habitation       BORSBEEK         Y   \n",
              "...         ...                   ...            ...       ...   \n",
              "103202     2017  maisons d'habitation        DOISCHE         Y   \n",
              "103209     2017  maisons d'habitation      FLORENNES         Y   \n",
              "103216     2017  maisons d'habitation  PHILIPPEVILLE         Y   \n",
              "103223     2017  maisons d'habitation       WALCOURT         Y   \n",
              "103230     2017  maisons d'habitation      VIROINVAL         Y   \n",
              "\n",
              "        MS_TOTAL_TRANSACTIONS  MS_TOTAL_PRICE  MS_TOTAL_SURFACE  \\\n",
              "6                          90    2.307720e+07          29278.00   \n",
              "13                       3179    6.842183e+08         597584.00   \n",
              "20                         68    1.986547e+07         113712.00   \n",
              "27                        168    2.833533e+07          28676.00   \n",
              "34                         60    1.430926e+07          19680.00   \n",
              "...                       ...             ...               ...   \n",
              "103202                     22    2.562400e+06          14001.00   \n",
              "103209                     69    9.462500e+06          45527.00   \n",
              "103216                     42    6.752500e+06          39407.04   \n",
              "103223                     88    1.328884e+07          81700.00   \n",
              "103230                     54    5.052925e+06          31442.00   \n",
              "\n",
              "        MS_MEAN_PRICE  CD_REFNIS_FR_ENCODED  price_per_surface  \n",
              "6       256413.346670                     2         788.209618  \n",
              "13      215230.681770                    17        1144.974326  \n",
              "20      292139.264710                    68         174.699856  \n",
              "27      168662.678570                    70         988.120031  \n",
              "34      238487.747830                    73         727.096792  \n",
              "...               ...                   ...                ...  \n",
              "103202  116472.727270                   135         183.015499  \n",
              "103209  137137.681160                   172         207.843697  \n",
              "103216  160773.809520                   427         171.352631  \n",
              "103223  151009.545450                   539         162.654100  \n",
              "103230   93572.685185                   528         160.706221  \n",
              "\n",
              "[4644 rows x 10 columns]"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "def price_per_surface(row):\n",
        "  return row['MS_TOTAL_PRICE'] / row['MS_TOTAL_SURFACE']\n",
        "\n",
        "# Add the new column to the dataframe\n",
        "df_houses = df_houses.assign(price_per_surface=df.apply(price_per_surface, axis=1))\n",
        "df_houses"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "What about if we want to see the % increase of transactions per year per city, and do the same for price per square meter.\n",
        "\n",
        "In this case we need to take year 2010 as the basis, and then do the calculation per row per year per city."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "gather": {
          "logged": 1671713234161
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Group the data by city and year\n",
        "df_grouped_houses_totalTransactions = df_houses.groupby(['CD_REFNIS_FR', 'CD_YEAR'])['MS_TOTAL_TRANSACTIONS'].sum().reset_index()\n",
        "df_grouped_houses_price_per_surface = df_houses.groupby(['CD_REFNIS_FR', 'CD_YEAR'])['price_per_surface'].sum().reset_index()\n",
        "\n",
        "# Calculate the percentage change in transactions per year for each city\n",
        "df_grouped_houses_totalTransactions['pct_change_transactions'] = df_grouped_houses_totalTransactions.groupby('CD_REFNIS_FR')['MS_TOTAL_TRANSACTIONS'].pct_change()*100\n",
        "df_grouped_houses_price_per_surface['pct_change_pricepersqm'] = df_grouped_houses_price_per_surface.groupby('CD_REFNIS_FR')['price_per_surface'].pct_change()*100\n",
        "\n",
        "# Filter by City to see the % change per year\n",
        "#df_grouped_houses_anvers = df_grouped_houses.loc[df_grouped_houses['CD_REFNIS_FR'] == 'ANVERS']\n",
        "#df_grouped_houses = df_grouped_houses.sort_values('CD_YEAR', ascending=True)\n",
        "#df_grouped_houses_anvers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "gather": {
          "logged": 1671714017142
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>price_per_surface</th>\n",
              "      <th>pct_change_pricepersqm</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2010</td>\n",
              "      <td>292.528065</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2011</td>\n",
              "      <td>238.854558</td>\n",
              "      <td>-18.348157</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2012</td>\n",
              "      <td>147.676510</td>\n",
              "      <td>-38.173041</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2013</td>\n",
              "      <td>193.593618</td>\n",
              "      <td>31.093035</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2014</td>\n",
              "      <td>248.690774</td>\n",
              "      <td>28.460213</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2015</td>\n",
              "      <td>327.467359</td>\n",
              "      <td>31.676521</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2016</td>\n",
              "      <td>305.742010</td>\n",
              "      <td>-6.634356</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2017</td>\n",
              "      <td>468.156627</td>\n",
              "      <td>53.121459</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2010</td>\n",
              "      <td>173.703680</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2011</td>\n",
              "      <td>201.074901</td>\n",
              "      <td>15.757422</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2012</td>\n",
              "      <td>176.486099</td>\n",
              "      <td>-12.228678</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2013</td>\n",
              "      <td>193.847443</td>\n",
              "      <td>9.837230</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2014</td>\n",
              "      <td>202.116502</td>\n",
              "      <td>4.265756</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2015</td>\n",
              "      <td>183.997991</td>\n",
              "      <td>-8.964390</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2016</td>\n",
              "      <td>236.458317</td>\n",
              "      <td>28.511358</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2017</td>\n",
              "      <td>276.695810</td>\n",
              "      <td>17.016738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2010</td>\n",
              "      <td>788.209618</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2011</td>\n",
              "      <td>878.700706</td>\n",
              "      <td>11.480587</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2012</td>\n",
              "      <td>623.582440</td>\n",
              "      <td>-29.033579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2013</td>\n",
              "      <td>849.785350</td>\n",
              "      <td>36.274740</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2014</td>\n",
              "      <td>772.641082</td>\n",
              "      <td>-9.078089</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2015</td>\n",
              "      <td>846.570191</td>\n",
              "      <td>9.568364</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2016</td>\n",
              "      <td>955.191323</td>\n",
              "      <td>12.830730</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2017</td>\n",
              "      <td>937.840567</td>\n",
              "      <td>-1.816469</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2010</td>\n",
              "      <td>345.928021</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2011</td>\n",
              "      <td>379.867416</td>\n",
              "      <td>9.811115</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2012</td>\n",
              "      <td>365.758508</td>\n",
              "      <td>-3.714166</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2013</td>\n",
              "      <td>390.472345</td>\n",
              "      <td>6.756873</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2014</td>\n",
              "      <td>213.630255</td>\n",
              "      <td>-45.289274</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2015</td>\n",
              "      <td>362.630299</td>\n",
              "      <td>69.746696</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2016</td>\n",
              "      <td>466.280851</td>\n",
              "      <td>28.582982</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2017</td>\n",
              "      <td>609.738644</td>\n",
              "      <td>30.766392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2010</td>\n",
              "      <td>202.224760</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2011</td>\n",
              "      <td>221.121605</td>\n",
              "      <td>9.344477</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2012</td>\n",
              "      <td>239.538890</td>\n",
              "      <td>8.329030</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2013</td>\n",
              "      <td>225.664047</td>\n",
              "      <td>-5.792313</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2014</td>\n",
              "      <td>199.000213</td>\n",
              "      <td>-11.815721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2015</td>\n",
              "      <td>263.067455</td>\n",
              "      <td>32.194559</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2016</td>\n",
              "      <td>214.632766</td>\n",
              "      <td>-18.411509</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2017</td>\n",
              "      <td>203.303090</td>\n",
              "      <td>-5.278633</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2010</td>\n",
              "      <td>182.980757</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2011</td>\n",
              "      <td>124.337789</td>\n",
              "      <td>-32.048708</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2012</td>\n",
              "      <td>154.642079</td>\n",
              "      <td>24.372550</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2013</td>\n",
              "      <td>119.200549</td>\n",
              "      <td>-22.918425</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2014</td>\n",
              "      <td>221.172857</td>\n",
              "      <td>85.546844</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2015</td>\n",
              "      <td>180.365980</td>\n",
              "      <td>-18.450219</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2016</td>\n",
              "      <td>247.547174</td>\n",
              "      <td>37.247154</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2017</td>\n",
              "      <td>282.550456</td>\n",
              "      <td>14.140045</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2010</td>\n",
              "      <td>407.869516</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2011</td>\n",
              "      <td>515.084632</td>\n",
              "      <td>26.286622</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2012</td>\n",
              "      <td>477.041720</td>\n",
              "      <td>-7.385760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2013</td>\n",
              "      <td>503.069966</td>\n",
              "      <td>5.456178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2014</td>\n",
              "      <td>469.519069</td>\n",
              "      <td>-6.669231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2015</td>\n",
              "      <td>593.658164</td>\n",
              "      <td>26.439628</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2016</td>\n",
              "      <td>606.654397</td>\n",
              "      <td>2.189178</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2017</td>\n",
              "      <td>604.667963</td>\n",
              "      <td>-0.327441</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2010</td>\n",
              "      <td>152.080298</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2011</td>\n",
              "      <td>186.999145</td>\n",
              "      <td>22.960796</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2012</td>\n",
              "      <td>119.556155</td>\n",
              "      <td>-36.065935</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2013</td>\n",
              "      <td>166.508804</td>\n",
              "      <td>39.272465</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2014</td>\n",
              "      <td>128.617517</td>\n",
              "      <td>-22.756327</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2015</td>\n",
              "      <td>232.684236</td>\n",
              "      <td>80.911778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2016</td>\n",
              "      <td>138.779337</td>\n",
              "      <td>-40.357224</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2017</td>\n",
              "      <td>200.258795</td>\n",
              "      <td>44.300153</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2010</td>\n",
              "      <td>149.540333</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2011</td>\n",
              "      <td>164.583874</td>\n",
              "      <td>10.059855</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2012</td>\n",
              "      <td>159.718068</td>\n",
              "      <td>-2.956429</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2013</td>\n",
              "      <td>156.324079</td>\n",
              "      <td>-2.124987</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2014</td>\n",
              "      <td>185.875988</td>\n",
              "      <td>18.904259</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2015</td>\n",
              "      <td>208.708156</td>\n",
              "      <td>12.283549</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2016</td>\n",
              "      <td>232.104617</td>\n",
              "      <td>11.210133</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2017</td>\n",
              "      <td>249.597523</td>\n",
              "      <td>7.536647</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2014</td>\n",
              "      <td>70.850202</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2015</td>\n",
              "      <td>137.795969</td>\n",
              "      <td>94.489167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2016</td>\n",
              "      <td>181.109865</td>\n",
              "      <td>31.433356</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2017</td>\n",
              "      <td>211.224772</td>\n",
              "      <td>16.627977</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2010</td>\n",
              "      <td>174.527260</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2011</td>\n",
              "      <td>169.128037</td>\n",
              "      <td>-3.093627</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2012</td>\n",
              "      <td>193.480469</td>\n",
              "      <td>14.398814</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2013</td>\n",
              "      <td>189.832250</td>\n",
              "      <td>-1.885575</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2014</td>\n",
              "      <td>188.084504</td>\n",
              "      <td>-0.920679</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2015</td>\n",
              "      <td>203.774941</td>\n",
              "      <td>8.342228</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2016</td>\n",
              "      <td>173.658778</td>\n",
              "      <td>-14.779130</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2017</td>\n",
              "      <td>178.346558</td>\n",
              "      <td>2.699420</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2010</td>\n",
              "      <td>1210.668153</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2011</td>\n",
              "      <td>1271.615927</td>\n",
              "      <td>5.034226</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2012</td>\n",
              "      <td>1418.780035</td>\n",
              "      <td>11.573000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2013</td>\n",
              "      <td>1508.864185</td>\n",
              "      <td>6.349409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2014</td>\n",
              "      <td>1391.208344</td>\n",
              "      <td>-7.797643</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2015</td>\n",
              "      <td>1608.841651</td>\n",
              "      <td>15.643473</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2016</td>\n",
              "      <td>1295.234055</td>\n",
              "      <td>-19.492757</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2017</td>\n",
              "      <td>1404.162706</td>\n",
              "      <td>8.409959</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2010</td>\n",
              "      <td>181.505814</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2011</td>\n",
              "      <td>193.559035</td>\n",
              "      <td>6.640680</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2012</td>\n",
              "      <td>165.152924</td>\n",
              "      <td>-14.675683</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2013</td>\n",
              "      <td>171.219024</td>\n",
              "      <td>3.673020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2014</td>\n",
              "      <td>196.278246</td>\n",
              "      <td>14.635770</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2015</td>\n",
              "      <td>211.209921</td>\n",
              "      <td>7.607402</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2016</td>\n",
              "      <td>223.030350</td>\n",
              "      <td>5.596532</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2017</td>\n",
              "      <td>231.411063</td>\n",
              "      <td>3.757656</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      CD_REFNIS_FR  CD_YEAR  price_per_surface  pct_change_pricepersqm\n",
              "0           AALTER     2010         292.528065                0.000000\n",
              "1           AALTER     2011         238.854558              -18.348157\n",
              "2           AALTER     2012         147.676510              -38.173041\n",
              "3           AALTER     2013         193.593618               31.093035\n",
              "4           AALTER     2014         248.690774               28.460213\n",
              "5           AALTER     2015         327.467359               31.676521\n",
              "6           AALTER     2016         305.742010               -6.634356\n",
              "7           AALTER     2017         468.156627               53.121459\n",
              "8         AARSCHOT     2010         173.703680                0.000000\n",
              "9         AARSCHOT     2011         201.074901               15.757422\n",
              "10        AARSCHOT     2012         176.486099              -12.228678\n",
              "11        AARSCHOT     2013         193.847443                9.837230\n",
              "12        AARSCHOT     2014         202.116502                4.265756\n",
              "13        AARSCHOT     2015         183.997991               -8.964390\n",
              "14        AARSCHOT     2016         236.458317               28.511358\n",
              "15        AARSCHOT     2017         276.695810               17.016738\n",
              "16      AARTSELAAR     2010         788.209618                0.000000\n",
              "17      AARTSELAAR     2011         878.700706               11.480587\n",
              "18      AARTSELAAR     2012         623.582440              -29.033579\n",
              "19      AARTSELAAR     2013         849.785350               36.274740\n",
              "20      AARTSELAAR     2014         772.641082               -9.078089\n",
              "21      AARTSELAAR     2015         846.570191                9.568364\n",
              "22      AARTSELAAR     2016         955.191323               12.830730\n",
              "23      AARTSELAAR     2017         937.840567               -1.816469\n",
              "24        AFFLIGEM     2010         345.928021                0.000000\n",
              "25        AFFLIGEM     2011         379.867416                9.811115\n",
              "26        AFFLIGEM     2012         365.758508               -3.714166\n",
              "27        AFFLIGEM     2013         390.472345                6.756873\n",
              "28        AFFLIGEM     2014         213.630255              -45.289274\n",
              "29        AFFLIGEM     2015         362.630299               69.746696\n",
              "30        AFFLIGEM     2016         466.280851               28.582982\n",
              "31        AFFLIGEM     2017         609.738644               30.766392\n",
              "32  AISEAU-PRESLES     2010         202.224760                0.000000\n",
              "33  AISEAU-PRESLES     2011         221.121605                9.344477\n",
              "34  AISEAU-PRESLES     2012         239.538890                8.329030\n",
              "35  AISEAU-PRESLES     2013         225.664047               -5.792313\n",
              "36  AISEAU-PRESLES     2014         199.000213              -11.815721\n",
              "37  AISEAU-PRESLES     2015         263.067455               32.194559\n",
              "38  AISEAU-PRESLES     2016         214.632766              -18.411509\n",
              "39  AISEAU-PRESLES     2017         203.303090               -5.278633\n",
              "40           ALKEN     2010         182.980757                0.000000\n",
              "41           ALKEN     2011         124.337789              -32.048708\n",
              "42           ALKEN     2012         154.642079               24.372550\n",
              "43           ALKEN     2013         119.200549              -22.918425\n",
              "44           ALKEN     2014         221.172857               85.546844\n",
              "45           ALKEN     2015         180.365980              -18.450219\n",
              "46           ALKEN     2016         247.547174               37.247154\n",
              "47           ALKEN     2017         282.550456               14.140045\n",
              "48           ALOST     2010         407.869516                0.000000\n",
              "49           ALOST     2011         515.084632               26.286622\n",
              "50           ALOST     2012         477.041720               -7.385760\n",
              "51           ALOST     2013         503.069966                5.456178\n",
              "52           ALOST     2014         469.519069               -6.669231\n",
              "53           ALOST     2015         593.658164               26.439628\n",
              "54           ALOST     2016         606.654397                2.189178\n",
              "55           ALOST     2017         604.667963               -0.327441\n",
              "56      ALVERINGEM     2010         152.080298                0.000000\n",
              "57      ALVERINGEM     2011         186.999145               22.960796\n",
              "58      ALVERINGEM     2012         119.556155              -36.065935\n",
              "59      ALVERINGEM     2013         166.508804               39.272465\n",
              "60      ALVERINGEM     2014         128.617517              -22.756327\n",
              "61      ALVERINGEM     2015         232.684236               80.911778\n",
              "62      ALVERINGEM     2016         138.779337              -40.357224\n",
              "63      ALVERINGEM     2017         200.258795               44.300153\n",
              "64            AMAY     2010         149.540333                0.000000\n",
              "65            AMAY     2011         164.583874               10.059855\n",
              "66            AMAY     2012         159.718068               -2.956429\n",
              "67            AMAY     2013         156.324079               -2.124987\n",
              "68            AMAY     2014         185.875988               18.904259\n",
              "69            AMAY     2015         208.708156               12.283549\n",
              "70            AMAY     2016         232.104617               11.210133\n",
              "71            AMAY     2017         249.597523                7.536647\n",
              "72         AMBLEVE     2014          70.850202                0.000000\n",
              "73         AMBLEVE     2015         137.795969               94.489167\n",
              "74         AMBLEVE     2016         181.109865               31.433356\n",
              "75         AMBLEVE     2017         211.224772               16.627977\n",
              "76         ANDENNE     2010         174.527260                0.000000\n",
              "77         ANDENNE     2011         169.128037               -3.093627\n",
              "78         ANDENNE     2012         193.480469               14.398814\n",
              "79         ANDENNE     2013         189.832250               -1.885575\n",
              "80         ANDENNE     2014         188.084504               -0.920679\n",
              "81         ANDENNE     2015         203.774941                8.342228\n",
              "82         ANDENNE     2016         173.658778              -14.779130\n",
              "83         ANDENNE     2017         178.346558                2.699420\n",
              "84      ANDERLECHT     2010        1210.668153                0.000000\n",
              "85      ANDERLECHT     2011        1271.615927                5.034226\n",
              "86      ANDERLECHT     2012        1418.780035               11.573000\n",
              "87      ANDERLECHT     2013        1508.864185                6.349409\n",
              "88      ANDERLECHT     2014        1391.208344               -7.797643\n",
              "89      ANDERLECHT     2015        1608.841651               15.643473\n",
              "90      ANDERLECHT     2016        1295.234055              -19.492757\n",
              "91      ANDERLECHT     2017        1404.162706                8.409959\n",
              "92       ANDERLUES     2010         181.505814                0.000000\n",
              "93       ANDERLUES     2011         193.559035                6.640680\n",
              "94       ANDERLUES     2012         165.152924              -14.675683\n",
              "95       ANDERLUES     2013         171.219024                3.673020\n",
              "96       ANDERLUES     2014         196.278246               14.635770\n",
              "97       ANDERLUES     2015         211.209921                7.607402\n",
              "98       ANDERLUES     2016         223.030350                5.596532\n",
              "99       ANDERLUES     2017         231.411063                3.757656"
            ]
          },
          "execution_count": 52,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pd.set_option('display.max_rows', 100)\n",
        "df_grouped_houses_price_per_surface = df_grouped_houses_price_per_surface.fillna(0)\n",
        "\n",
        "df_grouped_houses_price_per_surface.head(100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "gather": {
          "logged": 1671714034322
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CD_REFNIS_FR</th>\n",
              "      <th>CD_YEAR</th>\n",
              "      <th>MS_TOTAL_TRANSACTIONS</th>\n",
              "      <th>pct_change_transactions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2010</td>\n",
              "      <td>70</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2011</td>\n",
              "      <td>69</td>\n",
              "      <td>-1.428571</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2012</td>\n",
              "      <td>60</td>\n",
              "      <td>-13.043478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2013</td>\n",
              "      <td>100</td>\n",
              "      <td>66.666667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2014</td>\n",
              "      <td>87</td>\n",
              "      <td>-13.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2015</td>\n",
              "      <td>57</td>\n",
              "      <td>-34.482759</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2016</td>\n",
              "      <td>73</td>\n",
              "      <td>28.070175</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>AALTER</td>\n",
              "      <td>2017</td>\n",
              "      <td>85</td>\n",
              "      <td>16.438356</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2010</td>\n",
              "      <td>138</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2011</td>\n",
              "      <td>158</td>\n",
              "      <td>14.492754</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2012</td>\n",
              "      <td>160</td>\n",
              "      <td>1.265823</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2013</td>\n",
              "      <td>168</td>\n",
              "      <td>5.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2014</td>\n",
              "      <td>198</td>\n",
              "      <td>17.857143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2015</td>\n",
              "      <td>128</td>\n",
              "      <td>-35.353535</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2016</td>\n",
              "      <td>145</td>\n",
              "      <td>13.281250</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>AARSCHOT</td>\n",
              "      <td>2017</td>\n",
              "      <td>167</td>\n",
              "      <td>15.172414</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2010</td>\n",
              "      <td>90</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2011</td>\n",
              "      <td>79</td>\n",
              "      <td>-12.222222</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2012</td>\n",
              "      <td>83</td>\n",
              "      <td>5.063291</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2013</td>\n",
              "      <td>85</td>\n",
              "      <td>2.409639</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2014</td>\n",
              "      <td>97</td>\n",
              "      <td>14.117647</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2015</td>\n",
              "      <td>76</td>\n",
              "      <td>-21.649485</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2016</td>\n",
              "      <td>78</td>\n",
              "      <td>2.631579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>AARTSELAAR</td>\n",
              "      <td>2017</td>\n",
              "      <td>108</td>\n",
              "      <td>38.461538</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2010</td>\n",
              "      <td>93</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2011</td>\n",
              "      <td>86</td>\n",
              "      <td>-7.526882</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2012</td>\n",
              "      <td>98</td>\n",
              "      <td>13.953488</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2013</td>\n",
              "      <td>89</td>\n",
              "      <td>-9.183673</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2014</td>\n",
              "      <td>85</td>\n",
              "      <td>-4.494382</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2015</td>\n",
              "      <td>66</td>\n",
              "      <td>-22.352941</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2016</td>\n",
              "      <td>97</td>\n",
              "      <td>46.969697</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>AFFLIGEM</td>\n",
              "      <td>2017</td>\n",
              "      <td>81</td>\n",
              "      <td>-16.494845</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2010</td>\n",
              "      <td>90</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2011</td>\n",
              "      <td>76</td>\n",
              "      <td>-15.555556</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2012</td>\n",
              "      <td>78</td>\n",
              "      <td>2.631579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2013</td>\n",
              "      <td>61</td>\n",
              "      <td>-21.794872</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2014</td>\n",
              "      <td>54</td>\n",
              "      <td>-11.475410</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2015</td>\n",
              "      <td>75</td>\n",
              "      <td>38.888889</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2016</td>\n",
              "      <td>75</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>AISEAU-PRESLES</td>\n",
              "      <td>2017</td>\n",
              "      <td>75</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2010</td>\n",
              "      <td>39</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2011</td>\n",
              "      <td>38</td>\n",
              "      <td>-2.564103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2012</td>\n",
              "      <td>45</td>\n",
              "      <td>18.421053</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2013</td>\n",
              "      <td>36</td>\n",
              "      <td>-20.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2014</td>\n",
              "      <td>34</td>\n",
              "      <td>-5.555556</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2015</td>\n",
              "      <td>40</td>\n",
              "      <td>17.647059</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2016</td>\n",
              "      <td>41</td>\n",
              "      <td>2.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>ALKEN</td>\n",
              "      <td>2017</td>\n",
              "      <td>54</td>\n",
              "      <td>31.707317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2010</td>\n",
              "      <td>738</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2011</td>\n",
              "      <td>813</td>\n",
              "      <td>10.162602</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2012</td>\n",
              "      <td>766</td>\n",
              "      <td>-5.781058</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2013</td>\n",
              "      <td>736</td>\n",
              "      <td>-3.916449</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2014</td>\n",
              "      <td>855</td>\n",
              "      <td>16.168478</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2015</td>\n",
              "      <td>633</td>\n",
              "      <td>-25.964912</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2016</td>\n",
              "      <td>737</td>\n",
              "      <td>16.429700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>ALOST</td>\n",
              "      <td>2017</td>\n",
              "      <td>892</td>\n",
              "      <td>21.031208</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2010</td>\n",
              "      <td>51</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2011</td>\n",
              "      <td>41</td>\n",
              "      <td>-19.607843</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2012</td>\n",
              "      <td>41</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2013</td>\n",
              "      <td>41</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2014</td>\n",
              "      <td>54</td>\n",
              "      <td>31.707317</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2015</td>\n",
              "      <td>15</td>\n",
              "      <td>-72.222222</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2016</td>\n",
              "      <td>30</td>\n",
              "      <td>100.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>ALVERINGEM</td>\n",
              "      <td>2017</td>\n",
              "      <td>45</td>\n",
              "      <td>50.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2010</td>\n",
              "      <td>123</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2011</td>\n",
              "      <td>113</td>\n",
              "      <td>-8.130081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2012</td>\n",
              "      <td>104</td>\n",
              "      <td>-7.964602</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2013</td>\n",
              "      <td>102</td>\n",
              "      <td>-1.923077</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2014</td>\n",
              "      <td>103</td>\n",
              "      <td>0.980392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2015</td>\n",
              "      <td>83</td>\n",
              "      <td>-19.417476</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2016</td>\n",
              "      <td>115</td>\n",
              "      <td>38.554217</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>AMAY</td>\n",
              "      <td>2017</td>\n",
              "      <td>122</td>\n",
              "      <td>6.086957</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2014</td>\n",
              "      <td>11</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2015</td>\n",
              "      <td>13</td>\n",
              "      <td>18.181818</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2016</td>\n",
              "      <td>17</td>\n",
              "      <td>30.769231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75</th>\n",
              "      <td>AMBLEVE</td>\n",
              "      <td>2017</td>\n",
              "      <td>15</td>\n",
              "      <td>-11.764706</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>76</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2010</td>\n",
              "      <td>177</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2011</td>\n",
              "      <td>192</td>\n",
              "      <td>8.474576</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>78</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2012</td>\n",
              "      <td>191</td>\n",
              "      <td>-0.520833</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>79</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2013</td>\n",
              "      <td>174</td>\n",
              "      <td>-8.900524</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>80</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2014</td>\n",
              "      <td>201</td>\n",
              "      <td>15.517241</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>81</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2015</td>\n",
              "      <td>186</td>\n",
              "      <td>-7.462687</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2016</td>\n",
              "      <td>172</td>\n",
              "      <td>-7.526882</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83</th>\n",
              "      <td>ANDENNE</td>\n",
              "      <td>2017</td>\n",
              "      <td>189</td>\n",
              "      <td>9.883721</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2010</td>\n",
              "      <td>380</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2011</td>\n",
              "      <td>401</td>\n",
              "      <td>5.526316</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2012</td>\n",
              "      <td>373</td>\n",
              "      <td>-6.982544</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2013</td>\n",
              "      <td>332</td>\n",
              "      <td>-10.991957</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2014</td>\n",
              "      <td>369</td>\n",
              "      <td>11.144578</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2015</td>\n",
              "      <td>298</td>\n",
              "      <td>-19.241192</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2016</td>\n",
              "      <td>293</td>\n",
              "      <td>-1.677852</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>91</th>\n",
              "      <td>ANDERLECHT</td>\n",
              "      <td>2017</td>\n",
              "      <td>307</td>\n",
              "      <td>4.778157</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2010</td>\n",
              "      <td>107</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>93</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2011</td>\n",
              "      <td>123</td>\n",
              "      <td>14.953271</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>94</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2012</td>\n",
              "      <td>91</td>\n",
              "      <td>-26.016260</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2013</td>\n",
              "      <td>71</td>\n",
              "      <td>-21.978022</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2014</td>\n",
              "      <td>83</td>\n",
              "      <td>16.901408</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2015</td>\n",
              "      <td>95</td>\n",
              "      <td>14.457831</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2016</td>\n",
              "      <td>99</td>\n",
              "      <td>4.210526</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>ANDERLUES</td>\n",
              "      <td>2017</td>\n",
              "      <td>85</td>\n",
              "      <td>-14.141414</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      CD_REFNIS_FR  CD_YEAR  MS_TOTAL_TRANSACTIONS  pct_change_transactions\n",
              "0           AALTER     2010                     70                 0.000000\n",
              "1           AALTER     2011                     69                -1.428571\n",
              "2           AALTER     2012                     60               -13.043478\n",
              "3           AALTER     2013                    100                66.666667\n",
              "4           AALTER     2014                     87               -13.000000\n",
              "5           AALTER     2015                     57               -34.482759\n",
              "6           AALTER     2016                     73                28.070175\n",
              "7           AALTER     2017                     85                16.438356\n",
              "8         AARSCHOT     2010                    138                 0.000000\n",
              "9         AARSCHOT     2011                    158                14.492754\n",
              "10        AARSCHOT     2012                    160                 1.265823\n",
              "11        AARSCHOT     2013                    168                 5.000000\n",
              "12        AARSCHOT     2014                    198                17.857143\n",
              "13        AARSCHOT     2015                    128               -35.353535\n",
              "14        AARSCHOT     2016                    145                13.281250\n",
              "15        AARSCHOT     2017                    167                15.172414\n",
              "16      AARTSELAAR     2010                     90                 0.000000\n",
              "17      AARTSELAAR     2011                     79               -12.222222\n",
              "18      AARTSELAAR     2012                     83                 5.063291\n",
              "19      AARTSELAAR     2013                     85                 2.409639\n",
              "20      AARTSELAAR     2014                     97                14.117647\n",
              "21      AARTSELAAR     2015                     76               -21.649485\n",
              "22      AARTSELAAR     2016                     78                 2.631579\n",
              "23      AARTSELAAR     2017                    108                38.461538\n",
              "24        AFFLIGEM     2010                     93                 0.000000\n",
              "25        AFFLIGEM     2011                     86                -7.526882\n",
              "26        AFFLIGEM     2012                     98                13.953488\n",
              "27        AFFLIGEM     2013                     89                -9.183673\n",
              "28        AFFLIGEM     2014                     85                -4.494382\n",
              "29        AFFLIGEM     2015                     66               -22.352941\n",
              "30        AFFLIGEM     2016                     97                46.969697\n",
              "31        AFFLIGEM     2017                     81               -16.494845\n",
              "32  AISEAU-PRESLES     2010                     90                 0.000000\n",
              "33  AISEAU-PRESLES     2011                     76               -15.555556\n",
              "34  AISEAU-PRESLES     2012                     78                 2.631579\n",
              "35  AISEAU-PRESLES     2013                     61               -21.794872\n",
              "36  AISEAU-PRESLES     2014                     54               -11.475410\n",
              "37  AISEAU-PRESLES     2015                     75                38.888889\n",
              "38  AISEAU-PRESLES     2016                     75                 0.000000\n",
              "39  AISEAU-PRESLES     2017                     75                 0.000000\n",
              "40           ALKEN     2010                     39                 0.000000\n",
              "41           ALKEN     2011                     38                -2.564103\n",
              "42           ALKEN     2012                     45                18.421053\n",
              "43           ALKEN     2013                     36               -20.000000\n",
              "44           ALKEN     2014                     34                -5.555556\n",
              "45           ALKEN     2015                     40                17.647059\n",
              "46           ALKEN     2016                     41                 2.500000\n",
              "47           ALKEN     2017                     54                31.707317\n",
              "48           ALOST     2010                    738                 0.000000\n",
              "49           ALOST     2011                    813                10.162602\n",
              "50           ALOST     2012                    766                -5.781058\n",
              "51           ALOST     2013                    736                -3.916449\n",
              "52           ALOST     2014                    855                16.168478\n",
              "53           ALOST     2015                    633               -25.964912\n",
              "54           ALOST     2016                    737                16.429700\n",
              "55           ALOST     2017                    892                21.031208\n",
              "56      ALVERINGEM     2010                     51                 0.000000\n",
              "57      ALVERINGEM     2011                     41               -19.607843\n",
              "58      ALVERINGEM     2012                     41                 0.000000\n",
              "59      ALVERINGEM     2013                     41                 0.000000\n",
              "60      ALVERINGEM     2014                     54                31.707317\n",
              "61      ALVERINGEM     2015                     15               -72.222222\n",
              "62      ALVERINGEM     2016                     30               100.000000\n",
              "63      ALVERINGEM     2017                     45                50.000000\n",
              "64            AMAY     2010                    123                 0.000000\n",
              "65            AMAY     2011                    113                -8.130081\n",
              "66            AMAY     2012                    104                -7.964602\n",
              "67            AMAY     2013                    102                -1.923077\n",
              "68            AMAY     2014                    103                 0.980392\n",
              "69            AMAY     2015                     83               -19.417476\n",
              "70            AMAY     2016                    115                38.554217\n",
              "71            AMAY     2017                    122                 6.086957\n",
              "72         AMBLEVE     2014                     11                 0.000000\n",
              "73         AMBLEVE     2015                     13                18.181818\n",
              "74         AMBLEVE     2016                     17                30.769231\n",
              "75         AMBLEVE     2017                     15               -11.764706\n",
              "76         ANDENNE     2010                    177                 0.000000\n",
              "77         ANDENNE     2011                    192                 8.474576\n",
              "78         ANDENNE     2012                    191                -0.520833\n",
              "79         ANDENNE     2013                    174                -8.900524\n",
              "80         ANDENNE     2014                    201                15.517241\n",
              "81         ANDENNE     2015                    186                -7.462687\n",
              "82         ANDENNE     2016                    172                -7.526882\n",
              "83         ANDENNE     2017                    189                 9.883721\n",
              "84      ANDERLECHT     2010                    380                 0.000000\n",
              "85      ANDERLECHT     2011                    401                 5.526316\n",
              "86      ANDERLECHT     2012                    373                -6.982544\n",
              "87      ANDERLECHT     2013                    332               -10.991957\n",
              "88      ANDERLECHT     2014                    369                11.144578\n",
              "89      ANDERLECHT     2015                    298               -19.241192\n",
              "90      ANDERLECHT     2016                    293                -1.677852\n",
              "91      ANDERLECHT     2017                    307                 4.778157\n",
              "92       ANDERLUES     2010                    107                 0.000000\n",
              "93       ANDERLUES     2011                    123                14.953271\n",
              "94       ANDERLUES     2012                     91               -26.016260\n",
              "95       ANDERLUES     2013                     71               -21.978022\n",
              "96       ANDERLUES     2014                     83                16.901408\n",
              "97       ANDERLUES     2015                     95                14.457831\n",
              "98       ANDERLUES     2016                     99                 4.210526\n",
              "99       ANDERLUES     2017                     85               -14.141414"
            ]
          },
          "execution_count": 53,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pd.set_option('display.max_rows', 100)\n",
        "df_grouped_houses_totalTransactions = df_grouped_houses_totalTransactions.fillna(0)\n",
        "df_grouped_houses_totalTransactions.head(100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "df_grouped_houses_price_per_surface"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "#### Validation Split\n",
        "\n",
        "The Validation Split is where you partition your data into a\n",
        "training set, and a validation set (and sometimes a\n",
        "test set as well).\n",
        "You train the model with the training set only. The\n",
        "validation and/or test sets, are held-back from training and\n",
        "are used to assess model performance. They provide a\n",
        "true understanding of how accurate predictions are on\n",
        "new or unseen data.\n",
        "An approach called k-fold cross validation can provide\n",
        "you with an even more robust understanding of model\n",
        "performance.\n",
        "Here, the entire dataset is again partitioned into training &\n",
        "validation sets, and the model is trained and assessed like\n",
        "before. However, this process is done multiple times with\n",
        "the training and test sets being rotated to encompass\n",
        "different sets of observations within the data. You do this\n",
        "k times - with your final predictive accuracy assessment\n",
        "being based upon the average of each of the iterations."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "In general, validation split is not needed for forecasting problems. Forecasting is the process of predicting future values of a time series based on its past values. In this context, the goal is to accurately predict future values, not to evaluate the performance of a model on a holdout dataset."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "### Summary  \n",
        "\n",
        "In this project we were able to take a dirty dataset and go step by step on a data cleaning and preparation journey.\n",
        "We introduced CRISP-DM to explain the different phases of a data science project, but we focused on data understanding and data preparation phases.\n",
        "For Data preparation we used the 8 step model defined by Data Science Infinity which clearly explains the steps that are usually done on data cleaning scenario.\n",
        "\n",
        "### Further Steps\n",
        "\n",
        "With the dataframes already cleaned, we could do the following:\n",
        "\n",
        "1. Create line charts to see the Nr of Transactions per year per municipality.\n",
        "2. Do the same but use the % change for Transactions and price per square meter, this will allow the users to see where the square meter prices are increasing faster.\n",
        "3. Others, send your feedback"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "python310-sdkv2"
    },
    "kernelspec": {
      "display_name": "Python 3.10.6 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      }
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    },
    "vscode": {
      "interpreter": {
        "hash": "26de051ba29f2982a8de78e945f0abaf191376122a1563185a90213a26c5da77"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}