data_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cac3589",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn import preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae52a9fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/raw_data_cleaned.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b8e7342",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={'Unnamed: 0': 'Food Name'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c7d8ce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9887c5a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in ['Plain Occurences', 'Cool Occurences', 'Warm Occurences',\n",
    "       'Cold Occurences', 'Heavy Cold Occurences', 'Heavy Warm Occurences',\n",
    "       'Hot Occurences', 'Heavy Hot Occurences']:\n",
    "    print(df[col].value_counts().sort_values(ascending=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f1835ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.isnull().sum().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09ac4576",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f54ab7c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cbb66fa",
   "metadata": {},
   "source": [
    "## Data Process"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f393f2e",
   "metadata": {},
   "source": [
    "### Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4b6a4ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41d06957",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(df):\n",
    "    df.loc[\"Water (g)\": \"pH\"] = (df.loc[\"Water (g)\": \"pH\"]-df.loc[\"Water (g)\": \"pH\"].mean())/df.loc[\"Water (g)\": \"pH\"].std()\n",
    "    \n",
    "    name = df.iloc[:, 0:1]\n",
    "    inputs = df.loc[:, 'Water (g)':'pH'].astype(float)\n",
    "    outputs = df.loc[:, 'pH':].drop(columns='pH')\n",
    "\n",
    "    # normalized_inputs = (inputs-inputs.mean())/inputs.std()\n",
    "    normalized_inputs=(inputs-inputs.min())/(inputs.max()-inputs.min())\n",
    "    \n",
    "    df = pd.concat([name, normalized_inputs, outputs], axis=1)\n",
    "    return df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d2760df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# new\n",
    "def normalize_data(df):\n",
    "    X = df.drop(['Unnamed: 0.1','Unnamed: 0'],axis=1)\n",
    "    scaler = MinMaxScaler()\n",
    "    scaler.fit(X)\n",
    "    X_array = scaler.transform(X)\n",
    "    new_data = pd.DataFrame(X_array,columns = X.columns)\n",
    "    new_data['food_names'] = df['Unnamed: 0.1']\n",
    "    \n",
    "    return new_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25bb50c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[\"Water (g)\": \"pH\"] = (df.loc[\"Water (g)\": \"pH\"]-df.loc[\"Water (g)\": \"pH\"].mean())/df.loc[\"Water (g)\": \"pH\"].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bade459",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = df.iloc[:, 0:1]\n",
    "inputs = df.loc[:, 'Water (g)':'pH'].astype(float)\n",
    "outputs = df.loc[:, 'pH':].drop(columns='pH')\n",
    "\n",
    "# normalized_inputs = (inputs-inputs.mean())/inputs.std()\n",
    "normalized_inputs=(inputs-inputs.min())/(inputs.max()-inputs.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2a38ac0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([name, normalized_inputs, outputs], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c21d54c1",
   "metadata": {},
   "source": [
    "### Y Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b960268",
   "metadata": {},
   "outputs": [],
   "source": [
    "def occurrences(df):\n",
    "    df[\"% Plain\"] = df[\"Plain Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    df[\"% Cool\"] = df[\"Cool Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    df[\"% Warm\"] = df[\"Warm Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    df[\"% Cold\"] = df[\"Cold Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    df[\"% Heavy Cold\"] = df[\"Heavy Cold Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    df[\"% Heavy Warm\"] = df[\"Heavy Warm Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    df[\"% Hot\"] = df[\"Hot Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    df[\"% Heavy Hot\"] = df[\"Heavy Hot Occurences\"]/(df[\"Plain Occurences\"] + df[\"Cool Occurences\"] + df[\"Warm Occurences\"] + df[\"Cold Occurences\"] + df[\"Heavy Cold Occurences\"] + df[\"Heavy Warm Occurences\"] + df[\"Hot Occurences\"] + df[\"Heavy Hot Occurences\"])\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "910535f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_scale_v1(df):\n",
    "    df[\"hot_cold_scale\"] = (df[\"% Plain\"] * (3.0/7.0)) + (df[\"% Cool\"] * (2.0/7.0)) + (df[\"% Warm\"] * (4.0/7.0)) + (df[\"% Cold\"] * (1.0/7.0)) + (df[\"% Heavy Cold\"] * (0.0/7.0)) + (df[\"% Heavy Warm\"] * (5.0/7.0)) + (df[\"% Hot\"] * (6.0/7.0)) + (df[\"% Heavy Hot\"] * (7.0/7.0))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "525c4bf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_scale(df):\n",
    "#     df[\"hot_cold_scale\"] = (df[\"% Plain\"] * (3.0/7.0)) + (df[\"% Cool\"] * (2.0/7.0)) + (df[\"% Warm\"] * (4.0/7.0)) + (df[\"% Cold\"] * (1.0/7.0)) + (df[\"% Heavy Cold\"] * (0.0/7.0)) + (df[\"% Heavy Warm\"] * (5.0/7.0)) + (df[\"% Hot\"] * (6.0/7.0)) + (df[\"% Heavy Hot\"] * (7.0/7.0))\n",
    "    df[\"hot_cold_scale\"] = (df[\"% Plain\"] * (3.0/6.0)) + (df[\"% Cool\"] * (2.0/6.0)) + (df[\"% Warm\"] * (4.0/6.0)) + (df[\"% Cold\"] * (1.0/6.0)) + (df[\"% Heavy Cold\"] * (0.0/6.0)) + (df[\"% Heavy Warm\"] * (4.5/6.0)) + (df[\"% Hot\"] * (5.0/6.0)) + (df[\"% Heavy Hot\"] * (6.0/6.0))\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2970bc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = occurrences(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10efbed4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mode_old(df):\n",
    "    newDf = pd.DataFrame(df.iloc[:, 39:47].idxmax(axis=1).str.rstrip(\"Occurences\"))\n",
    "    df[\"Mode\"] = newDf[0]\n",
    "    return df\n",
    "\n",
    "def mode(df):\n",
    "    df['Plain'] = df['Plain Occurences'].astype(int)\n",
    "    df['Cold'] = df['Cool Occurences'].astype(int) + df['Cold Occurences'].astype(int) + df['Heavy Cold Occurences'].astype(int)\n",
    "    df['Hot'] = df['Warm Occurences'].astype(int) + df['Heavy Warm Occurences'].astype(int) + df['Hot Occurences'].astype(int) + df['Heavy Hot Occurences'].astype(int)\n",
    "    newDf = pd.DataFrame(df.loc[:, ['Plain', 'Cold', 'Hot']].idxmax(axis=1))\n",
    "    df[\"Mode\"] = newDf[0]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f9b3dbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = linear_scale(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf1e15f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = mode(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56bffca3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83f2e8f8",
   "metadata": {},
   "source": [
    "### NaN Values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30a8bb60",
   "metadata": {},
   "source": [
    "Iterative Imputation\n",
    "https://machinelearningmastery.com/iterative-imputation-for-missing-values-in-machine-learning/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25536f82",
   "metadata": {},
   "outputs": [],
   "source": [
    "def nanValuesRid(df):\n",
    "    not_too_many_null_cols = df.columns[df.isnull().mean() < 0.3]\n",
    "    df = df[not_too_many_null_cols]\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e780d5b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02b78f77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def iterative_imputation(df):\n",
    "    ii_imp = IterativeImputer(estimator=ExtraTreesRegressor(), max_iter=10, random_state=1121218)\n",
    "\n",
    "    name = df.iloc[:, 0:1]\n",
    "    inputs = df.loc[:, 'Water (g)':'pH']\n",
    "    outputs = df.loc[:, 'pH':].drop(columns='pH')\n",
    "    \n",
    "    not_too_many_null_cols = df.columns[df.isnull().mean() < 0.3]\n",
    "    \n",
    "    inputs = ii_imp.fit_transform(inputs)\n",
    "    new_inputs = pd.DataFrame(inputs, columns = not_too_many_null_cols[1:35]) \n",
    "    \n",
    "    return pd.concat([name, new_inputs, outputs], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24861d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_encoder(df):\n",
    "    # reading word labels\n",
    "    label_encoder = preprocessing.LabelEncoder()\n",
    "\n",
    "    # encode word labels in column\n",
    "    df['hot_cold_scale'] = label_encoder.fit_transform(df['hot_cold_scale'])\n",
    "\n",
    "    df['hot_cold_scale'].unique()\n",
    "\n",
    "    return df\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e89f3b3",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c087cd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/raw_data_cleaned.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45cb3d2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a9688a0",
   "metadata": {},
   "source": [
    "### Run All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "807709bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_process(df):\n",
    "    df = normalize(df)\n",
    "    df = occurrences(df)\n",
    "    df = linear_scale(df)\n",
    "    df = mode(df)\n",
    "    \n",
    "    df = iterative_imputation(df)\n",
    "    \n",
    "    df = label_encoder(df)\n",
    "    \n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51a4eafa",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df = data_process(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3182e07f",
   "metadata": {},
   "source": [
    "## Save Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "238cdec0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a8950ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#just keep adjust code and re-running and overwriting this file, to then use in pipeline to see results\n",
    "df.to_csv('./data/data_processed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dca7d6d4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}